[gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs

[gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs

[Michał Górny]

---
 pym/portage/sync/modules/rsync/rsync.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 4471f5bbe..ec28af366 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -92,6 +92,15 @@ class RsyncSync(NewBase):
 		# Support overriding job count.
 		self.verify_jobs = self.repo.module_specific_options.get(
 				'sync-rsync-verify-jobs', None)
+		if self.verify_jobs is not None:
+			try:
+				self.verify_jobs = int(self.verify_jobs)
+				if self.verify_jobs <= 0:
+					raise ValueError(self.verify_jobs)
+			except ValueError:
+				writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
+					level=logging.WARNING, noiselevel=-1)
+				self.verify_jobs = None
 
 		# Real local timestamp file.
 		self.servertimestampfile = os.path.join(
@@ -280,7 +289,7 @@ class RsyncSync(NewBase):
 			if self.repo.sync_openpgp_key_path is not None:
 				command += ['-K', self.repo.sync_openpgp_key_path]
 			if self.verify_jobs is not None:
-				command += ['-j', self.verify_jobs]
+				command += ['-j', str(self.verify_jobs)]
 			try:
 				exitcode = portage.process.spawn(command, **self.spawn_kwargs)
 			except CommandNotFound as e:
-- 
2.16.1

[gentoo-portage-dev] [PATCH v2 2/9] rsync: Use gemato routines directly instead of calling the CLI tool

[Michał Górny]

---
 pym/portage/sync/modules/rsync/rsync.py | 66 ++++++++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 9 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index ec28af366..39c4066d8 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -6,6 +6,7 @@ import logging
 import time
 import signal
 import socket
+import io
 import re
 import random
 import tempfile
@@ -25,6 +26,13 @@ from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import NewBase
 
+try:
+	from gemato.exceptions import GematoException
+	import gemato.openpgp
+	import gemato.recursiveloader
+except ImportError:
+	gemato = None
+
 if sys.hexversion >= 0x3000000:
 	# pylint: disable=W0622
 	_unicode = str
@@ -285,17 +293,57 @@ class RsyncSync(NewBase):
 
 		# if synced successfully, verify now
 		if exitcode == 0 and not local_state_unchanged and self.verify_metamanifest:
-			command = ['gemato', 'verify', '-s', self.repo.location]
-			if self.repo.sync_openpgp_key_path is not None:
-				command += ['-K', self.repo.sync_openpgp_key_path]
-			if self.verify_jobs is not None:
-				command += ['-j', str(self.verify_jobs)]
-			try:
-				exitcode = portage.process.spawn(command, **self.spawn_kwargs)
-			except CommandNotFound as e:
-				writemsg_level("!!! Command not found: %s\n" % (command[0],),
+			if gemato is None:
+				writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
 					level=logging.ERROR, noiselevel=-1)
 				exitcode = 127
+			else:
+				# Use isolated environment if key is specified,
+				# system environment otherwise
+				if self.repo.sync_openpgp_key_path is not None:
+					openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+				else:
+					openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+
+				try:
+					with openpgp_env_cls() as openpgp_env:
+						if self.repo.sync_openpgp_key_path is not None:
+							out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+							with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+								openpgp_env.import_key(f)
+							out.ebegin('Refreshing keys from keyserver')
+							openpgp_env.refresh_keys()
+							out.eend(0)
+
+						m = gemato.recursiveloader.ManifestRecursiveLoader(
+								os.path.join(self.repo.location, 'Manifest'),
+								verify_openpgp=True,
+								openpgp_env=openpgp_env,
+								max_jobs=self.verify_jobs)
+						if not m.openpgp_signed:
+							raise RuntimeError('OpenPGP signature not found on Manifest')
+
+						ts = m.find_timestamp()
+						if ts is None:
+							raise RuntimeError('Timestamp not found in Manifest')
+
+						out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+						out.einfo('Valid OpenPGP signature found:')
+						out.einfo('- primary key: %s' % (
+							m.openpgp_signature.primary_key_fingerprint))
+						out.einfo('- subkey: %s' % (
+							m.openpgp_signature.fingerprint))
+						out.einfo('- timestamp: %s UTC' % (
+							m.openpgp_signature.timestamp))
+
+						out.ebegin('Verifying %s' % (self.repo.location,))
+						m.assert_directory_verifies()
+						out.eend(0)
+				except GematoException as e:
+					writemsg_level("!!! Manifest verification failed:\n%s\n"
+							% (e,),
+							level=logging.ERROR, noiselevel=-1)
+					exitcode = 1
 
 		return (exitcode, updatecache_flg)
 
-- 
2.16.1

[gentoo-portage-dev] [PATCH v2 3/9] rsync: Verify the Manifest signature even if tree is unchanged

[Michał Górny]

Always verify the Manifest signature if verification is enabled.
Skipping the deep tree verification for unchanged case is reasonable
but we need to make sure the Manifest signature stays valid to catch
the case of the signing key being revoked.
---
 pym/portage/sync/modules/rsync/rsync.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 39c4066d8..e6e218868 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -292,7 +292,7 @@ class RsyncSync(NewBase):
 		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
 
 		# if synced successfully, verify now
-		if exitcode == 0 and not local_state_unchanged and self.verify_metamanifest:
+		if exitcode == 0 and self.verify_metamanifest:
 			if gemato is None:
 				writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
 					level=logging.ERROR, noiselevel=-1)
@@ -315,6 +315,8 @@ class RsyncSync(NewBase):
 							openpgp_env.refresh_keys()
 							out.eend(0)
 
+						# we always verify the Manifest signature, in case
+						# we had to deal with key revocation case
 						m = gemato.recursiveloader.ManifestRecursiveLoader(
 								os.path.join(self.repo.location, 'Manifest'),
 								verify_openpgp=True,
@@ -336,9 +338,12 @@ class RsyncSync(NewBase):
 						out.einfo('- timestamp: %s UTC' % (
 							m.openpgp_signature.timestamp))
 
-						out.ebegin('Verifying %s' % (self.repo.location,))
-						m.assert_directory_verifies()
-						out.eend(0)
+						# if nothing has changed, skip the actual Manifest
+						# verification
+						if not local_state_unchanged:
+							out.ebegin('Verifying %s' % (self.repo.location,))
+							m.assert_directory_verifies()
+							out.eend(0)
 				except GematoException as e:
 					writemsg_level("!!! Manifest verification failed:\n%s\n"
 							% (e,),
-- 
2.16.1

[gentoo-portage-dev] [PATCH v2 4/9] rsync: Pre-indent the try-finally block for gemato key scope

[Michał Górny]

---
 pym/portage/sync/modules/rsync/rsync.py | 467 ++++++++++++++++----------------
 1 file changed, 235 insertions(+), 232 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index e6e218868..5c0b53f9e 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -110,247 +110,250 @@ class RsyncSync(NewBase):
 					level=logging.WARNING, noiselevel=-1)
 				self.verify_jobs = None
 
-		# Real local timestamp file.
-		self.servertimestampfile = os.path.join(
-			self.repo.location, "metadata", "timestamp.chk")
-
-		content = portage.util.grabfile(self.servertimestampfile)
-		timestamp = 0
-		if content:
-			try:
-				timestamp = time.mktime(time.strptime(content[0],
-					TIMESTAMP_FORMAT))
-			except (OverflowError, ValueError):
-				pass
-		del content
-
-		try:
-			self.rsync_initial_timeout = \
-				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
-		except ValueError:
-			self.rsync_initial_timeout = 15
-
-		try:
-			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
-		except SystemExit as e:
-			raise # Needed else can't exit
-		except:
-			maxretries = -1 #default number of retries
-
-		if syncuri.startswith("file://"):
-			self.proto = "file"
-			dosyncuri = syncuri[7:]
-			unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
-				dosyncuri, timestamp, opts)
-			self._process_exitcode(exitcode, dosyncuri, out, 1)
-			return (exitcode, updatecache_flg)
-
-		retries=0
 		try:
-			self.proto, user_name, hostname, port = re.split(
-				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
-				syncuri, maxsplit=4)[1:5]
-		except ValueError:
-			writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
-				noiselevel=-1, level=logging.ERROR)
-			return (1, False)
+			# Real local timestamp file.
+			self.servertimestampfile = os.path.join(
+				self.repo.location, "metadata", "timestamp.chk")
 
-		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+			content = portage.util.grabfile(self.servertimestampfile)
+			timestamp = 0
+			if content:
+				try:
+					timestamp = time.mktime(time.strptime(content[0],
+						TIMESTAMP_FORMAT))
+				except (OverflowError, ValueError):
+					pass
+			del content
 
-		if port is None:
-			port=""
-		if user_name is None:
-			user_name=""
-		if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
-			getaddrinfo_host = hostname
-		else:
-			# getaddrinfo needs the brackets stripped
-			getaddrinfo_host = hostname[1:-1]
-		updatecache_flg = False
-		all_rsync_opts = set(self.rsync_opts)
-		all_rsync_opts.update(self.extra_rsync_opts)
+			try:
+				self.rsync_initial_timeout = \
+					int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
+			except ValueError:
+				self.rsync_initial_timeout = 15
 
-		family = socket.AF_UNSPEC
-		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
-			family = socket.AF_INET
-		elif socket.has_ipv6 and \
-			("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
-			family = socket.AF_INET6
+			try:
+				maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
+			except SystemExit as e:
+				raise # Needed else can't exit
+			except:
+				maxretries = -1 #default number of retries
+
+			if syncuri.startswith("file://"):
+				self.proto = "file"
+				dosyncuri = syncuri[7:]
+				unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
+					dosyncuri, timestamp, opts)
+				self._process_exitcode(exitcode, dosyncuri, out, 1)
+				return (exitcode, updatecache_flg)
+
+			retries=0
+			try:
+				self.proto, user_name, hostname, port = re.split(
+					r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
+					syncuri, maxsplit=4)[1:5]
+			except ValueError:
+				writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
+					noiselevel=-1, level=logging.ERROR)
+				return (1, False)
 
-		addrinfos = None
-		uris = []
+			self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
-		try:
-			addrinfos = getaddrinfo_validate(
-				socket.getaddrinfo(getaddrinfo_host, None,
-				family, socket.SOCK_STREAM))
-		except socket.error as e:
-			writemsg_level(
-				"!!! getaddrinfo failed for '%s': %s\n"
-				% (_unicode_decode(hostname), _unicode(e)),
-				noiselevel=-1, level=logging.ERROR)
-
-		if addrinfos:
-
-			AF_INET = socket.AF_INET
-			AF_INET6 = None
-			if socket.has_ipv6:
-				AF_INET6 = socket.AF_INET6
-
-			ips_v4 = []
-			ips_v6 = []
-
-			for addrinfo in addrinfos:
-				if addrinfo[0] == AF_INET:
-					ips_v4.append("%s" % addrinfo[4][0])
-				elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
-					# IPv6 addresses need to be enclosed in square brackets
-					ips_v6.append("[%s]" % addrinfo[4][0])
-
-			random.shuffle(ips_v4)
-			random.shuffle(ips_v6)
-
-			# Give priority to the address family that
-			# getaddrinfo() returned first.
-			if AF_INET6 is not None and addrinfos and \
-				addrinfos[0][0] == AF_INET6:
-				ips = ips_v6 + ips_v4
-			else:
-				ips = ips_v4 + ips_v6
-
-			for ip in ips:
-				uris.append(syncuri.replace(
-					"//" + user_name + hostname + port + "/",
-					"//" + user_name + ip + port + "/", 1))
-
-		if not uris:
-			# With some configurations we need to use the plain hostname
-			# rather than try to resolve the ip addresses (bug #340817).
-			uris.append(syncuri)
-
-		# reverse, for use with pop()
-		uris.reverse()
-		uris_orig = uris[:]
-
-		effective_maxretries = maxretries
-		if effective_maxretries < 0:
-			effective_maxretries = len(uris) - 1
-
-		local_state_unchanged = True
-		while (1):
-			if uris:
-				dosyncuri = uris.pop()
-			elif maxretries < 0 or retries > maxretries:
-				writemsg("!!! Exhausted addresses for %s\n"
-					% _unicode_decode(hostname), noiselevel=-1)
-				return (1, False)
-			else:
-				uris.extend(uris_orig)
-				dosyncuri = uris.pop()
-
-			if (retries==0):
-				if "--ask" in opts:
-					uq = UserQuery(opts)
-					if uq.query("Do you want to sync your Portage tree " + \
-						"with the mirror at\n" + blue(dosyncuri) + bold("?"),
-						enter_invalid) == "No":
-						print()
-						print("Quitting.")
-						print()
-						sys.exit(128 + signal.SIGINT)
-				self.logger(self.xterm_titles,
-					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in opts:
-					print(">>> Starting rsync with "+dosyncuri+"...")
-			else:
-				self.logger(self.xterm_titles,
-					">>> Starting retry %d of %d with %s" % \
-						(retries, effective_maxretries, dosyncuri))
-				writemsg_stdout(
-					"\n\n>>> Starting retry %d of %d with %s\n" % \
-					(retries, effective_maxretries, dosyncuri), noiselevel=-1)
-
-			if dosyncuri.startswith('ssh://'):
-				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
-
-			unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
-				dosyncuri, timestamp, opts)
-			if not unchanged:
-				local_state_unchanged = False
-			if is_synced:
-				break
-
-			retries=retries+1
-
-			if maxretries < 0 or retries <= maxretries:
-				print(">>> Retrying...")
-			else:
-				# over retries
-				# exit loop
-				exitcode = EXCEEDED_MAX_RETRIES
-				break
-		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
-
-		# if synced successfully, verify now
-		if exitcode == 0 and self.verify_metamanifest:
-			if gemato is None:
-				writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
-					level=logging.ERROR, noiselevel=-1)
-				exitcode = 127
+			if port is None:
+				port=""
+			if user_name is None:
+				user_name=""
+			if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
+				getaddrinfo_host = hostname
 			else:
-				# Use isolated environment if key is specified,
-				# system environment otherwise
-				if self.repo.sync_openpgp_key_path is not None:
-					openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+				# getaddrinfo needs the brackets stripped
+				getaddrinfo_host = hostname[1:-1]
+			updatecache_flg = False
+			all_rsync_opts = set(self.rsync_opts)
+			all_rsync_opts.update(self.extra_rsync_opts)
+
+			family = socket.AF_UNSPEC
+			if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
+				family = socket.AF_INET
+			elif socket.has_ipv6 and \
+				("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
+				family = socket.AF_INET6
+
+			addrinfos = None
+			uris = []
+
+			try:
+				addrinfos = getaddrinfo_validate(
+					socket.getaddrinfo(getaddrinfo_host, None,
+					family, socket.SOCK_STREAM))
+			except socket.error as e:
+				writemsg_level(
+					"!!! getaddrinfo failed for '%s': %s\n"
+					% (_unicode_decode(hostname), _unicode(e)),
+					noiselevel=-1, level=logging.ERROR)
+
+			if addrinfos:
+
+				AF_INET = socket.AF_INET
+				AF_INET6 = None
+				if socket.has_ipv6:
+					AF_INET6 = socket.AF_INET6
+
+				ips_v4 = []
+				ips_v6 = []
+
+				for addrinfo in addrinfos:
+					if addrinfo[0] == AF_INET:
+						ips_v4.append("%s" % addrinfo[4][0])
+					elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
+						# IPv6 addresses need to be enclosed in square brackets
+						ips_v6.append("[%s]" % addrinfo[4][0])
+
+				random.shuffle(ips_v4)
+				random.shuffle(ips_v6)
+
+				# Give priority to the address family that
+				# getaddrinfo() returned first.
+				if AF_INET6 is not None and addrinfos and \
+					addrinfos[0][0] == AF_INET6:
+					ips = ips_v6 + ips_v4
+				else:
+					ips = ips_v4 + ips_v6
+
+				for ip in ips:
+					uris.append(syncuri.replace(
+						"//" + user_name + hostname + port + "/",
+						"//" + user_name + ip + port + "/", 1))
+
+			if not uris:
+				# With some configurations we need to use the plain hostname
+				# rather than try to resolve the ip addresses (bug #340817).
+				uris.append(syncuri)
+
+			# reverse, for use with pop()
+			uris.reverse()
+			uris_orig = uris[:]
+
+			effective_maxretries = maxretries
+			if effective_maxretries < 0:
+				effective_maxretries = len(uris) - 1
+
+			local_state_unchanged = True
+			while (1):
+				if uris:
+					dosyncuri = uris.pop()
+				elif maxretries < 0 or retries > maxretries:
+					writemsg("!!! Exhausted addresses for %s\n"
+						% _unicode_decode(hostname), noiselevel=-1)
+					return (1, False)
+				else:
+					uris.extend(uris_orig)
+					dosyncuri = uris.pop()
+
+				if (retries==0):
+					if "--ask" in opts:
+						uq = UserQuery(opts)
+						if uq.query("Do you want to sync your Portage tree " + \
+							"with the mirror at\n" + blue(dosyncuri) + bold("?"),
+							enter_invalid) == "No":
+							print()
+							print("Quitting.")
+							print()
+							sys.exit(128 + signal.SIGINT)
+					self.logger(self.xterm_titles,
+						">>> Starting rsync with " + dosyncuri)
+					if "--quiet" not in opts:
+						print(">>> Starting rsync with "+dosyncuri+"...")
+				else:
+					self.logger(self.xterm_titles,
+						">>> Starting retry %d of %d with %s" % \
+							(retries, effective_maxretries, dosyncuri))
+					writemsg_stdout(
+						"\n\n>>> Starting retry %d of %d with %s\n" % \
+						(retries, effective_maxretries, dosyncuri), noiselevel=-1)
+
+				if dosyncuri.startswith('ssh://'):
+					dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
+
+				unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
+					dosyncuri, timestamp, opts)
+				if not unchanged:
+					local_state_unchanged = False
+				if is_synced:
+					break
+
+				retries=retries+1
+
+				if maxretries < 0 or retries <= maxretries:
+					print(">>> Retrying...")
 				else:
-					openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+					# over retries
+					# exit loop
+					exitcode = EXCEEDED_MAX_RETRIES
+					break
+			self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+
+			# if synced successfully, verify now
+			if exitcode == 0 and self.verify_metamanifest:
+				if gemato is None:
+					writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
+						level=logging.ERROR, noiselevel=-1)
+					exitcode = 127
+				else:
+					# Use isolated environment if key is specified,
+					# system environment otherwise
+					if self.repo.sync_openpgp_key_path is not None:
+						openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+					else:
+						openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+
+					try:
+						with openpgp_env_cls() as openpgp_env:
+							if self.repo.sync_openpgp_key_path is not None:
+								out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+								with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+									openpgp_env.import_key(f)
+								out.ebegin('Refreshing keys from keyserver')
+								openpgp_env.refresh_keys()
+								out.eend(0)
+
+							# we always verify the Manifest signature, in case
+							# we had to deal with key revocation case
+							m = gemato.recursiveloader.ManifestRecursiveLoader(
+									os.path.join(self.repo.location, 'Manifest'),
+									verify_openpgp=True,
+									openpgp_env=openpgp_env,
+									max_jobs=self.verify_jobs)
+							if not m.openpgp_signed:
+								raise RuntimeError('OpenPGP signature not found on Manifest')
+
+							ts = m.find_timestamp()
+							if ts is None:
+								raise RuntimeError('Timestamp not found in Manifest')
+
+							out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+							out.einfo('Valid OpenPGP signature found:')
+							out.einfo('- primary key: %s' % (
+								m.openpgp_signature.primary_key_fingerprint))
+							out.einfo('- subkey: %s' % (
+								m.openpgp_signature.fingerprint))
+							out.einfo('- timestamp: %s UTC' % (
+								m.openpgp_signature.timestamp))
+
+							# if nothing has changed, skip the actual Manifest
+							# verification
+							if not local_state_unchanged:
+								out.ebegin('Verifying %s' % (self.repo.location,))
+								m.assert_directory_verifies()
+								out.eend(0)
+					except GematoException as e:
+						writemsg_level("!!! Manifest verification failed:\n%s\n"
+								% (e,),
+								level=logging.ERROR, noiselevel=-1)
+						exitcode = 1
 
-				try:
-					with openpgp_env_cls() as openpgp_env:
-						if self.repo.sync_openpgp_key_path is not None:
-							out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
-							with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
-								openpgp_env.import_key(f)
-							out.ebegin('Refreshing keys from keyserver')
-							openpgp_env.refresh_keys()
-							out.eend(0)
-
-						# we always verify the Manifest signature, in case
-						# we had to deal with key revocation case
-						m = gemato.recursiveloader.ManifestRecursiveLoader(
-								os.path.join(self.repo.location, 'Manifest'),
-								verify_openpgp=True,
-								openpgp_env=openpgp_env,
-								max_jobs=self.verify_jobs)
-						if not m.openpgp_signed:
-							raise RuntimeError('OpenPGP signature not found on Manifest')
-
-						ts = m.find_timestamp()
-						if ts is None:
-							raise RuntimeError('Timestamp not found in Manifest')
-
-						out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
-						out.einfo('Valid OpenPGP signature found:')
-						out.einfo('- primary key: %s' % (
-							m.openpgp_signature.primary_key_fingerprint))
-						out.einfo('- subkey: %s' % (
-							m.openpgp_signature.fingerprint))
-						out.einfo('- timestamp: %s UTC' % (
-							m.openpgp_signature.timestamp))
-
-						# if nothing has changed, skip the actual Manifest
-						# verification
-						if not local_state_unchanged:
-							out.ebegin('Verifying %s' % (self.repo.location,))
-							m.assert_directory_verifies()
-							out.eend(0)
-				except GematoException as e:
-					writemsg_level("!!! Manifest verification failed:\n%s\n"
-							% (e,),
-							level=logging.ERROR, noiselevel=-1)
-					exitcode = 1
-
-		return (exitcode, updatecache_flg)
+			return (exitcode, updatecache_flg)
+		finally:
+			pass
 
 
 	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
-- 
2.16.1

[gentoo-portage-dev] [PATCH v2 5/9] rsync: Load and update keys early

[Michał Górny]

Load and update keys early to avoid delaying failures post rsync. Any
failure will prevent verification from happening, and presumably most of
the users will prefer fixing it and trying to sync again. For that case,
it is better to perform the task before actual rsync to avoid
unnecessarily rsyncing twice.
---
 pym/portage/sync/modules/rsync/rsync.py | 103 ++++++++++++++++++--------------
 1 file changed, 57 insertions(+), 46 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 5c0b53f9e..dc4674548 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -110,7 +110,33 @@ class RsyncSync(NewBase):
 					level=logging.WARNING, noiselevel=-1)
 				self.verify_jobs = None
 
+		openpgp_env = None
+		if self.verify_metamanifest and gemato is not None:
+			# Use isolated environment if key is specified,
+			# system environment otherwise
+			if self.repo.sync_openpgp_key_path is not None:
+				openpgp_env = gemato.openpgp.OpenPGPEnvironment()
+			else:
+				openpgp_env = gemato.openpgp.OpenPGPSystemEnvironment()
+
 		try:
+			# Load and update the keyring early. If it fails, then verification
+			# will not be performed and the user will have to fix it and try again,
+			# so we may as well bail out before actual rsync happens.
+			if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
+				try:
+					out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+					with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+						openpgp_env.import_key(f)
+					out.ebegin('Refreshing keys from keyserver')
+					openpgp_env.refresh_keys()
+					out.eend(0)
+				except GematoException as e:
+					writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"
+							% (e,),
+							level=logging.ERROR, noiselevel=-1)
+					return (1, False)
+
 			# Real local timestamp file.
 			self.servertimestampfile = os.path.join(
 				self.repo.location, "metadata", "timestamp.chk")
@@ -299,52 +325,36 @@ class RsyncSync(NewBase):
 						level=logging.ERROR, noiselevel=-1)
 					exitcode = 127
 				else:
-					# Use isolated environment if key is specified,
-					# system environment otherwise
-					if self.repo.sync_openpgp_key_path is not None:
-						openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
-					else:
-						openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
-
 					try:
-						with openpgp_env_cls() as openpgp_env:
-							if self.repo.sync_openpgp_key_path is not None:
-								out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
-								with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
-									openpgp_env.import_key(f)
-								out.ebegin('Refreshing keys from keyserver')
-								openpgp_env.refresh_keys()
-								out.eend(0)
-
-							# we always verify the Manifest signature, in case
-							# we had to deal with key revocation case
-							m = gemato.recursiveloader.ManifestRecursiveLoader(
-									os.path.join(self.repo.location, 'Manifest'),
-									verify_openpgp=True,
-									openpgp_env=openpgp_env,
-									max_jobs=self.verify_jobs)
-							if not m.openpgp_signed:
-								raise RuntimeError('OpenPGP signature not found on Manifest')
-
-							ts = m.find_timestamp()
-							if ts is None:
-								raise RuntimeError('Timestamp not found in Manifest')
-
-							out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
-							out.einfo('Valid OpenPGP signature found:')
-							out.einfo('- primary key: %s' % (
-								m.openpgp_signature.primary_key_fingerprint))
-							out.einfo('- subkey: %s' % (
-								m.openpgp_signature.fingerprint))
-							out.einfo('- timestamp: %s UTC' % (
-								m.openpgp_signature.timestamp))
-
-							# if nothing has changed, skip the actual Manifest
-							# verification
-							if not local_state_unchanged:
-								out.ebegin('Verifying %s' % (self.repo.location,))
-								m.assert_directory_verifies()
-								out.eend(0)
+						# we always verify the Manifest signature, in case
+						# we had to deal with key revocation case
+						m = gemato.recursiveloader.ManifestRecursiveLoader(
+								os.path.join(self.repo.location, 'Manifest'),
+								verify_openpgp=True,
+								openpgp_env=openpgp_env,
+								max_jobs=self.verify_jobs)
+						if not m.openpgp_signed:
+							raise RuntimeError('OpenPGP signature not found on Manifest')
+
+						ts = m.find_timestamp()
+						if ts is None:
+							raise RuntimeError('Timestamp not found in Manifest')
+
+						out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+						out.einfo('Valid OpenPGP signature found:')
+						out.einfo('- primary key: %s' % (
+							m.openpgp_signature.primary_key_fingerprint))
+						out.einfo('- subkey: %s' % (
+							m.openpgp_signature.fingerprint))
+						out.einfo('- timestamp: %s UTC' % (
+							m.openpgp_signature.timestamp))
+
+						# if nothing has changed, skip the actual Manifest
+						# verification
+						if not local_state_unchanged:
+							out.ebegin('Verifying %s' % (self.repo.location,))
+							m.assert_directory_verifies()
+							out.eend(0)
 					except GematoException as e:
 						writemsg_level("!!! Manifest verification failed:\n%s\n"
 								% (e,),
@@ -353,7 +363,8 @@ class RsyncSync(NewBase):
 
 			return (exitcode, updatecache_flg)
 		finally:
-			pass
+			if openpgp_env is not None:
+				openpgp_env.close()
 
 
 	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
-- 
2.16.1

[gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old

[Michał Górny]

Issue an explicit warning if the Manifest timestamp for Gentoo
repository is 24 hours behind the system clock. This is meant to detect
attacks based on preventing the user from upgrading.
---
 cnf/repos.conf                             |  1 +
 man/portage.5                              |  4 ++++
 pym/portage/sync/modules/rsync/__init__.py |  1 +
 pym/portage/sync/modules/rsync/rsync.py    | 21 +++++++++++++++++++++
 4 files changed, 27 insertions(+)

diff --git a/cnf/repos.conf b/cnf/repos.conf
index 4a40ff4fc..984ecd220 100644
--- a/cnf/repos.conf
+++ b/cnf/repos.conf
@@ -7,6 +7,7 @@ sync-type = rsync
 sync-uri = rsync://rsync.gentoo.org/gentoo-portage
 auto-sync = yes
 sync-rsync-verify-metamanifest = yes
+sync-rsync-verify-max-age = 24
 sync-openpgp-key-path = /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg
 
 # for daily squashfs snapshots
diff --git a/man/portage.5 b/man/portage.5
index d4f755f51..778dedfd5 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1086,6 +1086,10 @@ directories if appropriate.
 Number of parallel jobs to use when verifying nested Manifests. Defaults
 to the apparent number of processors.
 .TP
+.B sync\-rsync\-verify\-max\-age
+Warn if repository is older than the specified number of hours. Disabled
+when 0. Defaults to disabled.
+.TP
 .B sync\-rsync\-verify\-metamanifest = yes|no
 Require the repository to contain a signed MetaManifest and verify
 it using \fBapp\-portage/gemato\fR. Defaults to no.
diff --git a/pym/portage/sync/modules/rsync/__init__.py b/pym/portage/sync/modules/rsync/__init__.py
index 27a2548c0..cb80f6d66 100644
--- a/pym/portage/sync/modules/rsync/__init__.py
+++ b/pym/portage/sync/modules/rsync/__init__.py
@@ -29,6 +29,7 @@ module_spec = {
 				'sync-rsync-extra-opts',
 				'sync-rsync-vcs-ignore',
 				'sync-rsync-verify-jobs',
+				'sync-rsync-verify-max-age',
 				'sync-rsync-verify-metamanifest',
 				),
 			}
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index dc4674548..732298b3f 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -6,6 +6,7 @@ import logging
 import time
 import signal
 import socket
+import datetime
 import io
 import re
 import random
@@ -109,6 +110,20 @@ class RsyncSync(NewBase):
 				writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
 					level=logging.WARNING, noiselevel=-1)
 				self.verify_jobs = None
+		# Support overriding max age.
+		self.max_age = self.repo.module_specific_options.get(
+				'sync-rsync-verify-max-age', '')
+		if self.max_age:
+			try:
+				self.max_age = int(self.max_age)
+				if self.max_age < 0:
+					raise ValueError(self.max_age)
+			except ValueError:
+				writemsg_level("!!! sync-rsync-max-age not a non-negative integer: %s\n" % (self.max_age,),
+					level=logging.WARNING, noiselevel=-1)
+				self.max_age = 0
+		else:
+			self.max_age = 0
 
 		openpgp_env = None
 		if self.verify_metamanifest and gemato is not None:
@@ -339,6 +354,12 @@ class RsyncSync(NewBase):
 						ts = m.find_timestamp()
 						if ts is None:
 							raise RuntimeError('Timestamp not found in Manifest')
+						if (self.max_age != 0 and
+								(datetime.datetime.utcnow() - ts.ts).hours > self.max_age):
+							out.ewarn('Manifest is over 24 hours old, this is suspicious!')
+							out.ewarn('You may want to try using another mirror and/or reporting this one:')
+							out.ewarn('  %s' % (dosyncuri,))
+							out.ewarn('')
 
 						out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
 						out.einfo('Valid OpenPGP signature found:')
-- 
2.16.1

[gentoo-portage-dev] [PATCH v2 7/9] git: Support verifying commit signature post-sync

[Michał Górny]

Add a new sync-git-verify-commit-signature option (defaulting to false)
that verifies the top commit signature after syncing. The verification
is currently done using built-in git routines.

The verification passes if the signature is good or untrusted.
In the latter case, a warning is printed. In any other case,
the verification causes sync to fail and an appropriate error is output.
---
 man/portage.5                            |  4 +++
 pym/portage/sync/modules/git/__init__.py |  3 +-
 pym/portage/sync/modules/git/git.py      | 48 ++++++++++++++++++++++++++++++--
 3 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/man/portage.5 b/man/portage.5
index 778dedfd5..2d5091109 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1007,6 +1007,10 @@ See also example for sync-git-clone-env.
 .B sync\-git\-pull\-extra\-opts
 Extra options to give to git when updating repository (git pull).
 .TP
+.B sync\-git\-verify\-commit\-signature = true|false
+Require the top commit in the repository to contain a good OpenPGP
+signature. Defaults to false.
+.TP
 .B sync\-hooks\-only\-on\-change
 If set to true, then sync of a given repository will not trigger postsync
 hooks unless hooks would have executed for a master repository or the
diff --git a/pym/portage/sync/modules/git/__init__.py b/pym/portage/sync/modules/git/__init__.py
index 2f1d35226..270d97186 100644
--- a/pym/portage/sync/modules/git/__init__.py
+++ b/pym/portage/sync/modules/git/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2017 Gentoo Foundation
+# Copyright 2014-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 doc = """Git plug-in module for portage.
@@ -58,6 +58,7 @@ module_spec = {
 				'sync-git-env',
 				'sync-git-pull-env',
 				'sync-git-pull-extra-opts',
+				'sync-git-verify-commit-signature',
 				),
 		}
 	}
diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py
index 8b4cab273..7e5ddf3b5 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2017 Gentoo Foundation
+# Copyright 2005-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import logging
@@ -7,7 +7,7 @@ import subprocess
 import portage
 from portage import os
 from portage.util import writemsg_level, shlex_split
-from portage.output import create_color_func
+from portage.output import create_color_func, EOutput
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")
 warn = create_color_func("WARN")
@@ -71,6 +71,7 @@ class GitSync(NewBase):
 		else:
 			# default
 			git_cmd_opts += " --depth 1"
+
 		if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
 			git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-clone-extra-opts']
 		git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
@@ -85,6 +86,8 @@ class GitSync(NewBase):
 			self.logger(self.xterm_titles, msg)
 			writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
 			return (exitcode, False)
+		if not self.verify_head():
+			return (1, False)
 		return (os.EX_OK, True)
 
 
@@ -125,12 +128,53 @@ class GitSync(NewBase):
 			self.logger(self.xterm_titles, msg)
 			writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
 			return (exitcode, False)
+		if not self.verify_head():
+			return (1, False)
 
 		current_rev = subprocess.check_output(rev_cmd,
 			cwd=portage._unicode_encode(self.repo.location))
 
 		return (os.EX_OK, current_rev != previous_rev)
 
+	def verify_head(self):
+		if (self.repo.module_specific_options.get(
+				'sync-git-verify-commit-signature', 'false') != 'true'):
+			return True
+
+		rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", "-1"]
+		try:
+			status = (portage._unicode_decode(
+				subprocess.check_output(rev_cmd,
+					cwd=portage._unicode_encode(self.repo.location)))
+				.strip())
+		except subprocess.CalledProcessError:
+			return False
+
+		out = EOutput()
+		if status == 'G':  # good signature is good
+			out.einfo('Trusted signature found on top commit')
+			return True
+		elif status == 'U':  # untrusted
+			out.ewarn('Top commit signature is valid but not trusted')
+			return True
+		else:
+			if status == 'B':
+				expl = 'bad signature'
+			elif status == 'X':
+				expl = 'expired signature'
+			elif status == 'Y':
+				expl = 'expired key'
+			elif status == 'R':
+				expl = 'revoked key'
+			elif status == 'E':
+				expl = 'unable to verify signature (missing key?)'
+			elif status == 'N':
+				expl = 'no signature'
+			else:
+				expl = 'unknown issue'
+			out.eerror('No valid signature found: %s' % (expl,))
+			return False
+
 	def retrieve_head(self, **kwargs):
 		'''Get information about the head commit'''
 		if kwargs:
-- 
2.16.1

[gentoo-portage-dev] [PATCH v2 8/9] git: Support running the verification against sync-openpgp-key-path

[Michał Górny]

---
 pym/portage/sync/modules/git/git.py | 101 +++++++++++++++++++++++++-----------
 1 file changed, 70 insertions(+), 31 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py
index 7e5ddf3b5..cec760d00 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -1,6 +1,7 @@
 # Copyright 2005-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import io
 import logging
 import subprocess
 
@@ -13,6 +14,12 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.sync.syncbase import NewBase
 
+try:
+	from gemato.exceptions import GematoException
+	import gemato.openpgp
+except ImportError:
+	gemato = None
+
 
 class GitSync(NewBase):
 	'''Git sync class'''
@@ -141,39 +148,71 @@ class GitSync(NewBase):
 				'sync-git-verify-commit-signature', 'false') != 'true'):
 			return True
 
-		rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", "-1"]
-		try:
-			status = (portage._unicode_decode(
-				subprocess.check_output(rev_cmd,
-					cwd=portage._unicode_encode(self.repo.location)))
-				.strip())
-		except subprocess.CalledProcessError:
-			return False
-
-		out = EOutput()
-		if status == 'G':  # good signature is good
-			out.einfo('Trusted signature found on top commit')
-			return True
-		elif status == 'U':  # untrusted
-			out.ewarn('Top commit signature is valid but not trusted')
-			return True
+		if self.repo.sync_openpgp_key_path is not None:
+			if gemato is None:
+				writemsg_level("!!! Verifying against specified key requires gemato-11.0+ installed\n",
+					level=logging.ERROR, noiselevel=-1)
+				return False
+			openpgp_env = gemato.openpgp.OpenPGPEnvironment()
 		else:
-			if status == 'B':
-				expl = 'bad signature'
-			elif status == 'X':
-				expl = 'expired signature'
-			elif status == 'Y':
-				expl = 'expired key'
-			elif status == 'R':
-				expl = 'revoked key'
-			elif status == 'E':
-				expl = 'unable to verify signature (missing key?)'
-			elif status == 'N':
-				expl = 'no signature'
+			openpgp_env = None
+
+		try:
+			out = EOutput()
+			env = None
+			if openpgp_env is not None:
+				try:
+					out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+					with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+						openpgp_env.import_key(f)
+					out.ebegin('Refreshing keys from keyserver')
+					openpgp_env.refresh_keys()
+					out.eend(0)
+				except GematoException as e:
+					writemsg_level("!!! Verification impossible due to keyring problem:\n%s\n"
+							% (e,),
+							level=logging.ERROR, noiselevel=-1)
+					return (1, False)
+
+				env = os.environ.copy()
+				env['GNUPGHOME'] = openpgp_env.home
+
+			rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", "-1"]
+			try:
+				status = (portage._unicode_decode(
+					subprocess.check_output(rev_cmd,
+						cwd=portage._unicode_encode(self.repo.location),
+						env=env))
+					.strip())
+			except subprocess.CalledProcessError:
+				return False
+
+			if status == 'G':  # good signature is good
+				out.einfo('Trusted signature found on top commit')
+				return True
+			elif status == 'U':  # untrusted
+				out.ewarn('Top commit signature is valid but not trusted')
+				return True
 			else:
-				expl = 'unknown issue'
-			out.eerror('No valid signature found: %s' % (expl,))
-			return False
+				if status == 'B':
+					expl = 'bad signature'
+				elif status == 'X':
+					expl = 'expired signature'
+				elif status == 'Y':
+					expl = 'expired key'
+				elif status == 'R':
+					expl = 'revoked key'
+				elif status == 'E':
+					expl = 'unable to verify signature (missing key?)'
+				elif status == 'N':
+					expl = 'no signature'
+				else:
+					expl = 'unknown issue'
+				out.eerror('No valid signature found: %s' % (expl,))
+				return False
+		finally:
+			if openpgp_env is not None:
+				openpgp_env.close()
 
 	def retrieve_head(self, **kwargs):
 		'''Get information about the head commit'''
-- 
2.16.1

[gentoo-portage-dev] [PATCH v2 9/9] max-age fixup

[Michał Górny]

---
 man/portage.5                           | 2 +-
 pym/portage/sync/modules/rsync/rsync.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/man/portage.5 b/man/portage.5
index 2d5091109..549c51c73 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1091,7 +1091,7 @@ Number of parallel jobs to use when verifying nested Manifests. Defaults
 to the apparent number of processors.
 .TP
 .B sync\-rsync\-verify\-max\-age
-Warn if repository is older than the specified number of hours. Disabled
+Warn if repository is older than the specified number of days. Disabled
 when 0. Defaults to disabled.
 .TP
 .B sync\-rsync\-verify\-metamanifest = yes|no
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 732298b3f..cc121e0fb 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -355,8 +355,8 @@ class RsyncSync(NewBase):
 						if ts is None:
 							raise RuntimeError('Timestamp not found in Manifest')
 						if (self.max_age != 0 and
-								(datetime.datetime.utcnow() - ts.ts).hours > self.max_age):
-							out.ewarn('Manifest is over 24 hours old, this is suspicious!')
+								(datetime.datetime.utcnow() - ts.ts).days > self.max_age):
+							out.ewarn('Manifest is over %d days old, this is suspicious!' % (self.max_age,))
 							out.ewarn('You may want to try using another mirror and/or reporting this one:')
 							out.ewarn('  %s' % (dosyncuri,))
 							out.ewarn('')
-- 
2.16.1

Re: [gentoo-portage-dev] [PATCH v2 9/9] max-age fixup

[Michał Górny]

W dniu pią, 02.02.2018 o godzinie 21∶42 +0100, użytkownik Michał Górny
napisał:
> ---
>  man/portage.5                           | 2 +-
>  pym/portage/sync/modules/rsync/rsync.py | 4 ++--
>  2 files changed, 3 insertions(+), 3 deletions(-)
> 
> diff --git a/man/portage.5 b/man/portage.5
> index 2d5091109..549c51c73 100644
> --- a/man/portage.5
> +++ b/man/portage.5
> @@ -1091,7 +1091,7 @@ Number of parallel jobs to use when verifying nested Manifests. Defaults
>  to the apparent number of processors.
>  .TP
>  .B sync\-rsync\-verify\-max\-age
> -Warn if repository is older than the specified number of hours. Disabled
> +Warn if repository is older than the specified number of days. Disabled
>  when 0. Defaults to disabled.
>  .TP
>  .B sync\-rsync\-verify\-metamanifest = yes|no
> diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
> index 732298b3f..cc121e0fb 100644
> --- a/pym/portage/sync/modules/rsync/rsync.py
> +++ b/pym/portage/sync/modules/rsync/rsync.py
> @@ -355,8 +355,8 @@ class RsyncSync(NewBase):
>  						if ts is None:
>  							raise RuntimeError('Timestamp not found in Manifest')
>  						if (self.max_age != 0 and
> -								(datetime.datetime.utcnow() - ts.ts).hours > self.max_age):
> -							out.ewarn('Manifest is over 24 hours old, this is suspicious!')
> +								(datetime.datetime.utcnow() - ts.ts).days > self.max_age):
> +							out.ewarn('Manifest is over %d days old, this is suspicious!' % (self.max_age,))
>  							out.ewarn('You may want to try using another mirror and/or reporting this one:')
>  							out.ewarn('  %s' % (dosyncuri,))
>  							out.ewarn('')

Note: I've merged this into 6/9 now.

-- 
Best regards,
Michał Górny

Re: [gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old

[M. J. Everitt]

[-- Attachment #1.1: Type: text/plain, Size: 957 bytes --]

On 02/02/18 20:42, Michał Górny wrote[excerpted]:
> index 27a2548c0..cb80f6d66 100644
> --- a/pym/portage/sync/modules/rsync/__init__.py
> +++ b/pym/portage/sync/modules/rsync/__init__.py
> @@ -109,6 +110,20 @@ class RsyncSync(NewBase):
>  				writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
>  					level=logging.WARNING, noiselevel=-1)
>  				self.verify_jobs = None
> +		# Support overriding max age.
> +		self.max_age = self.repo.module_specific_options.get(
> +				'sync-rsync-verify-max-age', '')
> +		if self.max_age:
> +			try:
> +				self.max_age = int(self.max_age)
> +				if self.max_age < 0:
> +					raise ValueError(self.max_age)
> +			except ValueError:
> +				writemsg_level("!!! sync-rsync-max-age not a non-negative integer: %s\n" % (self.max_age,),
A beautiful double-negative .. but would read better as "not a positive
integer" or "is a negative integer" ..

MJE


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

Re: [gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old

[Michał Górny]

W dniu nie, 04.02.2018 o godzinie 13∶48 +0000, użytkownik M. J. Everitt
napisał:
> On 02/02/18 20:42, Michał Górny wrote[excerpted]:
> > index 27a2548c0..cb80f6d66 100644
> > --- a/pym/portage/sync/modules/rsync/__init__.py
> > +++ b/pym/portage/sync/modules/rsync/__init__.py
> > @@ -109,6 +110,20 @@ class RsyncSync(NewBase):
> >  				writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
> >  					level=logging.WARNING, noiselevel=-1)
> >  				self.verify_jobs = None
> > +		# Support overriding max age.
> > +		self.max_age = self.repo.module_specific_options.get(
> > +				'sync-rsync-verify-max-age', '')
> > +		if self.max_age:
> > +			try:
> > +				self.max_age = int(self.max_age)
> > +				if self.max_age < 0:
> > +					raise ValueError(self.max_age)
> > +			except ValueError:
> > +				writemsg_level("!!! sync-rsync-max-age not a non-negative integer: %s\n" % (self.max_age,),
> 
> A beautiful double-negative .. but would read better as "not a positive
> integer" or "is a negative integer" ..
> 

Except that neither is correct. Zero is not positive but valid.
And the error also covers the case when it's not an integer at all.

-- 
Best regards,
Michał Górny