[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     dfb277c58f8b0d55c1b0c9884cd91f3204888b38
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 22 10:59:32 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:35 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=dfb277c5

RsyncSync: support file:// sync-uri

This will be useful for unit tests that will sync from a local
file:// sync-uri.

---
 pym/portage/sync/modules/rsync/rsync.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 76d83f2..74c10e7 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -66,6 +66,9 @@ class RsyncSync(SyncBase):
 			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
 		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
+		self.extra_rsync_opts = portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
+
 		# Real local timestamp file.
 		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
@@ -93,6 +96,14 @@ class RsyncSync(SyncBase):
 		except:
 			maxretries = -1 #default number of retries
 
+		if syncuri.startswith("file://"):
+			self.proto = "file"
+			dosyncuri = syncuri[6:]
+			is_synced, exitcode = self._do_rsync(
+				dosyncuri, timestamp, opts)
+			self._process_exitcode(exitcode, dosyncuri, out, 1)
+			return (exitcode, exitcode == os.EX_OK)
+
 		retries=0
 		try:
 			self.proto, user_name, hostname, port = re.split(
@@ -116,8 +127,6 @@ class RsyncSync(SyncBase):
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
 		all_rsync_opts = set(self.rsync_opts)
-		self.extra_rsync_opts = portage.util.shlex_split(
-			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
 		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     2f55335dd6ec65d2b8a9c57008ded60df49950db
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Feb  9 20:06:23 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Feb  9 20:11:44 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2f55335d

portage/sync/modules/rsync: Fixed missed is_synced setting for timestamps match

_do_rsync() was not returning True if the timestamps matched, so it was cycking through
the retries.
Reported by pchrist in #gentoo-portage.

---
 pym/portage/sync/modules/rsync/rsync.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 817164d..d3b0ae3 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -464,6 +464,7 @@ class RsyncSync(NewBase):
 
 		if exitcode == os.EX_OK:
 			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				is_synced = True
 				self.logger(self.xterm_titles,
 					">>> Cancelling sync -- Already current.")
 				print()

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     7833f43bbbe7d863d6564615101fd558a9be6f1f
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon May 18 17:04:56 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon May 18 18:07:53 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7833f43b

portage/sync/modules/rsync: Fix UnicodeDecodeError:  bug 549826

X-Gentoo-Bug: 549826
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=549826
X-Gentoo-forums: https://forums.gentoo.org/viewtopic-t-1017380.html

 pym/portage/sync/modules/rsync/rsync.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d3b0ae3..d84c36d 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -12,6 +12,7 @@ import tempfile
 
 import portage
 from portage import os
+from portage import _unicode_decode
 from portage.util import writemsg_level
 from portage.output import create_color_func, yellow, blue, bold
 good = create_color_func("GOOD")
@@ -23,6 +24,11 @@ from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import NewBase
 
+if sys.hexversion >= 0x3000000:
+	# pylint: disable=W0622
+	_unicode = str
+else:
+	_unicode = unicode
 
 SERVER_OUT_OF_DATE = -1
 EXCEEDED_MAX_RETRIES = -2
@@ -145,7 +151,8 @@ class RsyncSync(NewBase):
 				family, socket.SOCK_STREAM))
 		except socket.error as e:
 			writemsg_level(
-				"!!! getaddrinfo failed for '%s': %s\n" % (hostname, e),
+				"!!! getaddrinfo failed for '%s': %s\n"
+				% (_unicode_decode(hostname), _unicode(e)),
 				noiselevel=-1, level=logging.ERROR)
 
 		if addrinfos:
@@ -197,8 +204,8 @@ class RsyncSync(NewBase):
 			if uris:
 				dosyncuri = uris.pop()
 			else:
-				writemsg("!!! Exhausted addresses for %s\n" % \
-					hostname, noiselevel=-1)
+				writemsg("!!! Exhausted addresses for %s\n"
+					% _unicode_decode(hostname), noiselevel=-1)
 				return (1, False)
 
 			if (retries==0):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     bb0282d265dbe209639280964b1733c842d466ed
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 14 19:52:23 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Jul 14 21:28:31 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=bb0282d2

RsyncSync: don't pass None sync-rsync-extra-opts value into shlex_split

This fixes an issue with SyncLocalTestCase hanging on my system.

 pym/portage/sync/modules/rsync/rsync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 8041f07..f08bf5c 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -73,7 +73,7 @@ class RsyncSync(NewBase):
 		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		self.extra_rsync_opts = list()
-		if 'sync-rsync-extra-opts' in self.repo.module_specific_options:
+		if self.repo.module_specific_options.get('sync-rsync-extra-opts'):
 			self.extra_rsync_opts.extend(portage.util.shlex_split(
 				self.repo.module_specific_options['sync-rsync-extra-opts']))
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     d984cea6ee08799f003b189e934f696b483e101a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Nov  7 04:49:45 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Nov  7 18:14:02 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d984cea6

RsyncSync: skip metadata-transfer when appropriate (bug 564988)

Fix flaws in logic involving the updatecache_flg variable, in order
to skip metadata-transfer when sync fails (or the server timestamp
has not changed).

X-Gentoo-Bug: 564988
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=564988
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 8ae8a5c..e0f76b3 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -112,10 +112,10 @@ class RsyncSync(NewBase):
 		if syncuri.startswith("file://"):
 			self.proto = "file"
 			dosyncuri = syncuri[6:]
-			is_synced, exitcode = self._do_rsync(
+			is_synced, exitcode, updatecache_flg = self._do_rsync(
 				dosyncuri, timestamp, opts)
 			self._process_exitcode(exitcode, dosyncuri, out, 1)
-			return (exitcode, exitcode == os.EX_OK)
+			return (exitcode, updatecache_flg)
 
 		retries=0
 		try:
@@ -138,7 +138,7 @@ class RsyncSync(NewBase):
 		else:
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
-		updatecache_flg=True
+		updatecache_flg = False
 		all_rsync_opts = set(self.rsync_opts)
 		all_rsync_opts.update(self.extra_rsync_opts)
 
@@ -240,7 +240,8 @@ class RsyncSync(NewBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			is_synced, exitcode, updatecache_flg = self._do_rsync(
+				dosyncuri, timestamp, opts)
 			if is_synced:
 				break
 
@@ -251,7 +252,6 @@ class RsyncSync(NewBase):
 			else:
 				# over retries
 				# exit loop
-				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
 		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
@@ -382,6 +382,7 @@ class RsyncSync(NewBase):
 
 
 	def _do_rsync(self, syncuri, timestamp, opts):
+		updatecache_flg = False
 		is_synced = False
 		if timestamp != 0 and "--quiet" not in opts:
 			print(">>> Checking server timestamp ...")
@@ -489,7 +490,7 @@ class RsyncSync(NewBase):
 				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
 				print(">>>")
 				print()
-				return is_synced, exitcode
+				return is_synced, exitcode, updatecache_flg
 			elif (servertimestamp != 0) and (servertimestamp < timestamp):
 				self.logger(self.xterm_titles,
 					">>> Server out of date: %s" % syncuri)
@@ -543,6 +544,8 @@ class RsyncSync(NewBase):
 							os.unlink(self.servertimestampfile)
 						except OSError:
 							pass
+					else:
+						updatecache_flg = True
 
 				if exitcode in [0,1,3,4,11,14,20,21]:
 					is_synced = True
@@ -554,4 +557,4 @@ class RsyncSync(NewBase):
 			# --prune-empty-directories.  Retry for a server that supports
 			# at least rsync protocol version 29 (>=rsync-2.6.4).
 			pass
-		return is_synced, exitcode
+		return is_synced, exitcode, updatecache_flg

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     c7c04b4f4f4f5e6f18ad76366b535dcbad72989e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 27 19:40:01 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Oct 27 19:42:44 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c7c04b4f

RsyncSync: fix PORTAGE_RSYNC_RETRIES (bug 497596)

When PORTAGE_RSYNC_RETRIES is set to a positive integer, recycle
the uris until the specified number of retries has been exhausted.

Bug: https://bugs.gentoo.org/497596

 pym/portage/sync/modules/rsync/rsync.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 45a70e7dd..01e4e5924 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -202,6 +202,7 @@ class RsyncSync(NewBase):
 
 		# reverse, for use with pop()
 		uris.reverse()
+		uris_orig = uris[:]
 
 		effective_maxretries = maxretries
 		if effective_maxretries < 0:
@@ -210,10 +211,13 @@ class RsyncSync(NewBase):
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
-			else:
+			elif maxretries < 0 or retries > maxretries:
 				writemsg("!!! Exhausted addresses for %s\n"
 					% _unicode_decode(hostname), noiselevel=-1)
 				return (1, False)
+			else:
+				uris.extend(uris_orig)
+				dosyncuri = uris.pop()
 
 			if (retries==0):
 				if "--ask" in opts:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     8b773727f009e63fbaecf937701d7f9f1a97f112
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 31 22:20:55 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jan 31 22:20:55 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b773727

rsync: handle CommandNotFound for gemato (bug 646184)

Bug: https://bugs.gentoo.org/646184

 pym/portage/sync/modules/rsync/rsync.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 7c020a563..69e4be734 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -13,6 +13,7 @@ import tempfile
 import portage
 from portage import os
 from portage import _unicode_decode
+from portage.exception import CommandNotFound
 from portage.util import writemsg_level
 from portage.output import create_color_func, yellow, blue, bold
 good = create_color_func("GOOD")
@@ -277,7 +278,12 @@ class RsyncSync(NewBase):
 				command += ['-K', self.repo.openpgp_key_path]
 			if self.verify_jobs is not None:
 				command += ['-j', self.verify_jobs]
-			exitcode = portage.process.spawn(command, **self.spawn_kwargs)
+			try:
+				exitcode = portage.process.spawn(command, **self.spawn_kwargs)
+			except CommandNotFound as e:
+				writemsg_level("!!! Command not found: %s\n" % (command[0],),
+					level=logging.ERROR, noiselevel=-1)
+				exitcode = 127
 
 		return (exitcode, updatecache_flg)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     5aa69880ca3665e633cb042d3e26f3a72536fc10
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 31 23:37:43 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jan 31 23:37:43 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5aa69880

rsync: if local state is unchanged then skip verification

 pym/portage/sync/modules/rsync/rsync.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 69e4be734..e2e6d0658 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -123,7 +123,7 @@ class RsyncSync(NewBase):
 		if syncuri.startswith("file://"):
 			self.proto = "file"
 			dosyncuri = syncuri[7:]
-			is_synced, exitcode, updatecache_flg = self._do_rsync(
+			unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
 				dosyncuri, timestamp, opts)
 			self._process_exitcode(exitcode, dosyncuri, out, 1)
 			return (exitcode, updatecache_flg)
@@ -219,6 +219,7 @@ class RsyncSync(NewBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
+		local_state_unchanged = True
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -255,8 +256,10 @@ class RsyncSync(NewBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			is_synced, exitcode, updatecache_flg = self._do_rsync(
+			unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
 				dosyncuri, timestamp, opts)
+			if not unchanged:
+				local_state_unchanged = False
 			if is_synced:
 				break
 
@@ -272,7 +275,7 @@ class RsyncSync(NewBase):
 		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
 
 		# if synced successfully, verify now
-		if exitcode == 0 and self.verify_metamanifest:
+		if exitcode == 0 and not local_state_unchanged and self.verify_metamanifest:
 			command = ['gemato', 'verify', '-s', self.repo.location]
 			if self.repo.openpgp_key_path is not None:
 				command += ['-K', self.repo.openpgp_key_path]
@@ -437,6 +440,7 @@ class RsyncSync(NewBase):
 		if "--debug" in opts:
 			print(rsynccommand)
 
+		local_state_unchanged = False
 		exitcode = os.EX_OK
 		servertimestamp = 0
 		# Even if there's no timestamp available locally, fetch the
@@ -521,6 +525,7 @@ class RsyncSync(NewBase):
 
 		if exitcode == os.EX_OK:
 			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				local_state_unchanged = True
 				is_synced = True
 				self.logger(self.xterm_titles,
 					">>> Cancelling sync -- Already current.")
@@ -532,7 +537,6 @@ class RsyncSync(NewBase):
 				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
 				print(">>>")
 				print()
-				return is_synced, exitcode, updatecache_flg
 			elif (servertimestamp != 0) and (servertimestamp < timestamp):
 				self.logger(self.xterm_titles,
 					">>> Server out of date: %s" % syncuri)
@@ -599,4 +603,5 @@ class RsyncSync(NewBase):
 			# --prune-empty-directories.  Retry for a server that supports
 			# at least rsync protocol version 29 (>=rsync-2.6.4).
 			pass
-		return is_synced, exitcode, updatecache_flg
+
+		return local_state_unchanged, is_synced, exitcode, updatecache_flg

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Michał Górny]

commit:     860b7f9d703c080451892f5d988636320bb11bc8
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Thu Feb  1 09:27:22 2018 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Feb  5 18:40:24 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=860b7f9d

rsync: Verify the value of sync-rsync-verify-jobs

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 4471f5bbe..ec28af366 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -92,6 +92,15 @@ class RsyncSync(NewBase):
 		# Support overriding job count.
 		self.verify_jobs = self.repo.module_specific_options.get(
 				'sync-rsync-verify-jobs', None)
+		if self.verify_jobs is not None:
+			try:
+				self.verify_jobs = int(self.verify_jobs)
+				if self.verify_jobs <= 0:
+					raise ValueError(self.verify_jobs)
+			except ValueError:
+				writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
+					level=logging.WARNING, noiselevel=-1)
+				self.verify_jobs = None
 
 		# Real local timestamp file.
 		self.servertimestampfile = os.path.join(
@@ -280,7 +289,7 @@ class RsyncSync(NewBase):
 			if self.repo.sync_openpgp_key_path is not None:
 				command += ['-K', self.repo.sync_openpgp_key_path]
 			if self.verify_jobs is not None:
-				command += ['-j', self.verify_jobs]
+				command += ['-j', str(self.verify_jobs)]
 			try:
 				exitcode = portage.process.spawn(command, **self.spawn_kwargs)
 			except CommandNotFound as e:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Michał Górny]

commit:     a33ebb12913282a3ddb579d2b7ac1c9fadbe97e5
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Thu Feb  1 11:41:22 2018 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Feb  5 18:41:33 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a33ebb12

rsync: Use gemato routines directly instead of calling the CLI tool

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 66 ++++++++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 9 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index ec28af366..39c4066d8 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -6,6 +6,7 @@ import logging
 import time
 import signal
 import socket
+import io
 import re
 import random
 import tempfile
@@ -25,6 +26,13 @@ from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import NewBase
 
+try:
+	from gemato.exceptions import GematoException
+	import gemato.openpgp
+	import gemato.recursiveloader
+except ImportError:
+	gemato = None
+
 if sys.hexversion >= 0x3000000:
 	# pylint: disable=W0622
 	_unicode = str
@@ -285,17 +293,57 @@ class RsyncSync(NewBase):
 
 		# if synced successfully, verify now
 		if exitcode == 0 and not local_state_unchanged and self.verify_metamanifest:
-			command = ['gemato', 'verify', '-s', self.repo.location]
-			if self.repo.sync_openpgp_key_path is not None:
-				command += ['-K', self.repo.sync_openpgp_key_path]
-			if self.verify_jobs is not None:
-				command += ['-j', str(self.verify_jobs)]
-			try:
-				exitcode = portage.process.spawn(command, **self.spawn_kwargs)
-			except CommandNotFound as e:
-				writemsg_level("!!! Command not found: %s\n" % (command[0],),
+			if gemato is None:
+				writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
 					level=logging.ERROR, noiselevel=-1)
 				exitcode = 127
+			else:
+				# Use isolated environment if key is specified,
+				# system environment otherwise
+				if self.repo.sync_openpgp_key_path is not None:
+					openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+				else:
+					openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+
+				try:
+					with openpgp_env_cls() as openpgp_env:
+						if self.repo.sync_openpgp_key_path is not None:
+							out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+							with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+								openpgp_env.import_key(f)
+							out.ebegin('Refreshing keys from keyserver')
+							openpgp_env.refresh_keys()
+							out.eend(0)
+
+						m = gemato.recursiveloader.ManifestRecursiveLoader(
+								os.path.join(self.repo.location, 'Manifest'),
+								verify_openpgp=True,
+								openpgp_env=openpgp_env,
+								max_jobs=self.verify_jobs)
+						if not m.openpgp_signed:
+							raise RuntimeError('OpenPGP signature not found on Manifest')
+
+						ts = m.find_timestamp()
+						if ts is None:
+							raise RuntimeError('Timestamp not found in Manifest')
+
+						out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+						out.einfo('Valid OpenPGP signature found:')
+						out.einfo('- primary key: %s' % (
+							m.openpgp_signature.primary_key_fingerprint))
+						out.einfo('- subkey: %s' % (
+							m.openpgp_signature.fingerprint))
+						out.einfo('- timestamp: %s UTC' % (
+							m.openpgp_signature.timestamp))
+
+						out.ebegin('Verifying %s' % (self.repo.location,))
+						m.assert_directory_verifies()
+						out.eend(0)
+				except GematoException as e:
+					writemsg_level("!!! Manifest verification failed:\n%s\n"
+							% (e,),
+							level=logging.ERROR, noiselevel=-1)
+					exitcode = 1
 
 		return (exitcode, updatecache_flg)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Michał Górny]

commit:     a02b5f9f0ae428eeb6cdc30bb3bcb7ac647358de
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Thu Feb  1 11:44:49 2018 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Feb  5 18:41:35 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a02b5f9f

rsync: Verify the Manifest signature even if tree is unchanged

Always verify the Manifest signature if verification is enabled.
Skipping the deep tree verification for unchanged case is reasonable
but we need to make sure the Manifest signature stays valid to catch
the case of the signing key being revoked.

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 39c4066d8..e6e218868 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -292,7 +292,7 @@ class RsyncSync(NewBase):
 		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
 
 		# if synced successfully, verify now
-		if exitcode == 0 and not local_state_unchanged and self.verify_metamanifest:
+		if exitcode == 0 and self.verify_metamanifest:
 			if gemato is None:
 				writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
 					level=logging.ERROR, noiselevel=-1)
@@ -315,6 +315,8 @@ class RsyncSync(NewBase):
 							openpgp_env.refresh_keys()
 							out.eend(0)
 
+						# we always verify the Manifest signature, in case
+						# we had to deal with key revocation case
 						m = gemato.recursiveloader.ManifestRecursiveLoader(
 								os.path.join(self.repo.location, 'Manifest'),
 								verify_openpgp=True,
@@ -336,9 +338,12 @@ class RsyncSync(NewBase):
 						out.einfo('- timestamp: %s UTC' % (
 							m.openpgp_signature.timestamp))
 
-						out.ebegin('Verifying %s' % (self.repo.location,))
-						m.assert_directory_verifies()
-						out.eend(0)
+						# if nothing has changed, skip the actual Manifest
+						# verification
+						if not local_state_unchanged:
+							out.ebegin('Verifying %s' % (self.repo.location,))
+							m.assert_directory_verifies()
+							out.eend(0)
 				except GematoException as e:
 					writemsg_level("!!! Manifest verification failed:\n%s\n"
 							% (e,),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Michał Górny]

commit:     c49715693d6602895021ff66a917bb9edc736d31
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Thu Feb  1 11:56:25 2018 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Feb  5 18:41:41 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c4971569

rsync: Load and update keys early

Load and update keys early to avoid delaying failures post rsync. Any
failure will prevent verification from happening, and presumably most of
the users will prefer fixing it and trying to sync again. For that case,
it is better to perform the task before actual rsync to avoid
unnecessarily rsyncing twice.

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 103 ++++++++++++++++++--------------
 1 file changed, 57 insertions(+), 46 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 5c0b53f9e..dc4674548 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -110,7 +110,33 @@ class RsyncSync(NewBase):
 					level=logging.WARNING, noiselevel=-1)
 				self.verify_jobs = None
 
+		openpgp_env = None
+		if self.verify_metamanifest and gemato is not None:
+			# Use isolated environment if key is specified,
+			# system environment otherwise
+			if self.repo.sync_openpgp_key_path is not None:
+				openpgp_env = gemato.openpgp.OpenPGPEnvironment()
+			else:
+				openpgp_env = gemato.openpgp.OpenPGPSystemEnvironment()
+
 		try:
+			# Load and update the keyring early. If it fails, then verification
+			# will not be performed and the user will have to fix it and try again,
+			# so we may as well bail out before actual rsync happens.
+			if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
+				try:
+					out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+					with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+						openpgp_env.import_key(f)
+					out.ebegin('Refreshing keys from keyserver')
+					openpgp_env.refresh_keys()
+					out.eend(0)
+				except GematoException as e:
+					writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"
+							% (e,),
+							level=logging.ERROR, noiselevel=-1)
+					return (1, False)
+
 			# Real local timestamp file.
 			self.servertimestampfile = os.path.join(
 				self.repo.location, "metadata", "timestamp.chk")
@@ -299,52 +325,36 @@ class RsyncSync(NewBase):
 						level=logging.ERROR, noiselevel=-1)
 					exitcode = 127
 				else:
-					# Use isolated environment if key is specified,
-					# system environment otherwise
-					if self.repo.sync_openpgp_key_path is not None:
-						openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
-					else:
-						openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
-
 					try:
-						with openpgp_env_cls() as openpgp_env:
-							if self.repo.sync_openpgp_key_path is not None:
-								out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
-								with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
-									openpgp_env.import_key(f)
-								out.ebegin('Refreshing keys from keyserver')
-								openpgp_env.refresh_keys()
-								out.eend(0)
-
-							# we always verify the Manifest signature, in case
-							# we had to deal with key revocation case
-							m = gemato.recursiveloader.ManifestRecursiveLoader(
-									os.path.join(self.repo.location, 'Manifest'),
-									verify_openpgp=True,
-									openpgp_env=openpgp_env,
-									max_jobs=self.verify_jobs)
-							if not m.openpgp_signed:
-								raise RuntimeError('OpenPGP signature not found on Manifest')
-
-							ts = m.find_timestamp()
-							if ts is None:
-								raise RuntimeError('Timestamp not found in Manifest')
-
-							out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
-							out.einfo('Valid OpenPGP signature found:')
-							out.einfo('- primary key: %s' % (
-								m.openpgp_signature.primary_key_fingerprint))
-							out.einfo('- subkey: %s' % (
-								m.openpgp_signature.fingerprint))
-							out.einfo('- timestamp: %s UTC' % (
-								m.openpgp_signature.timestamp))
-
-							# if nothing has changed, skip the actual Manifest
-							# verification
-							if not local_state_unchanged:
-								out.ebegin('Verifying %s' % (self.repo.location,))
-								m.assert_directory_verifies()
-								out.eend(0)
+						# we always verify the Manifest signature, in case
+						# we had to deal with key revocation case
+						m = gemato.recursiveloader.ManifestRecursiveLoader(
+								os.path.join(self.repo.location, 'Manifest'),
+								verify_openpgp=True,
+								openpgp_env=openpgp_env,
+								max_jobs=self.verify_jobs)
+						if not m.openpgp_signed:
+							raise RuntimeError('OpenPGP signature not found on Manifest')
+
+						ts = m.find_timestamp()
+						if ts is None:
+							raise RuntimeError('Timestamp not found in Manifest')
+
+						out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+						out.einfo('Valid OpenPGP signature found:')
+						out.einfo('- primary key: %s' % (
+							m.openpgp_signature.primary_key_fingerprint))
+						out.einfo('- subkey: %s' % (
+							m.openpgp_signature.fingerprint))
+						out.einfo('- timestamp: %s UTC' % (
+							m.openpgp_signature.timestamp))
+
+						# if nothing has changed, skip the actual Manifest
+						# verification
+						if not local_state_unchanged:
+							out.ebegin('Verifying %s' % (self.repo.location,))
+							m.assert_directory_verifies()
+							out.eend(0)
 					except GematoException as e:
 						writemsg_level("!!! Manifest verification failed:\n%s\n"
 								% (e,),
@@ -353,7 +363,8 @@ class RsyncSync(NewBase):
 
 			return (exitcode, updatecache_flg)
 		finally:
-			pass
+			if openpgp_env is not None:
+				openpgp_env.close()
 
 
 	def _process_exitcode(self, exitcode, syncuri, out, maxretries):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Michał Górny]

commit:     56affbecaa1e5faecdf5176a574aa97d098d486b
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Fri Feb  2 19:32:24 2018 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Feb  5 18:41:38 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=56affbec

rsync: Pre-indent the try-finally block for gemato key scope

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 467 ++++++++++++++++----------------
 1 file changed, 235 insertions(+), 232 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index e6e218868..5c0b53f9e 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -110,247 +110,250 @@ class RsyncSync(NewBase):
 					level=logging.WARNING, noiselevel=-1)
 				self.verify_jobs = None
 
-		# Real local timestamp file.
-		self.servertimestampfile = os.path.join(
-			self.repo.location, "metadata", "timestamp.chk")
-
-		content = portage.util.grabfile(self.servertimestampfile)
-		timestamp = 0
-		if content:
-			try:
-				timestamp = time.mktime(time.strptime(content[0],
-					TIMESTAMP_FORMAT))
-			except (OverflowError, ValueError):
-				pass
-		del content
-
-		try:
-			self.rsync_initial_timeout = \
-				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
-		except ValueError:
-			self.rsync_initial_timeout = 15
-
-		try:
-			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
-		except SystemExit as e:
-			raise # Needed else can't exit
-		except:
-			maxretries = -1 #default number of retries
-
-		if syncuri.startswith("file://"):
-			self.proto = "file"
-			dosyncuri = syncuri[7:]
-			unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
-				dosyncuri, timestamp, opts)
-			self._process_exitcode(exitcode, dosyncuri, out, 1)
-			return (exitcode, updatecache_flg)
-
-		retries=0
 		try:
-			self.proto, user_name, hostname, port = re.split(
-				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
-				syncuri, maxsplit=4)[1:5]
-		except ValueError:
-			writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
-				noiselevel=-1, level=logging.ERROR)
-			return (1, False)
+			# Real local timestamp file.
+			self.servertimestampfile = os.path.join(
+				self.repo.location, "metadata", "timestamp.chk")
 
-		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+			content = portage.util.grabfile(self.servertimestampfile)
+			timestamp = 0
+			if content:
+				try:
+					timestamp = time.mktime(time.strptime(content[0],
+						TIMESTAMP_FORMAT))
+				except (OverflowError, ValueError):
+					pass
+			del content
 
-		if port is None:
-			port=""
-		if user_name is None:
-			user_name=""
-		if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
-			getaddrinfo_host = hostname
-		else:
-			# getaddrinfo needs the brackets stripped
-			getaddrinfo_host = hostname[1:-1]
-		updatecache_flg = False
-		all_rsync_opts = set(self.rsync_opts)
-		all_rsync_opts.update(self.extra_rsync_opts)
+			try:
+				self.rsync_initial_timeout = \
+					int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
+			except ValueError:
+				self.rsync_initial_timeout = 15
 
-		family = socket.AF_UNSPEC
-		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
-			family = socket.AF_INET
-		elif socket.has_ipv6 and \
-			("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
-			family = socket.AF_INET6
+			try:
+				maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
+			except SystemExit as e:
+				raise # Needed else can't exit
+			except:
+				maxretries = -1 #default number of retries
+
+			if syncuri.startswith("file://"):
+				self.proto = "file"
+				dosyncuri = syncuri[7:]
+				unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
+					dosyncuri, timestamp, opts)
+				self._process_exitcode(exitcode, dosyncuri, out, 1)
+				return (exitcode, updatecache_flg)
+
+			retries=0
+			try:
+				self.proto, user_name, hostname, port = re.split(
+					r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
+					syncuri, maxsplit=4)[1:5]
+			except ValueError:
+				writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
+					noiselevel=-1, level=logging.ERROR)
+				return (1, False)
 
-		addrinfos = None
-		uris = []
+			self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
-		try:
-			addrinfos = getaddrinfo_validate(
-				socket.getaddrinfo(getaddrinfo_host, None,
-				family, socket.SOCK_STREAM))
-		except socket.error as e:
-			writemsg_level(
-				"!!! getaddrinfo failed for '%s': %s\n"
-				% (_unicode_decode(hostname), _unicode(e)),
-				noiselevel=-1, level=logging.ERROR)
-
-		if addrinfos:
-
-			AF_INET = socket.AF_INET
-			AF_INET6 = None
-			if socket.has_ipv6:
-				AF_INET6 = socket.AF_INET6
-
-			ips_v4 = []
-			ips_v6 = []
-
-			for addrinfo in addrinfos:
-				if addrinfo[0] == AF_INET:
-					ips_v4.append("%s" % addrinfo[4][0])
-				elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
-					# IPv6 addresses need to be enclosed in square brackets
-					ips_v6.append("[%s]" % addrinfo[4][0])
-
-			random.shuffle(ips_v4)
-			random.shuffle(ips_v6)
-
-			# Give priority to the address family that
-			# getaddrinfo() returned first.
-			if AF_INET6 is not None and addrinfos and \
-				addrinfos[0][0] == AF_INET6:
-				ips = ips_v6 + ips_v4
-			else:
-				ips = ips_v4 + ips_v6
-
-			for ip in ips:
-				uris.append(syncuri.replace(
-					"//" + user_name + hostname + port + "/",
-					"//" + user_name + ip + port + "/", 1))
-
-		if not uris:
-			# With some configurations we need to use the plain hostname
-			# rather than try to resolve the ip addresses (bug #340817).
-			uris.append(syncuri)
-
-		# reverse, for use with pop()
-		uris.reverse()
-		uris_orig = uris[:]
-
-		effective_maxretries = maxretries
-		if effective_maxretries < 0:
-			effective_maxretries = len(uris) - 1
-
-		local_state_unchanged = True
-		while (1):
-			if uris:
-				dosyncuri = uris.pop()
-			elif maxretries < 0 or retries > maxretries:
-				writemsg("!!! Exhausted addresses for %s\n"
-					% _unicode_decode(hostname), noiselevel=-1)
-				return (1, False)
-			else:
-				uris.extend(uris_orig)
-				dosyncuri = uris.pop()
-
-			if (retries==0):
-				if "--ask" in opts:
-					uq = UserQuery(opts)
-					if uq.query("Do you want to sync your Portage tree " + \
-						"with the mirror at\n" + blue(dosyncuri) + bold("?"),
-						enter_invalid) == "No":
-						print()
-						print("Quitting.")
-						print()
-						sys.exit(128 + signal.SIGINT)
-				self.logger(self.xterm_titles,
-					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in opts:
-					print(">>> Starting rsync with "+dosyncuri+"...")
-			else:
-				self.logger(self.xterm_titles,
-					">>> Starting retry %d of %d with %s" % \
-						(retries, effective_maxretries, dosyncuri))
-				writemsg_stdout(
-					"\n\n>>> Starting retry %d of %d with %s\n" % \
-					(retries, effective_maxretries, dosyncuri), noiselevel=-1)
-
-			if dosyncuri.startswith('ssh://'):
-				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
-
-			unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
-				dosyncuri, timestamp, opts)
-			if not unchanged:
-				local_state_unchanged = False
-			if is_synced:
-				break
-
-			retries=retries+1
-
-			if maxretries < 0 or retries <= maxretries:
-				print(">>> Retrying...")
-			else:
-				# over retries
-				# exit loop
-				exitcode = EXCEEDED_MAX_RETRIES
-				break
-		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
-
-		# if synced successfully, verify now
-		if exitcode == 0 and self.verify_metamanifest:
-			if gemato is None:
-				writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
-					level=logging.ERROR, noiselevel=-1)
-				exitcode = 127
+			if port is None:
+				port=""
+			if user_name is None:
+				user_name=""
+			if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
+				getaddrinfo_host = hostname
 			else:
-				# Use isolated environment if key is specified,
-				# system environment otherwise
-				if self.repo.sync_openpgp_key_path is not None:
-					openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+				# getaddrinfo needs the brackets stripped
+				getaddrinfo_host = hostname[1:-1]
+			updatecache_flg = False
+			all_rsync_opts = set(self.rsync_opts)
+			all_rsync_opts.update(self.extra_rsync_opts)
+
+			family = socket.AF_UNSPEC
+			if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
+				family = socket.AF_INET
+			elif socket.has_ipv6 and \
+				("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
+				family = socket.AF_INET6
+
+			addrinfos = None
+			uris = []
+
+			try:
+				addrinfos = getaddrinfo_validate(
+					socket.getaddrinfo(getaddrinfo_host, None,
+					family, socket.SOCK_STREAM))
+			except socket.error as e:
+				writemsg_level(
+					"!!! getaddrinfo failed for '%s': %s\n"
+					% (_unicode_decode(hostname), _unicode(e)),
+					noiselevel=-1, level=logging.ERROR)
+
+			if addrinfos:
+
+				AF_INET = socket.AF_INET
+				AF_INET6 = None
+				if socket.has_ipv6:
+					AF_INET6 = socket.AF_INET6
+
+				ips_v4 = []
+				ips_v6 = []
+
+				for addrinfo in addrinfos:
+					if addrinfo[0] == AF_INET:
+						ips_v4.append("%s" % addrinfo[4][0])
+					elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
+						# IPv6 addresses need to be enclosed in square brackets
+						ips_v6.append("[%s]" % addrinfo[4][0])
+
+				random.shuffle(ips_v4)
+				random.shuffle(ips_v6)
+
+				# Give priority to the address family that
+				# getaddrinfo() returned first.
+				if AF_INET6 is not None and addrinfos and \
+					addrinfos[0][0] == AF_INET6:
+					ips = ips_v6 + ips_v4
+				else:
+					ips = ips_v4 + ips_v6
+
+				for ip in ips:
+					uris.append(syncuri.replace(
+						"//" + user_name + hostname + port + "/",
+						"//" + user_name + ip + port + "/", 1))
+
+			if not uris:
+				# With some configurations we need to use the plain hostname
+				# rather than try to resolve the ip addresses (bug #340817).
+				uris.append(syncuri)
+
+			# reverse, for use with pop()
+			uris.reverse()
+			uris_orig = uris[:]
+
+			effective_maxretries = maxretries
+			if effective_maxretries < 0:
+				effective_maxretries = len(uris) - 1
+
+			local_state_unchanged = True
+			while (1):
+				if uris:
+					dosyncuri = uris.pop()
+				elif maxretries < 0 or retries > maxretries:
+					writemsg("!!! Exhausted addresses for %s\n"
+						% _unicode_decode(hostname), noiselevel=-1)
+					return (1, False)
+				else:
+					uris.extend(uris_orig)
+					dosyncuri = uris.pop()
+
+				if (retries==0):
+					if "--ask" in opts:
+						uq = UserQuery(opts)
+						if uq.query("Do you want to sync your Portage tree " + \
+							"with the mirror at\n" + blue(dosyncuri) + bold("?"),
+							enter_invalid) == "No":
+							print()
+							print("Quitting.")
+							print()
+							sys.exit(128 + signal.SIGINT)
+					self.logger(self.xterm_titles,
+						">>> Starting rsync with " + dosyncuri)
+					if "--quiet" not in opts:
+						print(">>> Starting rsync with "+dosyncuri+"...")
+				else:
+					self.logger(self.xterm_titles,
+						">>> Starting retry %d of %d with %s" % \
+							(retries, effective_maxretries, dosyncuri))
+					writemsg_stdout(
+						"\n\n>>> Starting retry %d of %d with %s\n" % \
+						(retries, effective_maxretries, dosyncuri), noiselevel=-1)
+
+				if dosyncuri.startswith('ssh://'):
+					dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
+
+				unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
+					dosyncuri, timestamp, opts)
+				if not unchanged:
+					local_state_unchanged = False
+				if is_synced:
+					break
+
+				retries=retries+1
+
+				if maxretries < 0 or retries <= maxretries:
+					print(">>> Retrying...")
 				else:
-					openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+					# over retries
+					# exit loop
+					exitcode = EXCEEDED_MAX_RETRIES
+					break
+			self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+
+			# if synced successfully, verify now
+			if exitcode == 0 and self.verify_metamanifest:
+				if gemato is None:
+					writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
+						level=logging.ERROR, noiselevel=-1)
+					exitcode = 127
+				else:
+					# Use isolated environment if key is specified,
+					# system environment otherwise
+					if self.repo.sync_openpgp_key_path is not None:
+						openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+					else:
+						openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+
+					try:
+						with openpgp_env_cls() as openpgp_env:
+							if self.repo.sync_openpgp_key_path is not None:
+								out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+								with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+									openpgp_env.import_key(f)
+								out.ebegin('Refreshing keys from keyserver')
+								openpgp_env.refresh_keys()
+								out.eend(0)
+
+							# we always verify the Manifest signature, in case
+							# we had to deal with key revocation case
+							m = gemato.recursiveloader.ManifestRecursiveLoader(
+									os.path.join(self.repo.location, 'Manifest'),
+									verify_openpgp=True,
+									openpgp_env=openpgp_env,
+									max_jobs=self.verify_jobs)
+							if not m.openpgp_signed:
+								raise RuntimeError('OpenPGP signature not found on Manifest')
+
+							ts = m.find_timestamp()
+							if ts is None:
+								raise RuntimeError('Timestamp not found in Manifest')
+
+							out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+							out.einfo('Valid OpenPGP signature found:')
+							out.einfo('- primary key: %s' % (
+								m.openpgp_signature.primary_key_fingerprint))
+							out.einfo('- subkey: %s' % (
+								m.openpgp_signature.fingerprint))
+							out.einfo('- timestamp: %s UTC' % (
+								m.openpgp_signature.timestamp))
+
+							# if nothing has changed, skip the actual Manifest
+							# verification
+							if not local_state_unchanged:
+								out.ebegin('Verifying %s' % (self.repo.location,))
+								m.assert_directory_verifies()
+								out.eend(0)
+					except GematoException as e:
+						writemsg_level("!!! Manifest verification failed:\n%s\n"
+								% (e,),
+								level=logging.ERROR, noiselevel=-1)
+						exitcode = 1
 
-				try:
-					with openpgp_env_cls() as openpgp_env:
-						if self.repo.sync_openpgp_key_path is not None:
-							out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
-							with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
-								openpgp_env.import_key(f)
-							out.ebegin('Refreshing keys from keyserver')
-							openpgp_env.refresh_keys()
-							out.eend(0)
-
-						# we always verify the Manifest signature, in case
-						# we had to deal with key revocation case
-						m = gemato.recursiveloader.ManifestRecursiveLoader(
-								os.path.join(self.repo.location, 'Manifest'),
-								verify_openpgp=True,
-								openpgp_env=openpgp_env,
-								max_jobs=self.verify_jobs)
-						if not m.openpgp_signed:
-							raise RuntimeError('OpenPGP signature not found on Manifest')
-
-						ts = m.find_timestamp()
-						if ts is None:
-							raise RuntimeError('Timestamp not found in Manifest')
-
-						out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
-						out.einfo('Valid OpenPGP signature found:')
-						out.einfo('- primary key: %s' % (
-							m.openpgp_signature.primary_key_fingerprint))
-						out.einfo('- subkey: %s' % (
-							m.openpgp_signature.fingerprint))
-						out.einfo('- timestamp: %s UTC' % (
-							m.openpgp_signature.timestamp))
-
-						# if nothing has changed, skip the actual Manifest
-						# verification
-						if not local_state_unchanged:
-							out.ebegin('Verifying %s' % (self.repo.location,))
-							m.assert_directory_verifies()
-							out.eend(0)
-				except GematoException as e:
-					writemsg_level("!!! Manifest verification failed:\n%s\n"
-							% (e,),
-							level=logging.ERROR, noiselevel=-1)
-					exitcode = 1
-
-		return (exitcode, updatecache_flg)
+			return (exitcode, updatecache_flg)
+		finally:
+			pass
 
 
 	def _process_exitcode(self, exitcode, syncuri, out, maxretries):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     d8b9d5d6250a4f9ebaf8db6cbab96a585c56aadd
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue May  1 19:52:28 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May  1 19:52:28 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8b9d5d6

rsync: surface key refresh exceptions early (bug 649276)

Since retry does not help for some types of errors, display errors as
soon as they occur.

Fixes: 5f29f03d0d97 ("rsync: add key refresh retry (bug 649276)")

 pym/portage/sync/modules/rsync/rsync.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 180bf8571..b2850715e 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -157,9 +157,21 @@ class RsyncSync(NewBase):
 					if retry_decorator is None:
 						openpgp_env.refresh_keys()
 					else:
+						def noisy_refresh_keys():
+							"""
+							Since retry does not help for some types of
+							errors, display errors as soon as they occur.
+							"""
+							try:
+								openpgp_env.refresh_keys()
+							except Exception as e:
+								writemsg_level("%s\n" % (e,),
+									level=logging.ERROR, noiselevel=-1)
+								raise # retry
+
 						loop = global_event_loop()
 						func_coroutine = functools.partial(loop.run_in_executor,
-							None, openpgp_env.refresh_keys)
+							None, noisy_refresh_keys)
 						decorated_func = retry_decorator(func_coroutine)
 						loop.run_until_complete(decorated_func())
 					out.eend(0)

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     35ca4ac2f8f0de61289bf812681c21d14d2524db
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May  3 06:26:19 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May  3 06:35:45 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=35ca4ac2

rsync: silence signature info for --quiet mode (bug 654664)

Silence messages like the following for --quiet mode:

 * Using keys from /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg
 * Refreshing keys from keyserver ...                                    [ ok ]

 * Manifest timestamp: 2018-05-03 05:38:39 UTC
 * Valid OpenPGP signature found:
 * - primary key: DCD05B71EAB94199527F44ACDB6B8C1F96D8BF6D
 * - subkey: E1D6ABB63BFCFB4BA02FDF1CEC590EEAC9189250
 * - timestamp: 2018-05-03 05:38:39 UTC
 * Verifying /usr/portage ...                                            [ ok ]

Bug: https://bugs.gentoo.org/654664

 pym/portage/sync/modules/rsync/rsync.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index b2850715e..1b8941ff6 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -66,7 +66,8 @@ class RsyncSync(NewBase):
 		opts = self.options.get('emerge_config').opts
 		self.usersync_uid = self.options.get('usersync_uid', None)
 		enter_invalid = '--ask-enter-invalid' in opts
-		out = portage.output.EOutput()
+		quiet = '--quiet' in opts
+		out = portage.output.EOutput(quiet=quiet)
 		syncuri = self.repo.sync_uri
 		if self.repo.module_specific_options.get(
 			'sync-rsync-vcs-ignore', 'false').lower() == 'true':
@@ -385,10 +386,12 @@ class RsyncSync(NewBase):
 							raise RuntimeError('Timestamp not found in Manifest')
 						if (self.max_age != 0 and
 								(datetime.datetime.utcnow() - ts.ts).days > self.max_age):
+							out.quiet = False
 							out.ewarn('Manifest is over %d days old, this is suspicious!' % (self.max_age,))
 							out.ewarn('You may want to try using another mirror and/or reporting this one:')
 							out.ewarn('  %s' % (dosyncuri,))
 							out.ewarn('')
+							out.quiet = quiet
 
 						out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
 						out.einfo('Valid OpenPGP signature found:')

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     505707a335683330ac69b1c7f428e8a1c66d0b2e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May  7 08:14:19 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May  7 08:17:18 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=505707a3

rsync: explicitly use ForkExecutor for key refresh retry (bug 654390)

The ThreadPoolExecutor that asyncio uses by default does not support
cancellation of tasks, therefore use ForkExecutor for task cancellation
support, in order to enforce timeouts.

Bug: https://bugs.gentoo.org/654390

 pym/portage/sync/modules/rsync/rsync.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 070798a53..382a1eaae 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -26,6 +26,7 @@ from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
 from portage.util._eventloop.global_event_loop import global_event_loop
 from portage.util import writemsg, writemsg_stdout
 from portage.util.futures import asyncio
+from portage.util.futures.executor.fork import ForkExecutor
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import NewBase
@@ -170,11 +171,16 @@ class RsyncSync(NewBase):
 									level=logging.ERROR, noiselevel=-1)
 								raise # retry
 
+						# The ThreadPoolExecutor that asyncio uses by default
+						# does not support cancellation of tasks, therefore
+						# use ForkExecutor for task cancellation support, in
+						# order to enforce timeouts.
 						loop = global_event_loop()
-						func_coroutine = functools.partial(loop.run_in_executor,
-							None, noisy_refresh_keys)
-						decorated_func = retry_decorator(func_coroutine, loop=loop)
-						loop.run_until_complete(decorated_func())
+						with ForkExecutor(loop=loop) as executor:
+							func_coroutine = functools.partial(loop.run_in_executor,
+								executor, noisy_refresh_keys)
+							decorated_func = retry_decorator(func_coroutine, loop=loop)
+							loop.run_until_complete(decorated_func())
 					out.eend(0)
 				except (GematoException, asyncio.TimeoutError) as e:
 					writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"