[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     cddccc60e653cad0760553b44bedd8c7cd544fe7
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Feb  8 01:50:23 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Feb  8 02:13:48 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cddccc60

Rsync module, add exists() and new()

---
 pym/portage/sync/modules/rsync/rsync.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index de561ad..12b4c9c 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -60,6 +60,10 @@ class RsyncSync(object):
 			spawn_kwargs = self.options.get('spawn_kwargs', None)
 			usersync_uid = self.options.get('usersync_uid', None)
 
+		if not self.exists():
+			if not self.new():
+				return (1, False)
+
 		enter_invalid = '--ask-enter-invalid' in myopts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
@@ -499,3 +503,26 @@ class RsyncSync(object):
 			for line in msg:
 				out.eerror(line)
 		return (exitcode, updatecache_flg)
+
+
+	def exists(self, **kwargs):
+		if kwargs:
+			self._kwargs(kwargs)
+		elif not self.repo:
+			return False
+		return os.path.exists(self.repo.location)
+
+
+
+	def new(self, **kwargs):
+		if kwargs:
+			self._kwargs(kwargs)
+		try:
+			if not os.path.exists(self.repo.location):
+				os.makedirs(self.repo.location)
+				self.logger(self.self.xterm_titles,
+					'Created New Directory %s ' % self.repo.location )
+		except IOError:
+			return False
+		return True
+

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     5555c80220be97ac9cba1440bf580529258a35cd
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Feb  8 01:49:44 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Feb  8 02:13:31 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5555c802

Move some local variables to classwide ones.

---
 pym/portage/sync/modules/rsync/rsync.py | 68 ++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 30 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 6f804c1..de561ad 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -38,33 +38,40 @@ class RsyncSync(object):
 
 
 	def __init__(self):
-		pass
+		self.settings = None
+		self.logger = None
+		self.repo = None
+		self.options = None
+
+
+	def _kwargs(self, kwargs):
+			self.options = kwargs.get('options', {})
+			self.settings = self.options.get('settings', None)
+			self.logger = self.options.get('logger', None)
+			self.repo = self.options.get('repo', None)
+			self.xterm_titles = self.options.get('xterm_titles', False)
 
 
 	def sync(self, **kwargs):
 		'''Rsync the repo'''
 		if kwargs:
-			options = kwargs.get('options', {})
-			settings = options.get('settings', None)
-			logger = options.get('logger', None)
-			myopts = options.get('emerge_config').opts
-			repo = options.get('repo', None)
-			spawn_kwargs = options.get('spawn_kwargs', None)
-			usersync_uid = options.get('usersync_uid', None)
-			xterm_titles = options.get('xterm_titles', False)
+			self._kwargs(kwargs)
+			myopts = self.options.get('emerge_config').opts
+			spawn_kwargs = self.options.get('spawn_kwargs', None)
+			usersync_uid = self.options.get('usersync_uid', None)
 
 		enter_invalid = '--ask-enter-invalid' in myopts
 		out = portage.output.EOutput()
-		syncuri = repo.sync_uri
+		syncuri = self.repo.sync_uri
 		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
-		vcs_dirs = vcs_dirs.intersection(os.listdir(repo.location))
+		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
 
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
-				(repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
+				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
 		rsync_binary = portage.process.find_binary("rsync")
 		if rsync_binary is None:
@@ -74,7 +81,7 @@ class RsyncSync(object):
 		mytimeout=180
 
 		rsync_opts = []
-		if settings["PORTAGE_RSYNC_OPTS"] == "":
+		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
 			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
 			rsync_opts.extend([
 				"--recursive",    # Recurse directories
@@ -101,7 +108,7 @@ class RsyncSync(object):
 
 			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
 			rsync_opts.extend(portage.util.shlex_split(
-				settings.get("PORTAGE_RSYNC_OPTS", "")))
+				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
 			for opt in ("--recursive", "--times"):
 				if opt not in rsync_opts:
 					portage.writemsg(yellow("WARNING:") + " adding required option " + \
@@ -145,7 +152,7 @@ class RsyncSync(object):
 
 		# Real local timestamp file.
 		servertimestampfile = os.path.join(
-			repo.location, "metadata", "timestamp.chk")
+			self.repo.location, "metadata", "timestamp.chk")
 
 		content = portage.util.grabfile(servertimestampfile)
 		mytimestamp = 0
@@ -159,12 +166,12 @@ class RsyncSync(object):
 
 		try:
 			rsync_initial_timeout = \
-				int(settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
+				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
 			rsync_initial_timeout = 15
 
 		try:
-			maxretries=int(settings["PORTAGE_RSYNC_RETRIES"])
+			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
 		except SystemExit as e:
 			raise # Needed else can't exit
 		except:
@@ -180,7 +187,7 @@ class RsyncSync(object):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = settings.get("PORTAGE_SSH_OPTS")
+		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -194,7 +201,7 @@ class RsyncSync(object):
 		updatecache_flg=True
 		all_rsync_opts = set(rsync_opts)
 		extra_rsync_opts = portage.util.shlex_split(
-			settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
+			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
 		all_rsync_opts.update(extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
@@ -280,11 +287,12 @@ class RsyncSync(object):
 						print("Quitting.")
 						print()
 						sys.exit(128 + signal.SIGINT)
-				logger(xterm_titles, ">>> Starting rsync with " + dosyncuri)
+				self.logger(self.xterm_titles,
+					">>> Starting rsync with " + dosyncuri)
 				if "--quiet" not in myopts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
-				logger(xterm_titles,
+				self.logger(self.xterm_titles,
 					">>> Starting retry %d of %d with %s" % \
 						(retries, effective_maxretries, dosyncuri))
 				writemsg_stdout(
@@ -320,7 +328,7 @@ class RsyncSync(object):
 				# requirement, since that's not necessarily true for the
 				# default directory used by the tempfile module.
 				if usersync_uid is not None:
-					tmpdir = settings['PORTAGE_TMPDIR']
+					tmpdir = self.settings['PORTAGE_TMPDIR']
 				else:
 					# use default dir from tempfile module
 					tmpdir = None
@@ -387,7 +395,7 @@ class RsyncSync(object):
 				del mycommand, mypids, content
 			if exitcode == os.EX_OK:
 				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					logger(xterm_titles,
+					self.logger(self.xterm_titles,
 						">>> Cancelling sync -- Already current.")
 					print()
 					print(">>>")
@@ -399,7 +407,7 @@ class RsyncSync(object):
 					print()
 					return (exitcode, updatecache_flg)
 				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					logger(xterm_titles,
+					self.logger(self.xterm_titles,
 						">>> Server out of date: %s" % dosyncuri)
 					print()
 					print(">>>")
@@ -411,7 +419,7 @@ class RsyncSync(object):
 					exitcode = SERVER_OUT_OF_DATE
 				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
 					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", repo.location]
+					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
 					exitcode = None
 					try:
 						exitcode = portage.process.spawn(mycommand,
@@ -461,7 +469,7 @@ class RsyncSync(object):
 				break
 
 		if (exitcode==0):
-			logger(xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -472,14 +480,14 @@ class RsyncSync(object):
 			msg = []
 			if exitcode==1:
 				msg.append("Rsync has reported that there is a syntax error. Please ensure")
-				msg.append("that sync-uri attribute for repository '%s' is proper." % repo.name)
-				msg.append("sync-uri: '%s'" % repo.sync_uri)
+				msg.append("that sync-uri attribute for repository '%s' is proper." % self.repo.name)
+				msg.append("sync-uri: '%s'" % self.repo.sync_uri)
 			elif exitcode==11:
 				msg.append("Rsync has reported that there is a File IO error. Normally")
 				msg.append("this means your disk is full, but can be caused by corruption")
-				msg.append("on the filesystem that contains repository '%s'. Please investigate" % repo.name)
+				msg.append("on the filesystem that contains repository '%s'. Please investigate" % self.repo.name)
 				msg.append("and try again after the problem has been fixed.")
-				msg.append("Location of repository: '%s'" % repo.location)
+				msg.append("Location of repository: '%s'" % self.repo.location)
 			elif exitcode==20:
 				msg.append("Rsync was killed before it finished.")
 			else:

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     4f85bc1ebf00aef9f0cd499d2e3db36f5fb904bb
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Feb  8 03:04:27 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Feb  8 03:05:07 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4f85bc1e

Separate out the actual sync code to a private function.

Fix new() to  return _sync().
Fix sync() to return new() or  _sysnc().

---
 pym/portage/sync/modules/rsync/rsync.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 12b4c9c..1c7abcc 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -56,14 +56,17 @@ class RsyncSync(object):
 		'''Rsync the repo'''
 		if kwargs:
 			self._kwargs(kwargs)
-			myopts = self.options.get('emerge_config').opts
-			spawn_kwargs = self.options.get('spawn_kwargs', None)
-			usersync_uid = self.options.get('usersync_uid', None)
 
 		if not self.exists():
-			if not self.new():
-				return (1, False)
+			return self.new()
+		return self._sync()
+
 
+	def _sync(self):
+		'''Internal sync function which performs only the sync'''
+		myopts = self.options.get('emerge_config').opts
+		spawn_kwargs = self.options.get('spawn_kwargs', None)
+		usersync_uid = self.options.get('usersync_uid', None)
 		enter_invalid = '--ask-enter-invalid' in myopts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
@@ -523,6 +526,6 @@ class RsyncSync(object):
 				self.logger(self.self.xterm_titles,
 					'Created New Directory %s ' % self.repo.location )
 		except IOError:
-			return False
-		return True
+			return (1, False)
+		return self._sync()
 

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     05c9320d9f265ff11daac605f80e33341e1233d3
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Feb  8 07:14:52 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Feb  8 07:14:52 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=05c9320d

Extend the module_spec with the new functions.

---
 pym/portage/sync/modules/rsync/__init__.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/__init__.py b/pym/portage/sync/modules/rsync/__init__.py
index db74ad4..a0239bf 100644
--- a/pym/portage/sync/modules/rsync/__init__.py
+++ b/pym/portage/sync/modules/rsync/__init__.py
@@ -14,8 +14,20 @@ module_spec = {
 			'name': "rsync",
 			'class': "RsyncSync",
 			'description': __doc__,
-			'functions': ['sync',],
-			'func_desc': {'sync': 'Performs rsync transfers on the repository'}
+			'functions': ['sync', 'new', 'exists'],
+			'func_desc': {
+				'sync': 'Performs rsync transfers on the repository',
+				'new': 'Creates the new repository at the specified location',
+				'exists': 'Returns a boolean if the specified directory exists',
+				},
+			'func_parameters': {
+				'kwargs': {
+					'type': dict,
+					'description': 'Standard python **kwargs parameter format' +
+						'Please refer to the sync modules specs at ' +
+						'"https://wiki.gentoo.org:Project:Portage" for details',
+					},
+				},
 			}
 		}
 	}

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     3d82d02319cc844da31d781d6b1416982d1f5177
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Feb  9 22:33:19 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Feb 19 05:09:00 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3d82d023

add missind xterm_titile init.

---
 pym/portage/sync/modules/rsync/rsync.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 1c7abcc..7495706 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -42,6 +42,7 @@ class RsyncSync(object):
 		self.logger = None
 		self.repo = None
 		self.options = None
+		self.xterm_titles = None
 
 
 	def _kwargs(self, kwargs):

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     29820671bf0ee97a5f915ac8a20ac683e3cad009
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 14 15:42:16 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Mar 14 16:18:18 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=29820671

portage/sync/modules/rsync: Use SyncBase class.

---
 pym/portage/sync/modules/rsync/rsync.py | 54 +++++----------------------------
 1 file changed, 7 insertions(+), 47 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 7122db8..0cdf414 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -17,13 +17,14 @@ from portage.output import create_color_func, yellow, blue, bold
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")
 warn = create_color_func("WARN")
-from portage.const import VCS_DIRS, TIMESTAMP_FORMAT
+from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
 from portage.util import writemsg, writemsg_stdout
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.userquery import userquery
+from portage.sync.syncbase import SyncBase
 
 
-class RsyncSync(object):
+class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
 	short_desc = "Perform sync operations on rsync based repositories"
@@ -33,40 +34,13 @@ class RsyncSync(object):
 		return "RsyncSync"
 
 
-	def can_progressbar(self, func):
-		return False
-
-
 	def __init__(self):
-		self.settings = None
-		self.logger = None
-		self.repo = None
-		self.options = None
-		self.xterm_titles = None
-
-
-	def _kwargs(self, kwargs):
-			self.options = kwargs.get('options', {})
-			self.settings = self.options.get('settings', None)
-			self.logger = self.options.get('logger', None)
-			self.repo = self.options.get('repo', None)
-			self.xterm_titles = self.options.get('xterm_titles', False)
-
-
-	def sync(self, **kwargs):
-		'''Rsync the repo'''
-		if kwargs:
-			self._kwargs(kwargs)
-
-		if not self.exists():
-			return self.new()
-		return self._sync()
+		SyncBase.__init__(self, "rsync", RSYNC_PACKAGE_ATOM)
 
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
 		myopts = self.options.get('emerge_config').opts
-		spawn_kwargs = self.options.get('spawn_kwargs', None)
 		usersync_uid = self.options.get('usersync_uid', None)
 		enter_invalid = '--ask-enter-invalid' in myopts
 		out = portage.output.EOutput()
@@ -81,11 +55,6 @@ class RsyncSync(object):
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		rsync_binary = portage.process.find_binary("rsync")
-		if rsync_binary is None:
-			print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.")
-			print("!!! Type \"emerge %s\" to enable rsync support." % portage.const.RSYNC_PACKAGE_ATOM)
-			return (os.EX_UNAVAILABLE, False)
 		mytimeout=180
 
 		rsync_opts = []
@@ -313,7 +282,7 @@ class RsyncSync(object):
 			if mytimestamp != 0 and "--quiet" not in myopts:
 				print(">>> Checking server timestamp ...")
 
-			rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts
+			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
 
 			if proto == 'ssh' and ssh_opts:
 				rsynccommand.append("--rsh=ssh " + ssh_opts)
@@ -363,7 +332,7 @@ class RsyncSync(object):
 
 						mypids.extend(portage.process.spawn(
 							mycommand, returnpid=True,
-							**portage._native_kwargs(spawn_kwargs)))
+							**portage._native_kwargs(self.spawn_kwargs)))
 						exitcode = os.waitpid(mypids[0], 0)[1]
 						if usersync_uid is not None:
 							portage.util.apply_permissions(tmpservertimestampfile,
@@ -431,7 +400,7 @@ class RsyncSync(object):
 					exitcode = None
 					try:
 						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(spawn_kwargs))
+							**portage._native_kwargs(self.spawn_kwargs))
 					finally:
 						if exitcode is None:
 							# interrupted
@@ -509,15 +478,6 @@ class RsyncSync(object):
 		return (exitcode, updatecache_flg)
 
 
-	def exists(self, **kwargs):
-		if kwargs:
-			self._kwargs(kwargs)
-		elif not self.repo:
-			return False
-		return os.path.exists(self.repo.location)
-
-
-
 	def new(self, **kwargs):
 		if kwargs:
 			self._kwargs(kwargs)

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     75db8472ad5584505963cfb931c645a6a10dffa6
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 14 15:42:16 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Mar 29 22:46:20 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=75db8472

portage/sync/modules/rsync: Use SyncBase class.

---
 pym/portage/sync/modules/rsync/rsync.py | 54 +++++----------------------------
 1 file changed, 7 insertions(+), 47 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 7122db8..0cdf414 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -17,13 +17,14 @@ from portage.output import create_color_func, yellow, blue, bold
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")
 warn = create_color_func("WARN")
-from portage.const import VCS_DIRS, TIMESTAMP_FORMAT
+from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
 from portage.util import writemsg, writemsg_stdout
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.userquery import userquery
+from portage.sync.syncbase import SyncBase
 
 
-class RsyncSync(object):
+class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
 	short_desc = "Perform sync operations on rsync based repositories"
@@ -33,40 +34,13 @@ class RsyncSync(object):
 		return "RsyncSync"
 
 
-	def can_progressbar(self, func):
-		return False
-
-
 	def __init__(self):
-		self.settings = None
-		self.logger = None
-		self.repo = None
-		self.options = None
-		self.xterm_titles = None
-
-
-	def _kwargs(self, kwargs):
-			self.options = kwargs.get('options', {})
-			self.settings = self.options.get('settings', None)
-			self.logger = self.options.get('logger', None)
-			self.repo = self.options.get('repo', None)
-			self.xterm_titles = self.options.get('xterm_titles', False)
-
-
-	def sync(self, **kwargs):
-		'''Rsync the repo'''
-		if kwargs:
-			self._kwargs(kwargs)
-
-		if not self.exists():
-			return self.new()
-		return self._sync()
+		SyncBase.__init__(self, "rsync", RSYNC_PACKAGE_ATOM)
 
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
 		myopts = self.options.get('emerge_config').opts
-		spawn_kwargs = self.options.get('spawn_kwargs', None)
 		usersync_uid = self.options.get('usersync_uid', None)
 		enter_invalid = '--ask-enter-invalid' in myopts
 		out = portage.output.EOutput()
@@ -81,11 +55,6 @@ class RsyncSync(object):
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		rsync_binary = portage.process.find_binary("rsync")
-		if rsync_binary is None:
-			print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.")
-			print("!!! Type \"emerge %s\" to enable rsync support." % portage.const.RSYNC_PACKAGE_ATOM)
-			return (os.EX_UNAVAILABLE, False)
 		mytimeout=180
 
 		rsync_opts = []
@@ -313,7 +282,7 @@ class RsyncSync(object):
 			if mytimestamp != 0 and "--quiet" not in myopts:
 				print(">>> Checking server timestamp ...")
 
-			rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts
+			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
 
 			if proto == 'ssh' and ssh_opts:
 				rsynccommand.append("--rsh=ssh " + ssh_opts)
@@ -363,7 +332,7 @@ class RsyncSync(object):
 
 						mypids.extend(portage.process.spawn(
 							mycommand, returnpid=True,
-							**portage._native_kwargs(spawn_kwargs)))
+							**portage._native_kwargs(self.spawn_kwargs)))
 						exitcode = os.waitpid(mypids[0], 0)[1]
 						if usersync_uid is not None:
 							portage.util.apply_permissions(tmpservertimestampfile,
@@ -431,7 +400,7 @@ class RsyncSync(object):
 					exitcode = None
 					try:
 						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(spawn_kwargs))
+							**portage._native_kwargs(self.spawn_kwargs))
 					finally:
 						if exitcode is None:
 							# interrupted
@@ -509,15 +478,6 @@ class RsyncSync(object):
 		return (exitcode, updatecache_flg)
 
 
-	def exists(self, **kwargs):
-		if kwargs:
-			self._kwargs(kwargs)
-		elif not self.repo:
-			return False
-		return os.path.exists(self.repo.location)
-
-
-
 	def new(self, **kwargs):
 		if kwargs:
 			self._kwargs(kwargs)

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     624b13e36a57293450bc957a2022998198624a7b
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 14 15:42:16 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri May  2 23:05:15 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=624b13e3

portage/sync/modules/rsync: Use SyncBase class.

---
 pym/portage/sync/modules/rsync/rsync.py | 54 +++++----------------------------
 1 file changed, 7 insertions(+), 47 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 7122db8..0cdf414 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -17,13 +17,14 @@ from portage.output import create_color_func, yellow, blue, bold
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")
 warn = create_color_func("WARN")
-from portage.const import VCS_DIRS, TIMESTAMP_FORMAT
+from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
 from portage.util import writemsg, writemsg_stdout
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.userquery import userquery
+from portage.sync.syncbase import SyncBase
 
 
-class RsyncSync(object):
+class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
 	short_desc = "Perform sync operations on rsync based repositories"
@@ -33,40 +34,13 @@ class RsyncSync(object):
 		return "RsyncSync"
 
 
-	def can_progressbar(self, func):
-		return False
-
-
 	def __init__(self):
-		self.settings = None
-		self.logger = None
-		self.repo = None
-		self.options = None
-		self.xterm_titles = None
-
-
-	def _kwargs(self, kwargs):
-			self.options = kwargs.get('options', {})
-			self.settings = self.options.get('settings', None)
-			self.logger = self.options.get('logger', None)
-			self.repo = self.options.get('repo', None)
-			self.xterm_titles = self.options.get('xterm_titles', False)
-
-
-	def sync(self, **kwargs):
-		'''Rsync the repo'''
-		if kwargs:
-			self._kwargs(kwargs)
-
-		if not self.exists():
-			return self.new()
-		return self._sync()
+		SyncBase.__init__(self, "rsync", RSYNC_PACKAGE_ATOM)
 
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
 		myopts = self.options.get('emerge_config').opts
-		spawn_kwargs = self.options.get('spawn_kwargs', None)
 		usersync_uid = self.options.get('usersync_uid', None)
 		enter_invalid = '--ask-enter-invalid' in myopts
 		out = portage.output.EOutput()
@@ -81,11 +55,6 @@ class RsyncSync(object):
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		rsync_binary = portage.process.find_binary("rsync")
-		if rsync_binary is None:
-			print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.")
-			print("!!! Type \"emerge %s\" to enable rsync support." % portage.const.RSYNC_PACKAGE_ATOM)
-			return (os.EX_UNAVAILABLE, False)
 		mytimeout=180
 
 		rsync_opts = []
@@ -313,7 +282,7 @@ class RsyncSync(object):
 			if mytimestamp != 0 and "--quiet" not in myopts:
 				print(">>> Checking server timestamp ...")
 
-			rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts
+			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
 
 			if proto == 'ssh' and ssh_opts:
 				rsynccommand.append("--rsh=ssh " + ssh_opts)
@@ -363,7 +332,7 @@ class RsyncSync(object):
 
 						mypids.extend(portage.process.spawn(
 							mycommand, returnpid=True,
-							**portage._native_kwargs(spawn_kwargs)))
+							**portage._native_kwargs(self.spawn_kwargs)))
 						exitcode = os.waitpid(mypids[0], 0)[1]
 						if usersync_uid is not None:
 							portage.util.apply_permissions(tmpservertimestampfile,
@@ -431,7 +400,7 @@ class RsyncSync(object):
 					exitcode = None
 					try:
 						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(spawn_kwargs))
+							**portage._native_kwargs(self.spawn_kwargs))
 					finally:
 						if exitcode is None:
 							# interrupted
@@ -509,15 +478,6 @@ class RsyncSync(object):
 		return (exitcode, updatecache_flg)
 
 
-	def exists(self, **kwargs):
-		if kwargs:
-			self._kwargs(kwargs)
-		elif not self.repo:
-			return False
-		return os.path.exists(self.repo.location)
-
-
-
 	def new(self, **kwargs):
 		if kwargs:
 			self._kwargs(kwargs)

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     1e2afc89d0e9ab3060936df8260eb64c5ab5e829
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri May  2 23:12:15 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1e2afc89

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 510 +++++++++++++++++---------------
 1 file changed, 269 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 0cdf414..e954363 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.userquery import userquery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,12 +44,11 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
@@ -55,97 +58,34 @@ class RsyncSync(SyncBase):
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +96,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +104,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +116,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +185,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -256,7 +194,7 @@ class RsyncSync(SyncBase):
 				return (1, False)
 
 			if (retries==0):
-				if "--ask" in myopts:
+				if "--ask" in opts:
 					if userquery("Do you want to sync your Portage tree " + \
 						"with the mirror at\n" + blue(dosyncuri) + bold("?"),
 						enter_invalid) == "No":
@@ -266,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -279,160 +217,10 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -444,9 +232,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -475,7 +267,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -490,3 +281,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     0c453489bbfaf281d903afb495fd7cef2bc2370a
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue May 13 04:17:42 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0c453489

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 510 +++++++++++++++++---------------
 1 file changed, 268 insertions(+), 242 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 0cdf414..730093f 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.userquery import userquery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -256,7 +193,7 @@ class RsyncSync(SyncBase):
 				return (1, False)
 
 			if (retries==0):
-				if "--ask" in myopts:
+				if "--ask" in opts:
 					if userquery("Do you want to sync your Portage tree " + \
 						"with the mirror at\n" + blue(dosyncuri) + bold("?"),
 						enter_invalid) == "No":
@@ -266,7 +203,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -279,160 +216,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -444,9 +230,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -475,7 +265,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -490,3 +279,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     3c30ee112cc194e6453c7c7cc9a7a55984bf469e
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 14 15:42:16 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Jun 16 15:36:56 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3c30ee11

portage/sync/modules/rsync: Use SyncBase class.

---
 pym/portage/sync/modules/rsync/rsync.py | 54 +++++----------------------------
 1 file changed, 7 insertions(+), 47 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 7122db8..0cdf414 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -17,13 +17,14 @@ from portage.output import create_color_func, yellow, blue, bold
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")
 warn = create_color_func("WARN")
-from portage.const import VCS_DIRS, TIMESTAMP_FORMAT
+from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
 from portage.util import writemsg, writemsg_stdout
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.userquery import userquery
+from portage.sync.syncbase import SyncBase
 
 
-class RsyncSync(object):
+class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
 	short_desc = "Perform sync operations on rsync based repositories"
@@ -33,40 +34,13 @@ class RsyncSync(object):
 		return "RsyncSync"
 
 
-	def can_progressbar(self, func):
-		return False
-
-
 	def __init__(self):
-		self.settings = None
-		self.logger = None
-		self.repo = None
-		self.options = None
-		self.xterm_titles = None
-
-
-	def _kwargs(self, kwargs):
-			self.options = kwargs.get('options', {})
-			self.settings = self.options.get('settings', None)
-			self.logger = self.options.get('logger', None)
-			self.repo = self.options.get('repo', None)
-			self.xterm_titles = self.options.get('xterm_titles', False)
-
-
-	def sync(self, **kwargs):
-		'''Rsync the repo'''
-		if kwargs:
-			self._kwargs(kwargs)
-
-		if not self.exists():
-			return self.new()
-		return self._sync()
+		SyncBase.__init__(self, "rsync", RSYNC_PACKAGE_ATOM)
 
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
 		myopts = self.options.get('emerge_config').opts
-		spawn_kwargs = self.options.get('spawn_kwargs', None)
 		usersync_uid = self.options.get('usersync_uid', None)
 		enter_invalid = '--ask-enter-invalid' in myopts
 		out = portage.output.EOutput()
@@ -81,11 +55,6 @@ class RsyncSync(object):
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		rsync_binary = portage.process.find_binary("rsync")
-		if rsync_binary is None:
-			print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.")
-			print("!!! Type \"emerge %s\" to enable rsync support." % portage.const.RSYNC_PACKAGE_ATOM)
-			return (os.EX_UNAVAILABLE, False)
 		mytimeout=180
 
 		rsync_opts = []
@@ -313,7 +282,7 @@ class RsyncSync(object):
 			if mytimestamp != 0 and "--quiet" not in myopts:
 				print(">>> Checking server timestamp ...")
 
-			rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts
+			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
 
 			if proto == 'ssh' and ssh_opts:
 				rsynccommand.append("--rsh=ssh " + ssh_opts)
@@ -363,7 +332,7 @@ class RsyncSync(object):
 
 						mypids.extend(portage.process.spawn(
 							mycommand, returnpid=True,
-							**portage._native_kwargs(spawn_kwargs)))
+							**portage._native_kwargs(self.spawn_kwargs)))
 						exitcode = os.waitpid(mypids[0], 0)[1]
 						if usersync_uid is not None:
 							portage.util.apply_permissions(tmpservertimestampfile,
@@ -431,7 +400,7 @@ class RsyncSync(object):
 					exitcode = None
 					try:
 						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(spawn_kwargs))
+							**portage._native_kwargs(self.spawn_kwargs))
 					finally:
 						if exitcode is None:
 							# interrupted
@@ -509,15 +478,6 @@ class RsyncSync(object):
 		return (exitcode, updatecache_flg)
 
 
-	def exists(self, **kwargs):
-		if kwargs:
-			self._kwargs(kwargs)
-		elif not self.repo:
-			return False
-		return os.path.exists(self.repo.location)
-
-
-
 	def new(self, **kwargs):
 		if kwargs:
 			self._kwargs(kwargs)

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     4649a5894a5b8e7719dbb39cd3dd60de2288ac3f
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Jun 16 15:36:57 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4649a589

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 510 +++++++++++++++++---------------
 1 file changed, 268 insertions(+), 242 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 0cdf414..730093f 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.userquery import userquery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -256,7 +193,7 @@ class RsyncSync(SyncBase):
 				return (1, False)
 
 			if (retries==0):
-				if "--ask" in myopts:
+				if "--ask" in opts:
 					if userquery("Do you want to sync your Portage tree " + \
 						"with the mirror at\n" + blue(dosyncuri) + bold("?"),
 						enter_invalid) == "No":
@@ -266,7 +203,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -279,160 +216,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -444,9 +230,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -475,7 +265,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -490,3 +279,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     22aa17dc61efbaa00ca24f32216a5ee5fc409221
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jun 16 20:08:41 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Jun 16 20:08:41 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=22aa17dc

sync/modules/rsync: Update the userquery import and use

Userquery has been made a new class in a new file.
This change uses the new UserQuery class.

---
 pym/portage/sync/modules/rsync/rsync.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 730093f..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -20,7 +20,7 @@ warn = create_color_func("WARN")
 from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
 from portage.util import writemsg, writemsg_stdout
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
-from _emerge.userquery import userquery
+from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
@@ -194,7 +194,8 @@ class RsyncSync(SyncBase):
 
 			if (retries==0):
 				if "--ask" in opts:
-					if userquery("Do you want to sync your Portage tree " + \
+					uq = UserQuery(opts)
+					if uq.query("Do you want to sync your Portage tree " + \
 						"with the mirror at\n" + blue(dosyncuri) + bold("?"),
 						enter_invalid) == "No":
 						print()

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     4cf8bff7ecfe84019c0285f5c4d3c86a369d8e4d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 14 15:42:16 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Jun 16 22:42:59 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4cf8bff7

portage/sync/modules/rsync: Use SyncBase class.

---
 pym/portage/sync/modules/rsync/rsync.py | 54 +++++----------------------------
 1 file changed, 7 insertions(+), 47 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 63e1940..d0ee886 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -17,13 +17,14 @@ from portage.output import create_color_func, yellow, blue, bold
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")
 warn = create_color_func("WARN")
-from portage.const import VCS_DIRS, TIMESTAMP_FORMAT
+from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
 from portage.util import writemsg, writemsg_stdout
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.UserQuery import UserQuery
+from portage.sync.syncbase import SyncBase
 
 
-class RsyncSync(object):
+class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
 	short_desc = "Perform sync operations on rsync based repositories"
@@ -33,40 +34,13 @@ class RsyncSync(object):
 		return "RsyncSync"
 
 
-	def can_progressbar(self, func):
-		return False
-
-
 	def __init__(self):
-		self.settings = None
-		self.logger = None
-		self.repo = None
-		self.options = None
-		self.xterm_titles = None
-
-
-	def _kwargs(self, kwargs):
-			self.options = kwargs.get('options', {})
-			self.settings = self.options.get('settings', None)
-			self.logger = self.options.get('logger', None)
-			self.repo = self.options.get('repo', None)
-			self.xterm_titles = self.options.get('xterm_titles', False)
-
-
-	def sync(self, **kwargs):
-		'''Rsync the repo'''
-		if kwargs:
-			self._kwargs(kwargs)
-
-		if not self.exists():
-			return self.new()
-		return self._sync()
+		SyncBase.__init__(self, "rsync", RSYNC_PACKAGE_ATOM)
 
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
 		myopts = self.options.get('emerge_config').opts
-		spawn_kwargs = self.options.get('spawn_kwargs', None)
 		usersync_uid = self.options.get('usersync_uid', None)
 		enter_invalid = '--ask-enter-invalid' in myopts
 		out = portage.output.EOutput()
@@ -81,11 +55,6 @@ class RsyncSync(object):
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		rsync_binary = portage.process.find_binary("rsync")
-		if rsync_binary is None:
-			print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.")
-			print("!!! Type \"emerge %s\" to enable rsync support." % portage.const.RSYNC_PACKAGE_ATOM)
-			return (os.EX_UNAVAILABLE, False)
 		mytimeout=180
 
 		rsync_opts = []
@@ -314,7 +283,7 @@ class RsyncSync(object):
 			if mytimestamp != 0 and "--quiet" not in myopts:
 				print(">>> Checking server timestamp ...")
 
-			rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts
+			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
 
 			if proto == 'ssh' and ssh_opts:
 				rsynccommand.append("--rsh=ssh " + ssh_opts)
@@ -364,7 +333,7 @@ class RsyncSync(object):
 
 						mypids.extend(portage.process.spawn(
 							mycommand, returnpid=True,
-							**portage._native_kwargs(spawn_kwargs)))
+							**portage._native_kwargs(self.spawn_kwargs)))
 						exitcode = os.waitpid(mypids[0], 0)[1]
 						if usersync_uid is not None:
 							portage.util.apply_permissions(tmpservertimestampfile,
@@ -432,7 +401,7 @@ class RsyncSync(object):
 					exitcode = None
 					try:
 						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(spawn_kwargs))
+							**portage._native_kwargs(self.spawn_kwargs))
 					finally:
 						if exitcode is None:
 							# interrupted
@@ -510,15 +479,6 @@ class RsyncSync(object):
 		return (exitcode, updatecache_flg)
 
 
-	def exists(self, **kwargs):
-		if kwargs:
-			self._kwargs(kwargs)
-		elif not self.repo:
-			return False
-		return os.path.exists(self.repo.location)
-
-
-
 	def new(self, **kwargs):
 		if kwargs:
 			self._kwargs(kwargs)

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     b88c68d42c88ec639878803e0b47be7e62849ccc
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Jun 16 22:44:10 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b88c68d4

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++---------------
 1 file changed, 267 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d0ee886..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     ceff59444d26e17d542cbc16c137efa0298ca884
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Sep  3 23:34:53 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ceff5944

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++---------------
 1 file changed, 267 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d0ee886..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     04a4a2193b3d222fa7ca103d07d7179fc0624e33
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Sep  5 20:26:13 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=04a4a219

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++---------------
 1 file changed, 267 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d0ee886..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     f9cd64ef1858c5e76e81cdb92249bc026bedd585
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Sep 27 01:40:15 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f9cd64ef

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++---------------
 1 file changed, 267 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d0ee886..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     a6bc4fb46ac15d213cc5f5591086ccf03c38db06
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Sep 29 17:20:20 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a6bc4fb4

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++---------------
 1 file changed, 267 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d0ee886..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     fc8df8594e5d7b14f75adb080aa928a48c01a448
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue Sep 30 00:42:25 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fc8df859

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++---------------
 1 file changed, 267 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d0ee886..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     3fbed9a9f964ee1b60ebf8d8bb4c94f406847702
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Oct 20 03:48:33 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3fbed9a9

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++---------------
 1 file changed, 267 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d0ee886..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     11eb94988b5797fb3d5f695645b25a897c170eb2
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May  2 17:04:39 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct 21 05:04:06 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=11eb9498

sync/modules/rsync: Refactor legacy code

Split up the _sync() into logical chunks.
Replace out my* variables.
Move constants out of code, to top of the file.

---
 pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++---------------
 1 file changed, 267 insertions(+), 241 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d0ee886..76d83f2 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import SyncBase
 
 
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
 class RsyncSync(SyncBase):
 	'''Rsync sync module'''
 
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase):
 
 	def _sync(self):
 		'''Internal sync function which performs only the sync'''
-		myopts = self.options.get('emerge_config').opts
-		usersync_uid = self.options.get('usersync_uid', None)
-		enter_invalid = '--ask-enter-invalid' in myopts
+		opts = self.options.get('emerge_config').opts
+		self.usersync_uid = self.options.get('usersync_uid', None)
+		enter_invalid = '--ask-enter-invalid' in opts
 		out = portage.output.EOutput()
 		syncuri = self.repo.sync_uri
-		dosyncuri = syncuri
 		vcs_dirs = frozenset(VCS_DIRS)
 		vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
 
-
 		for vcs_dir in vcs_dirs:
 			writemsg_level(("!!! %s appears to be under revision " + \
 				"control (contains %s).\n!!! Aborting rsync sync.\n") % \
 				(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
 			return (1, False)
-		mytimeout=180
+		self.timeout=180
 
 		rsync_opts = []
 		if self.settings["PORTAGE_RSYNC_OPTS"] == "":
-			portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
-			rsync_opts.extend([
-				"--recursive",    # Recurse directories
-				"--links",        # Consider symlinks
-				"--safe-links",   # Ignore links outside of tree
-				"--perms",        # Preserve permissions
-				"--times",        # Preserive mod times
-				"--omit-dir-times",
-				"--compress",     # Compress the data transmitted
-				"--force",        # Force deletion on non-empty dirs
-				"--whole-file",   # Don't do block transfers, only entire files
-				"--delete",       # Delete files that aren't in the master tree
-				"--stats",        # Show final statistics about what was transfered
-				"--human-readable",
-				"--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-				"--exclude=/distfiles",   # Exclude distfiles from consideration
-				"--exclude=/local",       # Exclude local     from consideration
-				"--exclude=/packages",    # Exclude packages  from consideration
-			])
-
-		else:
-			# The below validation is not needed when using the above hardcoded
-			# defaults.
-
-			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(portage.util.shlex_split(
-				self.settings.get("PORTAGE_RSYNC_OPTS", "")))
-			for opt in ("--recursive", "--times"):
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + " adding required option " + \
-					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-					rsync_opts.append(opt)
-
-			for exclude in ("distfiles", "local", "packages"):
-				opt = "--exclude=/%s" % exclude
-				if opt not in rsync_opts:
-					portage.writemsg(yellow("WARNING:") + \
-					" adding required option %s not included in "  % opt + \
-					"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
-					rsync_opts.append(opt)
-
-			if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
-				def rsync_opt_startswith(opt_prefix):
-					for x in rsync_opts:
-						if x.startswith(opt_prefix):
-							return (1, False)
-					return (0, False)
-
-				if not rsync_opt_startswith("--timeout="):
-					rsync_opts.append("--timeout=%d" % mytimeout)
-
-				for opt in ("--compress", "--whole-file"):
-					if opt not in rsync_opts:
-						portage.writemsg(yellow("WARNING:") + " adding required option " + \
-						"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
-						rsync_opts.append(opt)
-
-		if "--quiet" in myopts:
-			rsync_opts.append("--quiet")    # Shut up a lot
+			rsync_opts = self._set_rsync_defaults()
 		else:
-			rsync_opts.append("--verbose")	# Print filelist
-
-		if "--verbose" in myopts:
-			rsync_opts.append("--progress")  # Progress meter for each file
-
-		if "--debug" in myopts:
-			rsync_opts.append("--checksum") # Force checksum on all files
+			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
 		# Real local timestamp file.
-		servertimestampfile = os.path.join(
+		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
 
-		content = portage.util.grabfile(servertimestampfile)
-		mytimestamp = 0
+		content = portage.util.grabfile(self.servertimestampfile)
+		timestamp = 0
 		if content:
 			try:
-				mytimestamp = time.mktime(time.strptime(content[0],
+				timestamp = time.mktime(time.strptime(content[0],
 					TIMESTAMP_FORMAT))
 			except (OverflowError, ValueError):
 				pass
 		del content
 
 		try:
-			rsync_initial_timeout = \
+			self.rsync_initial_timeout = \
 				int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 		except ValueError:
-			rsync_initial_timeout = 15
+			self.rsync_initial_timeout = 15
 
 		try:
 			maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase):
 
 		retries=0
 		try:
-			proto, user_name, hostname, port = re.split(
+			self.proto, user_name, hostname, port = re.split(
 				r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
 				syncuri, maxsplit=4)[1:5]
 		except ValueError:
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase):
 				noiselevel=-1, level=logging.ERROR)
 			return (1, False)
 
-		ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+		self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
 
 		if port is None:
 			port=""
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase):
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
-		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = portage.util.shlex_split(
+		all_rsync_opts = set(self.rsync_opts)
+		self.extra_rsync_opts = portage.util.shlex_split(
 			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
-		all_rsync_opts.update(extra_rsync_opts)
+		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC
 		if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase):
 		if effective_maxretries < 0:
 			effective_maxretries = len(uris) - 1
 
-		SERVER_OUT_OF_DATE = -1
-		EXCEEDED_MAX_RETRIES = -2
 		while (1):
 			if uris:
 				dosyncuri = uris.pop()
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase):
 						sys.exit(128 + signal.SIGINT)
 				self.logger(self.xterm_titles,
 					">>> Starting rsync with " + dosyncuri)
-				if "--quiet" not in myopts:
+				if "--quiet" not in opts:
 					print(">>> Starting rsync with "+dosyncuri+"...")
 			else:
 				self.logger(self.xterm_titles,
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			if mytimestamp != 0 and "--quiet" not in myopts:
-				print(">>> Checking server timestamp ...")
-
-			rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts
-
-			if proto == 'ssh' and ssh_opts:
-				rsynccommand.append("--rsh=ssh " + ssh_opts)
-
-			if "--debug" in myopts:
-				print(rsynccommand)
-
-			exitcode = os.EX_OK
-			servertimestamp = 0
-			# Even if there's no timestamp available locally, fetch the
-			# timestamp anyway as an initial probe to verify that the server is
-			# responsive.  This protects us from hanging indefinitely on a
-			# connection attempt to an unresponsive server which rsync's
-			# --timeout option does not prevent.
-			if True:
-				# Temporary file for remote server timestamp comparison.
-				# NOTE: If FEATURES=usersync is enabled then the tempfile
-				# needs to be in a directory that's readable by the usersync
-				# user. We assume that PORTAGE_TMPDIR will satisfy this
-				# requirement, since that's not necessarily true for the
-				# default directory used by the tempfile module.
-				if usersync_uid is not None:
-					tmpdir = self.settings['PORTAGE_TMPDIR']
-				else:
-					# use default dir from tempfile module
-					tmpdir = None
-				fd, tmpservertimestampfile = \
-					tempfile.mkstemp(dir=tmpdir)
-				os.close(fd)
-				if usersync_uid is not None:
-					portage.util.apply_permissions(tmpservertimestampfile,
-						uid=usersync_uid)
-				mycommand = rsynccommand[:]
-				mycommand.append(dosyncuri.rstrip("/") + \
-					"/metadata/timestamp.chk")
-				mycommand.append(tmpservertimestampfile)
-				content = None
-				mypids = []
-				try:
-					# Timeout here in case the server is unresponsive.  The
-					# --timeout rsync option doesn't apply to the initial
-					# connection attempt.
-					try:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.register(
-								rsync_initial_timeout)
-
-						mypids.extend(portage.process.spawn(
-							mycommand, returnpid=True,
-							**portage._native_kwargs(self.spawn_kwargs)))
-						exitcode = os.waitpid(mypids[0], 0)[1]
-						if usersync_uid is not None:
-							portage.util.apply_permissions(tmpservertimestampfile,
-								uid=os.getuid())
-						content = portage.grabfile(tmpservertimestampfile)
-					finally:
-						if rsync_initial_timeout:
-							portage.exception.AlarmSignal.unregister()
-						try:
-							os.unlink(tmpservertimestampfile)
-						except OSError:
-							pass
-				except portage.exception.AlarmSignal:
-					# timed out
-					print('timed out')
-					# With waitpid and WNOHANG, only check the
-					# first element of the tuple since the second
-					# element may vary (bug #337465).
-					if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
-						os.kill(mypids[0], signal.SIGTERM)
-						os.waitpid(mypids[0], 0)
-					# This is the same code rsync uses for timeout.
-					exitcode = 30
-				else:
-					if exitcode != os.EX_OK:
-						if exitcode & 0xff:
-							exitcode = (exitcode & 0xff) << 8
-						else:
-							exitcode = exitcode >> 8
-
-				if content:
-					try:
-						servertimestamp = time.mktime(time.strptime(
-							content[0], TIMESTAMP_FORMAT))
-					except (OverflowError, ValueError):
-						pass
-				del mycommand, mypids, content
-			if exitcode == os.EX_OK:
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Cancelling sync -- Already current.")
-					print()
-					print(">>>")
-					print(">>> Timestamps on the server and in the local repository are the same.")
-					print(">>> Cancelling all further sync action. You are already up to date.")
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					return (exitcode, updatecache_flg)
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					self.logger(self.xterm_titles,
-						">>> Server out of date: %s" % dosyncuri)
-					print()
-					print(">>>")
-					print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
-					print(">>>")
-					print(">>> In order to force sync, remove '%s'." % servertimestampfile)
-					print(">>>")
-					print()
-					exitcode = SERVER_OUT_OF_DATE
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand = rsynccommand + [dosyncuri+"/", self.repo.location]
-					exitcode = None
-					try:
-						exitcode = portage.process.spawn(mycommand,
-							**portage._native_kwargs(self.spawn_kwargs))
-					finally:
-						if exitcode is None:
-							# interrupted
-							exitcode = 128 + signal.SIGINT
-
-						#   0	Success
-						#   1	Syntax or usage error
-						#   2	Protocol incompatibility
-						#   5	Error starting client-server protocol
-						#  35	Timeout waiting for daemon connection
-						if exitcode not in (0, 1, 2, 5, 35):
-							# If the exit code is not among those listed above,
-							# then we may have a partial/inconsistent sync
-							# state, so our previously read timestamp as well
-							# as the corresponding file can no longer be
-							# trusted.
-							mytimestamp = 0
-							try:
-								os.unlink(servertimestampfile)
-							except OSError:
-								pass
-
-					if exitcode in [0,1,3,4,11,14,20,21]:
-						break
-			elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			if is_synced:
 				break
-			else:
-				# Code 2 indicates protocol incompatibility, which is expected
-				# for servers with protocol < 29 that don't support
-				# --prune-empty-directories.  Retry for a server that supports
-				# at least rsync protocol version 29 (>=rsync-2.6.4).
-				pass
 
 			retries=retries+1
 
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase):
 				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
+		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+		return (exitcode, updatecache_flg)
+
 
+	def _process_exitcode(self, exitcode, syncuri, out, maxretries):
 		if (exitcode==0):
-			self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri)
+			self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
 		elif exitcode == SERVER_OUT_OF_DATE:
 			exitcode = 1
 		elif exitcode == EXCEEDED_MAX_RETRIES:
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase):
 				msg.append("(and possibly your system's filesystem) configuration.")
 			for line in msg:
 				out.eerror(line)
-		return (exitcode, updatecache_flg)
 
 
 	def new(self, **kwargs):
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase):
 			return (1, False)
 		return self._sync()
 
+
+	def _set_rsync_defaults(self):
+		portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+		rsync_opts = [
+			"--recursive",    # Recurse directories
+			"--links",        # Consider symlinks
+			"--safe-links",   # Ignore links outside of tree
+			"--perms",        # Preserve permissions
+			"--times",        # Preserive mod times
+			"--omit-dir-times",
+			"--compress",     # Compress the data transmitted
+			"--force",        # Force deletion on non-empty dirs
+			"--whole-file",   # Don't do block transfers, only entire files
+			"--delete",       # Delete files that aren't in the master tree
+			"--stats",        # Show final statistics about what was transfered
+			"--human-readable",
+			"--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+			"--exclude=/distfiles",   # Exclude distfiles from consideration
+			"--exclude=/local",       # Exclude local     from consideration
+			"--exclude=/packages",    # Exclude packages  from consideration
+		]
+		return rsync_opts
+
+
+	def _validate_rsync_opts(self, rsync_opts, syncuri):
+		# The below validation is not needed when using the above hardcoded
+		# defaults.
+
+		portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+		rsync_opts.extend(portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+		for opt in ("--recursive", "--times"):
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + " adding required option " + \
+				"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+				rsync_opts.append(opt)
+
+		for exclude in ("distfiles", "local", "packages"):
+			opt = "--exclude=/%s" % exclude
+			if opt not in rsync_opts:
+				portage.writemsg(yellow("WARNING:") + \
+				" adding required option %s not included in "  % opt + \
+				"PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+				rsync_opts.append(opt)
+
+		if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+			def rsync_opt_startswith(opt_prefix):
+				for x in rsync_opts:
+					if x.startswith(opt_prefix):
+						return (1, False)
+				return (0, False)
+
+			if not rsync_opt_startswith("--timeout="):
+				rsync_opts.append("--timeout=%d" % self.timeout)
+
+			for opt in ("--compress", "--whole-file"):
+				if opt not in rsync_opts:
+					portage.writemsg(yellow("WARNING:") + " adding required option " + \
+					"%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+					rsync_opts.append(opt)
+		return rsync_opts
+
+
+	@staticmethod
+	def _rsync_opts_extend(opts, rsync_opts):
+		if "--quiet" in opts:
+			rsync_opts.append("--quiet")    # Shut up a lot
+		else:
+			rsync_opts.append("--verbose")	# Print filelist
+
+		if "--verbose" in opts:
+			rsync_opts.append("--progress")  # Progress meter for each file
+
+		if "--debug" in opts:
+			rsync_opts.append("--checksum") # Force checksum on all files
+		return rsync_opts
+
+
+	def _do_rsync(self, syncuri, timestamp, opts):
+		is_synced = False
+		if timestamp != 0 and "--quiet" not in opts:
+			print(">>> Checking server timestamp ...")
+
+		rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+		if self.proto == 'ssh' and self.ssh_opts:
+			rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+		if "--debug" in opts:
+			print(rsynccommand)
+
+		exitcode = os.EX_OK
+		servertimestamp = 0
+		# Even if there's no timestamp available locally, fetch the
+		# timestamp anyway as an initial probe to verify that the server is
+		# responsive.  This protects us from hanging indefinitely on a
+		# connection attempt to an unresponsive server which rsync's
+		# --timeout option does not prevent.
+
+		#if True:
+		# Temporary file for remote server timestamp comparison.
+		# NOTE: If FEATURES=usersync is enabled then the tempfile
+		# needs to be in a directory that's readable by the usersync
+		# user. We assume that PORTAGE_TMPDIR will satisfy this
+		# requirement, since that's not necessarily true for the
+		# default directory used by the tempfile module.
+		if self.usersync_uid is not None:
+			tmpdir = self.settings['PORTAGE_TMPDIR']
+		else:
+			# use default dir from tempfile module
+			tmpdir = None
+		fd, tmpservertimestampfile = \
+			tempfile.mkstemp(dir=tmpdir)
+		os.close(fd)
+		if self.usersync_uid is not None:
+			portage.util.apply_permissions(tmpservertimestampfile,
+				uid=self.usersync_uid)
+		command = rsynccommand[:]
+		command.append(syncuri.rstrip("/") + \
+			"/metadata/timestamp.chk")
+		command.append(tmpservertimestampfile)
+		content = None
+		pids = []
+		try:
+			# Timeout here in case the server is unresponsive.  The
+			# --timeout rsync option doesn't apply to the initial
+			# connection attempt.
+			try:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.register(
+						self.rsync_initial_timeout)
+
+				pids.extend(portage.process.spawn(
+					command, returnpid=True,
+					**portage._native_kwargs(self.spawn_kwargs)))
+				exitcode = os.waitpid(pids[0], 0)[1]
+				if self.usersync_uid is not None:
+					portage.util.apply_permissions(tmpservertimestampfile,
+						uid=os.getuid())
+				content = portage.grabfile(tmpservertimestampfile)
+			finally:
+				if self.rsync_initial_timeout:
+					portage.exception.AlarmSignal.unregister()
+				try:
+					os.unlink(tmpservertimestampfile)
+				except OSError:
+					pass
+		except portage.exception.AlarmSignal:
+			# timed out
+			print('timed out')
+			# With waitpid and WNOHANG, only check the
+			# first element of the tuple since the second
+			# element may vary (bug #337465).
+			if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+				os.kill(pids[0], signal.SIGTERM)
+				os.waitpid(pids[0], 0)
+			# This is the same code rsync uses for timeout.
+			exitcode = 30
+		else:
+			if exitcode != os.EX_OK:
+				if exitcode & 0xff:
+					exitcode = (exitcode & 0xff) << 8
+				else:
+					exitcode = exitcode >> 8
+
+		if content:
+			try:
+				servertimestamp = time.mktime(time.strptime(
+					content[0], TIMESTAMP_FORMAT))
+			except (OverflowError, ValueError):
+				pass
+		del command, pids, content
+
+		if exitcode == os.EX_OK:
+			if (servertimestamp != 0) and (servertimestamp == timestamp):
+				self.logger(self.xterm_titles,
+					">>> Cancelling sync -- Already current.")
+				print()
+				print(">>>")
+				print(">>> Timestamps on the server and in the local repository are the same.")
+				print(">>> Cancelling all further sync action. You are already up to date.")
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				return is_synced, exitcode
+			elif (servertimestamp != 0) and (servertimestamp < timestamp):
+				self.logger(self.xterm_titles,
+					">>> Server out of date: %s" % syncuri)
+				print()
+				print(">>>")
+				print(">>> SERVER OUT OF DATE: %s" % syncuri)
+				print(">>>")
+				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+				print(">>>")
+				print()
+				exitcode = SERVER_OUT_OF_DATE
+			elif (servertimestamp == 0) or (servertimestamp > timestamp):
+				# actual sync
+				command = rsynccommand + [syncuri+"/", self.repo.location]
+				exitcode = None
+				try:
+					exitcode = portage.process.spawn(command,
+						**portage._native_kwargs(self.spawn_kwargs))
+				finally:
+					if exitcode is None:
+						# interrupted
+						exitcode = 128 + signal.SIGINT
+
+					#   0	Success
+					#   1	Syntax or usage error
+					#   2	Protocol incompatibility
+					#   5	Error starting client-server protocol
+					#  35	Timeout waiting for daemon connection
+					if exitcode not in (0, 1, 2, 5, 35):
+						# If the exit code is not among those listed above,
+						# then we may have a partial/inconsistent sync
+						# state, so our previously read timestamp as well
+						# as the corresponding file can no longer be
+						# trusted.
+						timestamp = 0
+						try:
+							os.unlink(self.servertimestampfile)
+						except OSError:
+							pass
+
+				if exitcode in [0,1,3,4,11,14,20,21]:
+					is_synced = True
+		elif exitcode in [1,3,4,11,14,20,21]:
+			is_synced = True
+		else:
+			# Code 2 indicates protocol incompatibility, which is expected
+			# for servers with protocol < 29 that don't support
+			# --prune-empty-directories.  Retry for a server that supports
+			# at least rsync protocol version 29 (>=rsync-2.6.4).
+			pass
+		return is_synced, exitcode

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     b1fddaac107100a6722008e7ae646531b223d794
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 22 10:59:32 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Oct 22 10:59:32 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b1fddaac

RsyncSync: support file:// sync-uri

This will be useful for unit tests that will sync from a local
file:// sync-uri.

---
 pym/portage/sync/modules/rsync/rsync.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 76d83f2..74c10e7 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -66,6 +66,9 @@ class RsyncSync(SyncBase):
 			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
 		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
+		self.extra_rsync_opts = portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
+
 		# Real local timestamp file.
 		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
@@ -93,6 +96,14 @@ class RsyncSync(SyncBase):
 		except:
 			maxretries = -1 #default number of retries
 
+		if syncuri.startswith("file://"):
+			self.proto = "file"
+			dosyncuri = syncuri[6:]
+			is_synced, exitcode = self._do_rsync(
+				dosyncuri, timestamp, opts)
+			self._process_exitcode(exitcode, dosyncuri, out, 1)
+			return (exitcode, exitcode == os.EX_OK)
+
 		retries=0
 		try:
 			self.proto, user_name, hostname, port = re.split(
@@ -116,8 +127,6 @@ class RsyncSync(SyncBase):
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
 		all_rsync_opts = set(self.rsync_opts)
-		self.extra_rsync_opts = portage.util.shlex_split(
-			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
 		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Michał Górny]

commit:     be0a1641e0a69ea04a54a7da00d6dcb402b552e6
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 22 10:59:32 2014 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Dec  1 21:49:42 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=be0a1641

RsyncSync: support file:// sync-uri

This will be useful for unit tests that will sync from a local
file:// sync-uri.

---
 pym/portage/sync/modules/rsync/rsync.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 76d83f2..74c10e7 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -66,6 +66,9 @@ class RsyncSync(SyncBase):
 			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
 		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
+		self.extra_rsync_opts = portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
+
 		# Real local timestamp file.
 		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
@@ -93,6 +96,14 @@ class RsyncSync(SyncBase):
 		except:
 			maxretries = -1 #default number of retries
 
+		if syncuri.startswith("file://"):
+			self.proto = "file"
+			dosyncuri = syncuri[6:]
+			is_synced, exitcode = self._do_rsync(
+				dosyncuri, timestamp, opts)
+			self._process_exitcode(exitcode, dosyncuri, out, 1)
+			return (exitcode, exitcode == os.EX_OK)
+
 		retries=0
 		try:
 			self.proto, user_name, hostname, port = re.split(
@@ -116,8 +127,6 @@ class RsyncSync(SyncBase):
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
 		all_rsync_opts = set(self.rsync_opts)
-		self.extra_rsync_opts = portage.util.shlex_split(
-			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
 		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC

[gentoo-commits] proj/portage:plugin-sync commit in: pym/portage/sync/modules/rsync/

[Brian Dolbec]

commit:     dfb277c58f8b0d55c1b0c9884cd91f3204888b38
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 22 10:59:32 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:35 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=dfb277c5

RsyncSync: support file:// sync-uri

This will be useful for unit tests that will sync from a local
file:// sync-uri.

---
 pym/portage/sync/modules/rsync/rsync.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 76d83f2..74c10e7 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -66,6 +66,9 @@ class RsyncSync(SyncBase):
 			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
 		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
+		self.extra_rsync_opts = portage.util.shlex_split(
+			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
+
 		# Real local timestamp file.
 		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
@@ -93,6 +96,14 @@ class RsyncSync(SyncBase):
 		except:
 			maxretries = -1 #default number of retries
 
+		if syncuri.startswith("file://"):
+			self.proto = "file"
+			dosyncuri = syncuri[6:]
+			is_synced, exitcode = self._do_rsync(
+				dosyncuri, timestamp, opts)
+			self._process_exitcode(exitcode, dosyncuri, out, 1)
+			return (exitcode, exitcode == os.EX_OK)
+
 		retries=0
 		try:
 			self.proto, user_name, hostname, port = re.split(
@@ -116,8 +127,6 @@ class RsyncSync(SyncBase):
 			getaddrinfo_host = hostname[1:-1]
 		updatecache_flg=True
 		all_rsync_opts = set(self.rsync_opts)
-		self.extra_rsync_opts = portage.util.shlex_split(
-			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
 		all_rsync_opts.update(self.extra_rsync_opts)
 
 		family = socket.AF_UNSPEC