public inbox for gentoo-portage-dev@lists.gentoo.org
 help / color / mirror / Atom feed
* [gentoo-portage-dev] [PATCH] conf: Enable to set rsync extra opts per repository
@ 2015-06-17 18:40 Étienne Buira
  2015-06-17 20:32 ` Brian Dolbec
  0 siblings, 1 reply; 15+ messages in thread
From: Étienne Buira @ 2015-06-17 18:40 UTC (permalink / raw
  To: gentoo-portage-dev

---
 man/portage.5                             |  5 +++
 pym/portage/package/ebuild/config.py      |  2 +
 pym/portage/repository/config.py          | 24 +++++++---
 pym/portage/sync/modules/rsync/rsync.py   |  6 ++-
 pym/portage/tests/sync/test_sync_local.py | 73 ++++++++++++++++++++++++++++---
 5 files changed, 95 insertions(+), 15 deletions(-)

diff --git a/man/portage.5 b/man/portage.5
index e77fc6e..e84142a 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1021,6 +1021,11 @@ group id will be changed.
 .br
 This key takes precedence over FEATURES=userpriv. If user or group id
 is provided, Portage no longer uses owner of the directory.
+.TP
+.B sync-rsync-extra-opts
+Extra options to give to rsync on repository synchronization. It takes
+precedence over a declaration in [DEFAULT] section, that takes
+precedence over PORTAGE_RSYNC_EXTRA_OPTS.
 .RE
 
 .I Example:
diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
index 3a4007b..08db363 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -515,6 +515,8 @@ class config(object):
 				v = confs.get("SYNC")
 				if v is not None:
 					portdir_sync = v
+				if 'PORTAGE_RSYNC_EXTRA_OPTS' in confs:
+					self['PORTAGE_RSYNC_EXTRA_OPTS'] = confs['PORTAGE_RSYNC_EXTRA_OPTS']
 
 			self["PORTDIR"] = portdir
 			self["PORTDIR_OVERLAY"] = portdir_overlay
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index b7c969d..196b87a 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -87,7 +87,7 @@ class RepoConfig(object):
 		'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
 		'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority',
 		'profile_formats', 'sign_commit', 'sign_manifest', 'sync_cvs_repo',
-		'sync_depth',
+		'sync_depth', 'sync_rsync_extra_opts',
 		'sync_type', 'sync_umask', 'sync_uri', 'sync_user', 'thin_manifest',
 		'update_changelog', 'user_location', '_eapis_banned',
 		'_eapis_deprecated', '_masters_orig')
@@ -180,6 +180,8 @@ class RepoConfig(object):
 
 		self.sync_depth = repo_opts.get('sync-depth')
 
+		self.sync_rsync_extra_opts = repo_opts.get('sync-rsync-extra-opts', None)
+
 		# Not implemented.
 		format = repo_opts.get('format')
 		if format is not None:
@@ -415,6 +417,8 @@ class RepoConfig(object):
 			repo_msg.append(indent + "sync-umask: " + self.sync_umask)
 		if self.sync_uri:
 			repo_msg.append(indent + "sync-uri: " + self.sync_uri)
+		if self.sync_rsync_extra_opts:
+			repo_msg.append(indent + "sync-rsync-extra-opts: " + self.sync_rsync_extra_opts)
 		if self.sync_user:
 			repo_msg.append(indent + "sync-user: " + self.sync_user)
 		if self.masters:
@@ -477,6 +481,9 @@ class RepoConfigLoader(object):
 		if prepos['DEFAULT'].masters is not None:
 			default_repo_opts['masters'] = \
 				' '.join(prepos['DEFAULT'].masters)
+		if prepos['DEFAULT'].sync_rsync_extra_opts is not None:
+			default_repo_opts['sync-rsync-extra-opts'] = \
+				prepos['DEFAULT'].sync_rsync_extra_opts
 
 		if overlays:
 			# We need a copy of the original repos.conf data, since we're
@@ -504,7 +511,7 @@ class RepoConfigLoader(object):
 						# repos.conf is allowed to override.
 						for k in ('aliases', 'auto_sync', 'eclass_overrides',
 							'force', 'masters', 'priority', 'sync_cvs_repo',
-							'sync_depth',
+							'sync_depth', 'sync_rsync_extra_opts',
 							'sync_type', 'sync_umask', 'sync_uri', 'sync_user',
 							):
 							v = getattr(repos_conf_opts, k, None)
@@ -543,9 +550,9 @@ class RepoConfigLoader(object):
 		return portdir
 
 	@staticmethod
-	def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir):
+	def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir, default_opts):
 		"""Parse files in paths to load config"""
-		parser = SafeConfigParser()
+		parser = SafeConfigParser(defaults=default_opts)
 
 		# use read_file/readfp in order to control decoding of unicode
 		try:
@@ -615,6 +622,7 @@ class RepoConfigLoader(object):
 		treemap = {}
 		ignored_map = {}
 		ignored_location_map = {}
+		default_opts = {}
 
 		if "PORTAGE_REPOSITORIES" in settings:
 			portdir = ""
@@ -627,10 +635,13 @@ class RepoConfigLoader(object):
 			# deprecated portdir_sync
 			portdir_sync = settings.get("SYNC", "")
 
+		default_opts['sync-rsync-extra-opts'] = \
+			settings.get("PORTAGE_RSYNC_EXTRA_OPTS", None)
+
 		try:
 			self._parse(paths, prepos, ignored_map,
 				ignored_location_map, settings.local_config,
-				portdir)
+				portdir, default_opts)
 		except ConfigParserError as e:
 			writemsg(
 				_("!!! Error while reading repo config file: %s\n") % e,
@@ -962,7 +973,8 @@ class RepoConfigLoader(object):
 	def config_string(self):
 		str_or_int_keys = ("auto_sync", "format", "location",
 			"main_repo", "priority", "sync_cvs_repo",
-			"sync_type", "sync_umask", "sync_uri", 'sync_user')
+			"sync_type", "sync_umask", "sync_uri", 'sync_user',
+			'sync_rsync_extra_opts')
 		str_tuple_keys = ("aliases", "eclass_overrides", "force")
 		repo_config_tuple_keys = ("masters",)
 		keys = str_or_int_keys + str_tuple_keys + repo_config_tuple_keys
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index d84c36d..1f09f60 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -72,8 +72,10 @@ class RsyncSync(NewBase):
 			rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
 		self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
 
-		self.extra_rsync_opts = portage.util.shlex_split(
-			self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
+		self.extra_rsync_opts = list()
+		if self.repo.sync_rsync_extra_opts is not None:
+			self.extra_rsync_opts.extend(portage.util.shlex_split(
+				self.repo.sync_rsync_extra_opts))
 
 		# Real local timestamp file.
 		self.servertimestampfile = os.path.join(
diff --git a/pym/portage/tests/sync/test_sync_local.py b/pym/portage/tests/sync/test_sync_local.py
index 65c20f8..f50caba 100644
--- a/pym/portage/tests/sync/test_sync_local.py
+++ b/pym/portage/tests/sync/test_sync_local.py
@@ -7,7 +7,7 @@ import textwrap
 import time
 
 import portage
-from portage import os, shutil
+from portage import os, shutil, _shell_quote
 from portage import _unicode_decode
 from portage.const import PORTAGE_PYM_PATH, TIMESTAMP_FORMAT
 from portage.process import find_binary
@@ -36,11 +36,14 @@ class SyncLocalTestCase(TestCase):
 			return
 
 		repos_conf = textwrap.dedent("""
+			[DEFAULT]
+			%(default_keys)s
 			[test_repo]
 			location = %(EPREFIX)s/var/repositories/test_repo
 			sync-type = %(sync-type)s
 			sync-uri = file:/%(EPREFIX)s/var/repositories/test_repo_sync
 			auto-sync = yes
+			%(repo_extra_keys)s
 		""")
 
 		profile = {
@@ -73,9 +76,17 @@ class SyncLocalTestCase(TestCase):
 		committer_name = "Gentoo Dev"
 		committer_email = "gentoo-dev@gentoo.org"
 
-		def change_sync_type(sync_type):
-			env["PORTAGE_REPOSITORIES"] = repos_conf % \
-				{"EPREFIX": eprefix, "sync-type": sync_type}
+		def repos_set_conf(sync_type, dflt_keys=None, xtra_keys=None):
+			env["PORTAGE_REPOSITORIES"] = repos_conf % {\
+				"EPREFIX": eprefix, "sync-type": sync_type,
+				"default_keys": "" if dflt_keys is None else dflt_keys,
+				"repo_extra_keys": "" if xtra_keys is None else xtra_keys}
+
+		def alter_ebuild():
+			with open(os.path.join(repo.location + "_sync",
+				"dev-libs", "A", "A-0.ebuild"), "a") as f:
+				f.write("\n")
+			os.unlink(os.path.join(metadata_dir, 'timestamp.chk'))
 
 		sync_cmds = (
 			(homedir, cmds["emerge"] + ("--sync",)),
@@ -90,6 +101,53 @@ class SyncLocalTestCase(TestCase):
 				repo.location + "_sync")),
 		)
 
+		rsync_opts_repos = (
+			(homedir, alter_ebuild),
+			(homedir, lambda: repos_set_conf("rsync", None,
+				"sync-rsync-extra-opts = --backup --backup-dir=%s" %
+				_shell_quote(repo.location + "_back"))),
+			(homedir, cmds['emerge'] + ("--sync",)),
+			(homedir, lambda: self.assertTrue(os.path.exists(
+				repo.location + "_back"))),
+			(homedir, lambda: shutil.rmtree(repo.location + "_back")),
+			(homedir, lambda: repos_set_conf("rsync")),
+		)
+
+		rsync_opts_repos_default = (
+			(homedir, alter_ebuild),
+			(homedir, lambda: repos_set_conf("rsync",
+					"sync-rsync-extra-opts = --backup --backup-dir=%s" %
+					_shell_quote(repo.location+"_back"))),
+			(homedir, cmds['emerge'] + ("--sync",)),
+			(homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
+			(homedir, lambda: shutil.rmtree(repo.location + "_back")),
+			(homedir, lambda: repos_set_conf("rsync")),
+		)
+
+		rsync_opts_repos_default_ovr = (
+			(homedir, alter_ebuild),
+			(homedir, lambda: repos_set_conf("rsync",
+				"sync-rsync-extra-opts = --backup --backup-dir=%s" %
+				_shell_quote(repo.location + "_back_nowhere"),
+				"sync-rsync-extra-opts = --backup --backup-dir=%s" %
+				_shell_quote(repo.location + "_back"))),
+			(homedir, cmds['emerge'] + ("--sync",)),
+			(homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
+			(homedir, lambda: shutil.rmtree(repo.location + "_back")),
+			(homedir, lambda: repos_set_conf("rsync")),
+		)
+
+		rsync_opts_repos_default_cancel = (
+			(homedir, alter_ebuild),
+			(homedir, lambda: repos_set_conf("rsync",
+				"sync-rsync-extra-opts = --backup --backup-dir=%s" %
+				_shell_quote(repo.location + "_back_nowhere"),
+				"sync-rsync-extra-opts = ")),
+			(homedir, cmds['emerge'] + ("--sync",)),
+			(homedir, lambda: self.assertFalse(os.path.exists(repo.location + "_back"))),
+			(homedir, lambda: repos_set_conf("rsync")),
+		)
+
 		delete_sync_repo = (
 			(homedir, lambda: shutil.rmtree(
 				repo.location + "_sync")),
@@ -107,7 +165,7 @@ class SyncLocalTestCase(TestCase):
 		)
 
 		sync_type_git = (
-			(homedir, lambda: change_sync_type("git")),
+			(homedir, lambda: repos_set_conf("git")),
 		)
 
 		pythonpath =  os.environ.get("PYTHONPATH")
@@ -132,10 +190,9 @@ class SyncLocalTestCase(TestCase):
 			"PATH" : os.environ["PATH"],
 			"PORTAGE_GRPNAME" : os.environ["PORTAGE_GRPNAME"],
 			"PORTAGE_USERNAME" : os.environ["PORTAGE_USERNAME"],
-			"PORTAGE_REPOSITORIES" : repos_conf %
-				{"EPREFIX": eprefix, "sync-type": "rsync"},
 			"PYTHONPATH" : pythonpath,
 		}
+		repos_set_conf("rsync")
 
 		if os.environ.get("SANDBOX_ON") == "1":
 			# avoid problems from nested sandbox instances
@@ -160,6 +217,8 @@ class SyncLocalTestCase(TestCase):
 				stdout = subprocess.PIPE
 
 			for cwd, cmd in rename_repo + sync_cmds + \
+				rsync_opts_repos + rsync_opts_repos_default + \
+				rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \
 				delete_sync_repo + git_repo_create + sync_type_git + \
 				rename_repo + sync_cmds:
 


^ permalink raw reply related	[flat|nested] 15+ messages in thread
end of thread, other threads:[~2015-07-21 17:18 UTC | newest]

Thread overview: 15+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2015-06-17 18:40 [gentoo-portage-dev] [PATCH] conf: Enable to set rsync extra opts per repository Étienne Buira
2015-06-17 20:32 ` Brian Dolbec
2015-06-18 21:32   ` Étienne Buira
2015-07-06 18:28     ` Brian Dolbec
2015-07-06 22:05       ` Brian Dolbec
2015-07-08 17:45       ` Étienne Buira
2015-07-08 17:46         ` [gentoo-portage-dev] [PATCH v3 1/2] sync: allow sync modules to have specific options Étienne Buira
2015-07-08 17:46           ` [gentoo-portage-dev] [PATCH v3 2/2] sync: Enable to set rsync extra opts per repository Étienne Buira
2015-07-14 19:52             ` [gentoo-portage-dev] [PATCH] RsyncSync: don't pass None sync-rsync-extra-opts value into shlex_split Zac Medico
2015-07-14 19:56               ` [gentoo-portage-dev] " Zac Medico
2015-07-14 20:04                 ` Brian Dolbec
2015-07-14 21:35                   ` Brian Dolbec
2015-07-21 17:18                     ` Étienne Buira
2015-06-18 21:36   ` [gentoo-portage-dev] [PATCH v2 1/2] sync: allow sync modules to have specific options Étienne Buira
2015-06-18 21:36     ` [gentoo-portage-dev] [PATCH v2 2/2] sync: Enable to set rsync extra opts per repository Étienne Buira

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox