From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 9B098138CEB for ; Wed, 17 Jun 2015 18:40:36 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id B01ECE078C; Wed, 17 Jun 2015 18:40:35 +0000 (UTC) Received: from mail-wi0-f177.google.com (mail-wi0-f177.google.com [209.85.212.177]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id F23C4E078A for ; Wed, 17 Jun 2015 18:40:34 +0000 (UTC) Received: by wibdq8 with SMTP id dq8so63328407wib.1 for ; Wed, 17 Jun 2015 11:40:33 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=date:from:to:subject:message-id:mime-version:content-type :content-disposition; bh=RmW/C5zo9JcXcPBuC2X8dOfHUwcbrdM6vm55trcHUIE=; b=l4LXf+HYjf5Z5Dxid/1Chl5ATQBBCtslQgp20AvFIAfsa9W8bg2hX9ThbCbQvHT6qB 9ht3weAv1EeNtY0jXy5XRuqIwq0jpVzoHZWsFi7YGbxb08fO8ruoc4OtXnJKk+Accxi3 z/Zv02eyheMf97f674bDswqTtKvkXdqTzs+20faBAlNwO+wwj5RV8PydxdKV3oC7fz52 JsU8hrpOJp2vqw/Y8FCB0Mis3ObG7AbPCguX2ENr9+yg00lC0locgOVDdAv94hZnhxuh Rw9GjaJZSWO8UzPNpM8J2+uHS4IBlCtwrtW7EeZ6uExSvd3QTfxeXdnyEPYnZv4vyJdJ AVIw== X-Received: by 10.194.22.105 with SMTP id c9mr8398571wjf.120.1434566433853; Wed, 17 Jun 2015 11:40:33 -0700 (PDT) Received: from rcKGHUlyQfVFW (sbr22-2-88-185-151-243.fbx.proxad.net. [88.185.151.243]) by mx.google.com with ESMTPSA id fi6sm9035319wib.6.2015.06.17.11.40.32 for (version=TLSv1.2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Wed, 17 Jun 2015 11:40:33 -0700 (PDT) Date: Wed, 17 Jun 2015 20:40:30 +0200 From: =?utf-8?Q?=C3=89tienne?= Buira To: gentoo-portage-dev@lists.gentoo.org Subject: [gentoo-portage-dev] [PATCH] conf: Enable to set rsync extra opts per repository Message-ID: <20150617184030.GC26811@rcKGHUlyQfVFW> Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline X-Archives-Salt: 0607a355-60ee-4288-8531-27f141fb7303 X-Archives-Hash: 901ae03e6613a75612950cce88712395 --- man/portage.5 | 5 +++ pym/portage/package/ebuild/config.py | 2 + pym/portage/repository/config.py | 24 +++++++--- pym/portage/sync/modules/rsync/rsync.py | 6 ++- pym/portage/tests/sync/test_sync_local.py | 73 ++++++++++++++++++++++++++++--- 5 files changed, 95 insertions(+), 15 deletions(-) diff --git a/man/portage.5 b/man/portage.5 index e77fc6e..e84142a 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1021,6 +1021,11 @@ group id will be changed. .br This key takes precedence over FEATURES=userpriv. If user or group id is provided, Portage no longer uses owner of the directory. +.TP +.B sync-rsync-extra-opts +Extra options to give to rsync on repository synchronization. It takes +precedence over a declaration in [DEFAULT] section, that takes +precedence over PORTAGE_RSYNC_EXTRA_OPTS. .RE .I Example: diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index 3a4007b..08db363 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -515,6 +515,8 @@ class config(object): v = confs.get("SYNC") if v is not None: portdir_sync = v + if 'PORTAGE_RSYNC_EXTRA_OPTS' in confs: + self['PORTAGE_RSYNC_EXTRA_OPTS'] = confs['PORTAGE_RSYNC_EXTRA_OPTS'] self["PORTDIR"] = portdir self["PORTDIR_OVERLAY"] = portdir_overlay diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index b7c969d..196b87a 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -87,7 +87,7 @@ class RepoConfig(object): 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name', 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority', 'profile_formats', 'sign_commit', 'sign_manifest', 'sync_cvs_repo', - 'sync_depth', + 'sync_depth', 'sync_rsync_extra_opts', 'sync_type', 'sync_umask', 'sync_uri', 'sync_user', 'thin_manifest', 'update_changelog', 'user_location', '_eapis_banned', '_eapis_deprecated', '_masters_orig') @@ -180,6 +180,8 @@ class RepoConfig(object): self.sync_depth = repo_opts.get('sync-depth') + self.sync_rsync_extra_opts = repo_opts.get('sync-rsync-extra-opts', None) + # Not implemented. format = repo_opts.get('format') if format is not None: @@ -415,6 +417,8 @@ class RepoConfig(object): repo_msg.append(indent + "sync-umask: " + self.sync_umask) if self.sync_uri: repo_msg.append(indent + "sync-uri: " + self.sync_uri) + if self.sync_rsync_extra_opts: + repo_msg.append(indent + "sync-rsync-extra-opts: " + self.sync_rsync_extra_opts) if self.sync_user: repo_msg.append(indent + "sync-user: " + self.sync_user) if self.masters: @@ -477,6 +481,9 @@ class RepoConfigLoader(object): if prepos['DEFAULT'].masters is not None: default_repo_opts['masters'] = \ ' '.join(prepos['DEFAULT'].masters) + if prepos['DEFAULT'].sync_rsync_extra_opts is not None: + default_repo_opts['sync-rsync-extra-opts'] = \ + prepos['DEFAULT'].sync_rsync_extra_opts if overlays: # We need a copy of the original repos.conf data, since we're @@ -504,7 +511,7 @@ class RepoConfigLoader(object): # repos.conf is allowed to override. for k in ('aliases', 'auto_sync', 'eclass_overrides', 'force', 'masters', 'priority', 'sync_cvs_repo', - 'sync_depth', + 'sync_depth', 'sync_rsync_extra_opts', 'sync_type', 'sync_umask', 'sync_uri', 'sync_user', ): v = getattr(repos_conf_opts, k, None) @@ -543,9 +550,9 @@ class RepoConfigLoader(object): return portdir @staticmethod - def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir): + def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir, default_opts): """Parse files in paths to load config""" - parser = SafeConfigParser() + parser = SafeConfigParser(defaults=default_opts) # use read_file/readfp in order to control decoding of unicode try: @@ -615,6 +622,7 @@ class RepoConfigLoader(object): treemap = {} ignored_map = {} ignored_location_map = {} + default_opts = {} if "PORTAGE_REPOSITORIES" in settings: portdir = "" @@ -627,10 +635,13 @@ class RepoConfigLoader(object): # deprecated portdir_sync portdir_sync = settings.get("SYNC", "") + default_opts['sync-rsync-extra-opts'] = \ + settings.get("PORTAGE_RSYNC_EXTRA_OPTS", None) + try: self._parse(paths, prepos, ignored_map, ignored_location_map, settings.local_config, - portdir) + portdir, default_opts) except ConfigParserError as e: writemsg( _("!!! Error while reading repo config file: %s\n") % e, @@ -962,7 +973,8 @@ class RepoConfigLoader(object): def config_string(self): str_or_int_keys = ("auto_sync", "format", "location", "main_repo", "priority", "sync_cvs_repo", - "sync_type", "sync_umask", "sync_uri", 'sync_user') + "sync_type", "sync_umask", "sync_uri", 'sync_user', + 'sync_rsync_extra_opts') str_tuple_keys = ("aliases", "eclass_overrides", "force") repo_config_tuple_keys = ("masters",) keys = str_or_int_keys + str_tuple_keys + repo_config_tuple_keys diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index d84c36d..1f09f60 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -72,8 +72,10 @@ class RsyncSync(NewBase): rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri) self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts) - self.extra_rsync_opts = portage.util.shlex_split( - self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")) + self.extra_rsync_opts = list() + if self.repo.sync_rsync_extra_opts is not None: + self.extra_rsync_opts.extend(portage.util.shlex_split( + self.repo.sync_rsync_extra_opts)) # Real local timestamp file. self.servertimestampfile = os.path.join( diff --git a/pym/portage/tests/sync/test_sync_local.py b/pym/portage/tests/sync/test_sync_local.py index 65c20f8..f50caba 100644 --- a/pym/portage/tests/sync/test_sync_local.py +++ b/pym/portage/tests/sync/test_sync_local.py @@ -7,7 +7,7 @@ import textwrap import time import portage -from portage import os, shutil +from portage import os, shutil, _shell_quote from portage import _unicode_decode from portage.const import PORTAGE_PYM_PATH, TIMESTAMP_FORMAT from portage.process import find_binary @@ -36,11 +36,14 @@ class SyncLocalTestCase(TestCase): return repos_conf = textwrap.dedent(""" + [DEFAULT] + %(default_keys)s [test_repo] location = %(EPREFIX)s/var/repositories/test_repo sync-type = %(sync-type)s sync-uri = file:/%(EPREFIX)s/var/repositories/test_repo_sync auto-sync = yes + %(repo_extra_keys)s """) profile = { @@ -73,9 +76,17 @@ class SyncLocalTestCase(TestCase): committer_name = "Gentoo Dev" committer_email = "gentoo-dev@gentoo.org" - def change_sync_type(sync_type): - env["PORTAGE_REPOSITORIES"] = repos_conf % \ - {"EPREFIX": eprefix, "sync-type": sync_type} + def repos_set_conf(sync_type, dflt_keys=None, xtra_keys=None): + env["PORTAGE_REPOSITORIES"] = repos_conf % {\ + "EPREFIX": eprefix, "sync-type": sync_type, + "default_keys": "" if dflt_keys is None else dflt_keys, + "repo_extra_keys": "" if xtra_keys is None else xtra_keys} + + def alter_ebuild(): + with open(os.path.join(repo.location + "_sync", + "dev-libs", "A", "A-0.ebuild"), "a") as f: + f.write("\n") + os.unlink(os.path.join(metadata_dir, 'timestamp.chk')) sync_cmds = ( (homedir, cmds["emerge"] + ("--sync",)), @@ -90,6 +101,53 @@ class SyncLocalTestCase(TestCase): repo.location + "_sync")), ) + rsync_opts_repos = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", None, + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists( + repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location+"_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default_ovr = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back_nowhere"), + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default_cancel = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back_nowhere"), + "sync-rsync-extra-opts = ")), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertFalse(os.path.exists(repo.location + "_back"))), + (homedir, lambda: repos_set_conf("rsync")), + ) + delete_sync_repo = ( (homedir, lambda: shutil.rmtree( repo.location + "_sync")), @@ -107,7 +165,7 @@ class SyncLocalTestCase(TestCase): ) sync_type_git = ( - (homedir, lambda: change_sync_type("git")), + (homedir, lambda: repos_set_conf("git")), ) pythonpath = os.environ.get("PYTHONPATH") @@ -132,10 +190,9 @@ class SyncLocalTestCase(TestCase): "PATH" : os.environ["PATH"], "PORTAGE_GRPNAME" : os.environ["PORTAGE_GRPNAME"], "PORTAGE_USERNAME" : os.environ["PORTAGE_USERNAME"], - "PORTAGE_REPOSITORIES" : repos_conf % - {"EPREFIX": eprefix, "sync-type": "rsync"}, "PYTHONPATH" : pythonpath, } + repos_set_conf("rsync") if os.environ.get("SANDBOX_ON") == "1": # avoid problems from nested sandbox instances @@ -160,6 +217,8 @@ class SyncLocalTestCase(TestCase): stdout = subprocess.PIPE for cwd, cmd in rename_repo + sync_cmds + \ + rsync_opts_repos + rsync_opts_repos_default + \ + rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \ delete_sync_repo + git_repo_create + sync_type_git + \ rename_repo + sync_cmds: