From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 8F8A2138D0C for ; Wed, 8 Jul 2015 17:47:30 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id EA98514018; Wed, 8 Jul 2015 17:47:29 +0000 (UTC) Received: from mail-wi0-f177.google.com (mail-wi0-f177.google.com [209.85.212.177]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 5B23214016 for ; Wed, 8 Jul 2015 17:47:29 +0000 (UTC) Received: by wifm2 with SMTP id m2so96728880wif.1 for ; Wed, 08 Jul 2015 10:47:28 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=MfJznygOB1/YIJjpvKO87Zh7hz1CXaE/AI4o9fxkhOE=; b=S9TQkYOWQKSK6atu9GeHiK+sGdMhshpkTRb/eCy8wAsg2YRS9TwDnE48SDJw6HZBE3 mDQ7IhtAikU+yP60SD2NiP4KTkwzTW1NUw5D1UwJ8U4jLwVtS5RE4BRIvLasGS3e7QPY ze6mKkUvethZxHIjK37KGYcQbcOOq507XH3Bqafh0FbY+msdIt/a6xjpAb9l7rqJJSal wsaZvjARfz/7LXgZQ1WTYiblJNbD0DuIO714WbEJJ72OLddjlrTerYFfpeP/XkV4XztW lHo9Ged9GkhhT0mxCd20m7fmx7/N4mbTCcoL1Axoruw6NfxjNFi7DYcpFrso2PQa1gi8 sCRA== X-Received: by 10.180.37.229 with SMTP id b5mr114245582wik.16.1436377648357; Wed, 08 Jul 2015 10:47:28 -0700 (PDT) Received: from localhost.localdomain (sbr22-2-88-185-151-243.fbx.proxad.net. [88.185.151.243]) by smtp.gmail.com with ESMTPSA id ei8sm4557777wjd.32.2015.07.08.10.47.27 (version=TLSv1.2 cipher=ECDHE-RSA-AES128-SHA bits=128/128); Wed, 08 Jul 2015 10:47:27 -0700 (PDT) From: =?UTF-8?q?=C3=89tienne=20Buira?= To: gentoo-portage-dev@lists.gentoo.org Cc: =?UTF-8?q?=C3=89tienne=20Buira?= Subject: [gentoo-portage-dev] [PATCH v3 2/2] sync: Enable to set rsync extra opts per repository Date: Wed, 8 Jul 2015 19:46:42 +0200 Message-Id: <1436377602-10820-2-git-send-email-etienne.buira@gmail.com> X-Mailer: git-send-email 2.3.6 In-Reply-To: <1436377602-10820-1-git-send-email-etienne.buira@gmail.com> References: <20150708174531.GA29970@rcKGHUlyQfVFW> <1436377602-10820-1-git-send-email-etienne.buira@gmail.com> Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org X-Archives-Salt: 8a0117d0-2fdc-491c-857f-f35b9fae06f9 X-Archives-Hash: 9bbe0610fcb19952b9b930b6dd879a89 --- man/portage.5 | 5 ++ pym/portage/package/ebuild/config.py | 2 + pym/portage/repository/config.py | 10 ++-- pym/portage/sync/modules/rsync/__init__.py | 2 +- pym/portage/sync/modules/rsync/rsync.py | 6 ++- pym/portage/tests/sync/test_sync_local.py | 73 +++++++++++++++++++++++++++--- 6 files changed, 85 insertions(+), 13 deletions(-) diff --git a/man/portage.5 b/man/portage.5 index e77fc6e..e84142a 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1021,6 +1021,11 @@ group id will be changed. .br This key takes precedence over FEATURES=userpriv. If user or group id is provided, Portage no longer uses owner of the directory. +.TP +.B sync-rsync-extra-opts +Extra options to give to rsync on repository synchronization. It takes +precedence over a declaration in [DEFAULT] section, that takes +precedence over PORTAGE_RSYNC_EXTRA_OPTS. .RE .I Example: diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index 3a4007b..08db363 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -515,6 +515,8 @@ class config(object): v = confs.get("SYNC") if v is not None: portdir_sync = v + if 'PORTAGE_RSYNC_EXTRA_OPTS' in confs: + self['PORTAGE_RSYNC_EXTRA_OPTS'] = confs['PORTAGE_RSYNC_EXTRA_OPTS'] self["PORTDIR"] = portdir self["PORTDIR_OVERLAY"] = portdir_overlay diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index a461ffb..eb183e2 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -545,9 +545,9 @@ class RepoConfigLoader(object): return portdir @staticmethod - def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir): + def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir, default_opts): """Parse files in paths to load config""" - parser = SafeConfigParser() + parser = SafeConfigParser(defaults=default_opts) # use read_file/readfp in order to control decoding of unicode try: @@ -619,6 +619,7 @@ class RepoConfigLoader(object): treemap = {} ignored_map = {} ignored_location_map = {} + default_opts = {} if "PORTAGE_REPOSITORIES" in settings: portdir = "" @@ -631,10 +632,13 @@ class RepoConfigLoader(object): # deprecated portdir_sync portdir_sync = settings.get("SYNC", "") + default_opts['sync-rsync-extra-opts'] = \ + settings.get("PORTAGE_RSYNC_EXTRA_OPTS", None) + try: self._parse(paths, prepos, ignored_map, ignored_location_map, settings.local_config, - portdir) + portdir, default_opts) except ConfigParserError as e: writemsg( _("!!! Error while reading repo config file: %s\n") % e, diff --git a/pym/portage/sync/modules/rsync/__init__.py b/pym/portage/sync/modules/rsync/__init__.py index 13832f8..f2bad09 100644 --- a/pym/portage/sync/modules/rsync/__init__.py +++ b/pym/portage/sync/modules/rsync/__init__.py @@ -23,7 +23,7 @@ module_spec = { 'exists': 'Returns a boolean if the specified directory exists', }, 'validate_config': CheckSyncConfig, - 'module_specific_options': (), + 'module_specific_options': ('sync-rsync-extra-opts',), } } } diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index d84c36d..8041f07 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -72,8 +72,10 @@ class RsyncSync(NewBase): rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri) self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts) - self.extra_rsync_opts = portage.util.shlex_split( - self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")) + self.extra_rsync_opts = list() + if 'sync-rsync-extra-opts' in self.repo.module_specific_options: + self.extra_rsync_opts.extend(portage.util.shlex_split( + self.repo.module_specific_options['sync-rsync-extra-opts'])) # Real local timestamp file. self.servertimestampfile = os.path.join( diff --git a/pym/portage/tests/sync/test_sync_local.py b/pym/portage/tests/sync/test_sync_local.py index 65c20f8..f50caba 100644 --- a/pym/portage/tests/sync/test_sync_local.py +++ b/pym/portage/tests/sync/test_sync_local.py @@ -7,7 +7,7 @@ import textwrap import time import portage -from portage import os, shutil +from portage import os, shutil, _shell_quote from portage import _unicode_decode from portage.const import PORTAGE_PYM_PATH, TIMESTAMP_FORMAT from portage.process import find_binary @@ -36,11 +36,14 @@ class SyncLocalTestCase(TestCase): return repos_conf = textwrap.dedent(""" + [DEFAULT] + %(default_keys)s [test_repo] location = %(EPREFIX)s/var/repositories/test_repo sync-type = %(sync-type)s sync-uri = file:/%(EPREFIX)s/var/repositories/test_repo_sync auto-sync = yes + %(repo_extra_keys)s """) profile = { @@ -73,9 +76,17 @@ class SyncLocalTestCase(TestCase): committer_name = "Gentoo Dev" committer_email = "gentoo-dev@gentoo.org" - def change_sync_type(sync_type): - env["PORTAGE_REPOSITORIES"] = repos_conf % \ - {"EPREFIX": eprefix, "sync-type": sync_type} + def repos_set_conf(sync_type, dflt_keys=None, xtra_keys=None): + env["PORTAGE_REPOSITORIES"] = repos_conf % {\ + "EPREFIX": eprefix, "sync-type": sync_type, + "default_keys": "" if dflt_keys is None else dflt_keys, + "repo_extra_keys": "" if xtra_keys is None else xtra_keys} + + def alter_ebuild(): + with open(os.path.join(repo.location + "_sync", + "dev-libs", "A", "A-0.ebuild"), "a") as f: + f.write("\n") + os.unlink(os.path.join(metadata_dir, 'timestamp.chk')) sync_cmds = ( (homedir, cmds["emerge"] + ("--sync",)), @@ -90,6 +101,53 @@ class SyncLocalTestCase(TestCase): repo.location + "_sync")), ) + rsync_opts_repos = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", None, + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists( + repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location+"_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default_ovr = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back_nowhere"), + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default_cancel = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back_nowhere"), + "sync-rsync-extra-opts = ")), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertFalse(os.path.exists(repo.location + "_back"))), + (homedir, lambda: repos_set_conf("rsync")), + ) + delete_sync_repo = ( (homedir, lambda: shutil.rmtree( repo.location + "_sync")), @@ -107,7 +165,7 @@ class SyncLocalTestCase(TestCase): ) sync_type_git = ( - (homedir, lambda: change_sync_type("git")), + (homedir, lambda: repos_set_conf("git")), ) pythonpath = os.environ.get("PYTHONPATH") @@ -132,10 +190,9 @@ class SyncLocalTestCase(TestCase): "PATH" : os.environ["PATH"], "PORTAGE_GRPNAME" : os.environ["PORTAGE_GRPNAME"], "PORTAGE_USERNAME" : os.environ["PORTAGE_USERNAME"], - "PORTAGE_REPOSITORIES" : repos_conf % - {"EPREFIX": eprefix, "sync-type": "rsync"}, "PYTHONPATH" : pythonpath, } + repos_set_conf("rsync") if os.environ.get("SANDBOX_ON") == "1": # avoid problems from nested sandbox instances @@ -160,6 +217,8 @@ class SyncLocalTestCase(TestCase): stdout = subprocess.PIPE for cwd, cmd in rename_repo + sync_cmds + \ + rsync_opts_repos + rsync_opts_repos_default + \ + rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \ delete_sync_repo + git_repo_create + sync_type_git + \ rename_repo + sync_cmds: -- 2.0.5