From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id CDD0F138CED for ; Thu, 18 Jun 2015 21:37:28 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 67B621402D; Thu, 18 Jun 2015 21:37:28 +0000 (UTC) Received: from mail-wi0-f169.google.com (mail-wi0-f169.google.com [209.85.212.169]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id ACFA214021 for ; Thu, 18 Jun 2015 21:37:27 +0000 (UTC) Received: by wilj4 with SMTP id j4so1994743wil.0 for ; Thu, 18 Jun 2015 14:37:26 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=from:to:cc:subject:date:message-id:in-reply-to:references :in-reply-to:references; bh=Afundtjz5Ey09Slxcmdf+6jfZFQnvGoXiZPiyjAIU10=; b=AYEj2oXlkE8lBBG6We5kSQflXTGa83+d2pSJMh/C7yGFJTdZDPyR/Iin0bafgY7bp5 owsfCOq6zTOWrVSHBFbttsU1DSou/PTH1X7javcxIW3CRqnj1mjzWBySq1vxqMclwx+d nYuza5QIN4rsldsL1EcfGPTZL3o6zzRHkUQNXsm7FQeFrOia53QVZ5spiWkPEtEFKACh TA4UrvL35H5TlZfeTZXKuZ5yd6NVaaSocOFMv0/8Mq568fDNgp/g2kZl2vmaMBnug1SR 1xRL2TQK+c12Vi5q5mc3CgyUvZod//wFn5thZ6XYttPqaF7fHsexrpYxPfEmUhxqVLG1 p5jA== X-Received: by 10.180.7.169 with SMTP id k9mr245915wia.84.1434663446689; Thu, 18 Jun 2015 14:37:26 -0700 (PDT) Received: from localhost.localdomain (sbr22-2-88-185-151-243.fbx.proxad.net. [88.185.151.243]) by mx.google.com with ESMTPSA id s10sm14017473wjy.35.2015.06.18.14.37.25 (version=TLSv1.2 cipher=ECDHE-RSA-AES128-SHA bits=128/128); Thu, 18 Jun 2015 14:37:26 -0700 (PDT) From: =?UTF-8?q?=C3=89tienne=20Buira?= To: gentoo-portage-dev@lists.gentoo.org Cc: =?UTF-8?q?=C3=89tienne=20Buira?= Subject: [gentoo-portage-dev] [PATCH v2 2/2] sync: Enable to set rsync extra opts per repository Date: Thu, 18 Jun 2015 23:36:24 +0200 Message-Id: X-Mailer: git-send-email 2.3.6 In-Reply-To: References: <20150617133217.303b3b68.dolsen@gentoo.org> In-Reply-To: References: Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org X-Archives-Salt: e0a51bcb-e5d7-4018-842f-ec48a7f0022a X-Archives-Hash: e5120231ee64d32b55d361ae66a84dc3 --- man/portage.5 | 5 +++ pym/portage/package/ebuild/config.py | 2 + pym/portage/repository/config.py | 10 +++-- pym/portage/sync/modules/rsync/rsync.py | 11 ++++- pym/portage/tests/sync/test_sync_local.py | 73 ++++++++++++++++++++++++++++--- 5 files changed, 89 insertions(+), 12 deletions(-) diff --git a/man/portage.5 b/man/portage.5 index e77fc6e..e84142a 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1021,6 +1021,11 @@ group id will be changed. .br This key takes precedence over FEATURES=userpriv. If user or group id is provided, Portage no longer uses owner of the directory. +.TP +.B sync-rsync-extra-opts +Extra options to give to rsync on repository synchronization. It takes +precedence over a declaration in [DEFAULT] section, that takes +precedence over PORTAGE_RSYNC_EXTRA_OPTS. .RE .I Example: diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index 3a4007b..08db363 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -515,6 +515,8 @@ class config(object): v = confs.get("SYNC") if v is not None: portdir_sync = v + if 'PORTAGE_RSYNC_EXTRA_OPTS' in confs: + self['PORTAGE_RSYNC_EXTRA_OPTS'] = confs['PORTAGE_RSYNC_EXTRA_OPTS'] self["PORTDIR"] = portdir self["PORTDIR_OVERLAY"] = portdir_overlay diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index 60eb1f2..8c16cb3 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -550,9 +550,9 @@ class RepoConfigLoader(object): return portdir @staticmethod - def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir): + def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir, default_opts): """Parse files in paths to load config""" - parser = SafeConfigParser() + parser = SafeConfigParser(defaults=default_opts) # use read_file/readfp in order to control decoding of unicode try: @@ -622,6 +622,7 @@ class RepoConfigLoader(object): treemap = {} ignored_map = {} ignored_location_map = {} + default_opts = {} if "PORTAGE_REPOSITORIES" in settings: portdir = "" @@ -634,10 +635,13 @@ class RepoConfigLoader(object): # deprecated portdir_sync portdir_sync = settings.get("SYNC", "") + default_opts['sync-rsync-extra-opts'] = \ + settings.get("PORTAGE_RSYNC_EXTRA_OPTS", None) + try: self._parse(paths, prepos, ignored_map, ignored_location_map, settings.local_config, - portdir) + portdir, default_opts) except ConfigParserError as e: writemsg( _("!!! Error while reading repo config file: %s\n") % e, diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py index d84c36d..2237901 100644 --- a/pym/portage/sync/modules/rsync/rsync.py +++ b/pym/portage/sync/modules/rsync/rsync.py @@ -44,6 +44,11 @@ class RsyncSync(NewBase): return "RsyncSync" + @staticmethod + def specific_options(): + return ("extra-opts",) + + def __init__(self): NewBase.__init__(self, "rsync", RSYNC_PACKAGE_ATOM) @@ -72,8 +77,10 @@ class RsyncSync(NewBase): rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri) self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts) - self.extra_rsync_opts = portage.util.shlex_split( - self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")) + self.extra_rsync_opts = list() + if self.repo.sync_rsync_extra_opts is not None: + self.extra_rsync_opts.extend(portage.util.shlex_split( + self.repo.sync_rsync_extra_opts)) # Real local timestamp file. self.servertimestampfile = os.path.join( diff --git a/pym/portage/tests/sync/test_sync_local.py b/pym/portage/tests/sync/test_sync_local.py index 65c20f8..f50caba 100644 --- a/pym/portage/tests/sync/test_sync_local.py +++ b/pym/portage/tests/sync/test_sync_local.py @@ -7,7 +7,7 @@ import textwrap import time import portage -from portage import os, shutil +from portage import os, shutil, _shell_quote from portage import _unicode_decode from portage.const import PORTAGE_PYM_PATH, TIMESTAMP_FORMAT from portage.process import find_binary @@ -36,11 +36,14 @@ class SyncLocalTestCase(TestCase): return repos_conf = textwrap.dedent(""" + [DEFAULT] + %(default_keys)s [test_repo] location = %(EPREFIX)s/var/repositories/test_repo sync-type = %(sync-type)s sync-uri = file:/%(EPREFIX)s/var/repositories/test_repo_sync auto-sync = yes + %(repo_extra_keys)s """) profile = { @@ -73,9 +76,17 @@ class SyncLocalTestCase(TestCase): committer_name = "Gentoo Dev" committer_email = "gentoo-dev@gentoo.org" - def change_sync_type(sync_type): - env["PORTAGE_REPOSITORIES"] = repos_conf % \ - {"EPREFIX": eprefix, "sync-type": sync_type} + def repos_set_conf(sync_type, dflt_keys=None, xtra_keys=None): + env["PORTAGE_REPOSITORIES"] = repos_conf % {\ + "EPREFIX": eprefix, "sync-type": sync_type, + "default_keys": "" if dflt_keys is None else dflt_keys, + "repo_extra_keys": "" if xtra_keys is None else xtra_keys} + + def alter_ebuild(): + with open(os.path.join(repo.location + "_sync", + "dev-libs", "A", "A-0.ebuild"), "a") as f: + f.write("\n") + os.unlink(os.path.join(metadata_dir, 'timestamp.chk')) sync_cmds = ( (homedir, cmds["emerge"] + ("--sync",)), @@ -90,6 +101,53 @@ class SyncLocalTestCase(TestCase): repo.location + "_sync")), ) + rsync_opts_repos = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", None, + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists( + repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location+"_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default_ovr = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back_nowhere"), + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back"))), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))), + (homedir, lambda: shutil.rmtree(repo.location + "_back")), + (homedir, lambda: repos_set_conf("rsync")), + ) + + rsync_opts_repos_default_cancel = ( + (homedir, alter_ebuild), + (homedir, lambda: repos_set_conf("rsync", + "sync-rsync-extra-opts = --backup --backup-dir=%s" % + _shell_quote(repo.location + "_back_nowhere"), + "sync-rsync-extra-opts = ")), + (homedir, cmds['emerge'] + ("--sync",)), + (homedir, lambda: self.assertFalse(os.path.exists(repo.location + "_back"))), + (homedir, lambda: repos_set_conf("rsync")), + ) + delete_sync_repo = ( (homedir, lambda: shutil.rmtree( repo.location + "_sync")), @@ -107,7 +165,7 @@ class SyncLocalTestCase(TestCase): ) sync_type_git = ( - (homedir, lambda: change_sync_type("git")), + (homedir, lambda: repos_set_conf("git")), ) pythonpath = os.environ.get("PYTHONPATH") @@ -132,10 +190,9 @@ class SyncLocalTestCase(TestCase): "PATH" : os.environ["PATH"], "PORTAGE_GRPNAME" : os.environ["PORTAGE_GRPNAME"], "PORTAGE_USERNAME" : os.environ["PORTAGE_USERNAME"], - "PORTAGE_REPOSITORIES" : repos_conf % - {"EPREFIX": eprefix, "sync-type": "rsync"}, "PYTHONPATH" : pythonpath, } + repos_set_conf("rsync") if os.environ.get("SANDBOX_ON") == "1": # avoid problems from nested sandbox instances @@ -160,6 +217,8 @@ class SyncLocalTestCase(TestCase): stdout = subprocess.PIPE for cwd, cmd in rename_repo + sync_cmds + \ + rsync_opts_repos + rsync_opts_repos_default + \ + rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \ delete_sync_repo + git_repo_create + sync_type_git + \ rename_repo + sync_cmds: -- 2.0.5