[gentoo-commits] proj/portage:master commit in: man/, cnf/, pym/portage/sync/modules/rsync/

[gentoo-commits] proj/portage:master commit in: man/, cnf/, pym/portage/sync/modules/rsync/

[Michał Górny]

commit:     eb98d1ac1f255a004e06debfa1611a65fdc493e2
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 24 21:01:06 2018 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Thu Jan 25 08:00:07 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=eb98d1ac

rsync: Introduce support for running full-tree gemato verification

Add two new configuration options to rsync repositories:
sync-rsync-verify-metamanifest and sync-rsync-openpgp-key-path.
The first controls whether gemato verification is run for
the repository (defaults to true for ::gentoo, false otherwise),
the second makes it possible to override the key path for custom
repositories.

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 cnf/repos.conf                             |  2 ++
 man/portage.5                              |  9 +++++++++
 pym/portage/sync/modules/rsync/__init__.py |  4 +++-
 pym/portage/sync/modules/rsync/rsync.py    | 20 +++++++++++++++++++-
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/cnf/repos.conf b/cnf/repos.conf
index 062fc0d10..0d2b1f4be 100644
--- a/cnf/repos.conf
+++ b/cnf/repos.conf
@@ -6,6 +6,8 @@ location = /usr/portage
 sync-type = rsync
 sync-uri = rsync://rsync.gentoo.org/gentoo-portage
 auto-sync = yes
+sync-rsync-verify-metamanifest = yes
+sync-rsync-openpgp-key-path = /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg
 
 # for daily squashfs snapshots
 #sync-type = squashdelta

diff --git a/man/portage.5 b/man/portage.5
index e724e1f08..2d444a86f 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1071,10 +1071,19 @@ Extra options to give to rsync on repository synchronization. It takes
 precedence over a declaration in [DEFAULT] section, that takes
 precedence over PORTAGE_RSYNC_EXTRA_OPTS.
 .TP
+.B sync\-rsync\-openpgp\-key\-path
+Path to the OpenPGP key(ring) used to verify MetaManifest. Used only
+if \fBsync\-rsync\-verify\-metamanifest\fR is enabled. If unset,
+the user's keyring is used.
+.TP
 .B sync-rsync-vcs-ignore = true|false
 Ignore vcs directories that may be present in the repository. It is the
 user's responsibility to set sync-rsync-extra-opts to protect vcs
 directories if appropriate.
+.TP
+.B sync\-rsync\-verify\-metamanifest = true|false
+Require the repository to contain a signed MetaManifest and verify
+it using \fBapp\-portage/gemato\fR. Defaults to false.
 
 .RE
 

diff --git a/pym/portage/sync/modules/rsync/__init__.py b/pym/portage/sync/modules/rsync/__init__.py
index c2fdc4188..df9a1995a 100644
--- a/pym/portage/sync/modules/rsync/__init__.py
+++ b/pym/portage/sync/modules/rsync/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2014 Gentoo Foundation
+# Copyright 2014-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 doc = """Rsync plug-in module for portage.
@@ -27,7 +27,9 @@ module_spec = {
 			'validate_config': CheckSyncConfig,
 			'module_specific_options': (
 				'sync-rsync-extra-opts',
+				'sync-rsync-openpgp-key-path',
 				'sync-rsync-vcs-ignore',
+				'sync-rsync-verify-metamanifest',
 				),
 			}
 		}

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index c80641ba3..47f0e1ea3 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2015 Gentoo Foundation
+# Copyright 1999-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import sys
@@ -82,6 +82,16 @@ class RsyncSync(NewBase):
 			self.extra_rsync_opts.extend(portage.util.shlex_split(
 				self.repo.module_specific_options['sync-rsync-extra-opts']))
 
+		# Process GLEP74 verification options.
+		# Default verification to 'on' for ::gentoo, 'off' otherwise.
+		self.verify_metamanifest = (
+				self.repo.module_specific_options.get(
+					'sync-rsync-verify-metamanifest', False))
+		# Default to gentoo-keys keyring.
+		self.openpgp_key_path = (
+				self.repo.module_specific_options.get(
+					'sync-rsync-openpgp-key-path', None))
+
 		# Real local timestamp file.
 		self.servertimestampfile = os.path.join(
 			self.repo.location, "metadata", "timestamp.chk")
@@ -259,6 +269,14 @@ class RsyncSync(NewBase):
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
 		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+
+		# if synced successfully, verify now
+		if exitcode == 0 and self.verify_metamanifest:
+			command = ['gemato', 'verify', '-s', self.repo.location]
+			if self.openpgp_key_path is not None:
+				command += ['-K', self.openpgp_key_path]
+			exitcode = portage.process.spawn(command, **self.spawn_kwargs)
+
 		return (exitcode, updatecache_flg)
 
 

[gentoo-commits] proj/portage:master commit in: man/, cnf/, pym/portage/sync/modules/rsync/

[Zac Medico]

commit:     7b448e90034de00bc177b3d809aeaf9b94d55ee2
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 13 17:01:45 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 17 02:25:25 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7b448e90

rsync: default to sync-rsync-verify-jobs = 1 (bug 650696)

Some users have reported that using all processors to verify
manifests results in overloading, therefore default to using
a single processor. On modern hardware, verification of the
gentoo repository completes in less than 20 seconds, so using
multiple processors is not really necessary. Also, gemato-13.0
disables parallel verification due to the unresolved deadlock
issue reported in bug 647964, so this brings the default
portage configuration into alignment with current gemato
behavior.

Bug: https://bugs.gentoo.org/650696
Bug: https://bugs.gentoo.org/647964

 cnf/repos.conf                          | 1 +
 man/portage.5                           | 8 +++++---
 pym/portage/sync/modules/rsync/rsync.py | 7 ++++++-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/cnf/repos.conf b/cnf/repos.conf
index 5759b8b43..987be6462 100644
--- a/cnf/repos.conf
+++ b/cnf/repos.conf
@@ -6,6 +6,7 @@ location = /usr/portage
 sync-type = rsync
 sync-uri = rsync://rsync.gentoo.org/gentoo-portage
 auto-sync = yes
+sync-rsync-verify-jobs = 1
 sync-rsync-verify-metamanifest = yes
 sync-rsync-verify-max-age = 24
 sync-openpgp-key-path = /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg

diff --git a/man/portage.5 b/man/portage.5
index 82a50a8f4..5adb07d82 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1105,9 +1105,11 @@ Ignore vcs directories that may be present in the repository. It is the
 user's responsibility to set sync-rsync-extra-opts to protect vcs
 directories if appropriate.
 .TP
-.B sync\-rsync\-verify\-jobs
-Number of parallel jobs to use when verifying nested Manifests. Defaults
-to the apparent number of processors.
+.B sync\-rsync\-verify\-jobs = 1
+Number of parallel jobs to use when verifying nested Manifests. When
+set to 0, this will use the apparent number of processors if parallel
+verification is supported by the installed version of app-portage/gemato.
+Defaults to 1.
 .TP
 .B sync\-rsync\-verify\-max\-age
 Warn if repository is older than the specified number of days. Disabled

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 763f41699..de8327a55 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -107,12 +107,17 @@ class RsyncSync(NewBase):
 		if self.verify_jobs is not None:
 			try:
 				self.verify_jobs = int(self.verify_jobs)
-				if self.verify_jobs <= 0:
+				if self.verify_jobs < 0:
 					raise ValueError(self.verify_jobs)
 			except ValueError:
 				writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
 					level=logging.WARNING, noiselevel=-1)
 				self.verify_jobs = None
+			else:
+				if self.verify_jobs == 0:
+					# Use the apparent number of processors if gemato
+					# supports it.
+					self.verify_jobs = None
 		# Support overriding max age.
 		self.max_age = self.repo.module_specific_options.get(
 				'sync-rsync-verify-max-age', '')