* [gentoo-portage-dev] [PATCH v2 2/9] rsync: Use gemato routines directly instead of calling the CLI tool
2018-02-02 20:42 [gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs Michał Górny
@ 2018-02-02 20:42 ` Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 3/9] rsync: Verify the Manifest signature even if tree is unchanged Michał Górny
` (6 subsequent siblings)
7 siblings, 0 replies; 12+ messages in thread
From: Michał Górny @ 2018-02-02 20:42 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Michał Górny
---
pym/portage/sync/modules/rsync/rsync.py | 66 ++++++++++++++++++++++++++++-----
1 file changed, 57 insertions(+), 9 deletions(-)
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index ec28af366..39c4066d8 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -6,6 +6,7 @@ import logging
import time
import signal
import socket
+import io
import re
import random
import tempfile
@@ -25,6 +26,13 @@ from portage.sync.getaddrinfo_validate import getaddrinfo_validate
from _emerge.UserQuery import UserQuery
from portage.sync.syncbase import NewBase
+try:
+ from gemato.exceptions import GematoException
+ import gemato.openpgp
+ import gemato.recursiveloader
+except ImportError:
+ gemato = None
+
if sys.hexversion >= 0x3000000:
# pylint: disable=W0622
_unicode = str
@@ -285,17 +293,57 @@ class RsyncSync(NewBase):
# if synced successfully, verify now
if exitcode == 0 and not local_state_unchanged and self.verify_metamanifest:
- command = ['gemato', 'verify', '-s', self.repo.location]
- if self.repo.sync_openpgp_key_path is not None:
- command += ['-K', self.repo.sync_openpgp_key_path]
- if self.verify_jobs is not None:
- command += ['-j', str(self.verify_jobs)]
- try:
- exitcode = portage.process.spawn(command, **self.spawn_kwargs)
- except CommandNotFound as e:
- writemsg_level("!!! Command not found: %s\n" % (command[0],),
+ if gemato is None:
+ writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
level=logging.ERROR, noiselevel=-1)
exitcode = 127
+ else:
+ # Use isolated environment if key is specified,
+ # system environment otherwise
+ if self.repo.sync_openpgp_key_path is not None:
+ openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+ else:
+ openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+
+ try:
+ with openpgp_env_cls() as openpgp_env:
+ if self.repo.sync_openpgp_key_path is not None:
+ out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+ with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+ openpgp_env.import_key(f)
+ out.ebegin('Refreshing keys from keyserver')
+ openpgp_env.refresh_keys()
+ out.eend(0)
+
+ m = gemato.recursiveloader.ManifestRecursiveLoader(
+ os.path.join(self.repo.location, 'Manifest'),
+ verify_openpgp=True,
+ openpgp_env=openpgp_env,
+ max_jobs=self.verify_jobs)
+ if not m.openpgp_signed:
+ raise RuntimeError('OpenPGP signature not found on Manifest')
+
+ ts = m.find_timestamp()
+ if ts is None:
+ raise RuntimeError('Timestamp not found in Manifest')
+
+ out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+ out.einfo('Valid OpenPGP signature found:')
+ out.einfo('- primary key: %s' % (
+ m.openpgp_signature.primary_key_fingerprint))
+ out.einfo('- subkey: %s' % (
+ m.openpgp_signature.fingerprint))
+ out.einfo('- timestamp: %s UTC' % (
+ m.openpgp_signature.timestamp))
+
+ out.ebegin('Verifying %s' % (self.repo.location,))
+ m.assert_directory_verifies()
+ out.eend(0)
+ except GematoException as e:
+ writemsg_level("!!! Manifest verification failed:\n%s\n"
+ % (e,),
+ level=logging.ERROR, noiselevel=-1)
+ exitcode = 1
return (exitcode, updatecache_flg)
--
2.16.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [gentoo-portage-dev] [PATCH v2 3/9] rsync: Verify the Manifest signature even if tree is unchanged
2018-02-02 20:42 [gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 2/9] rsync: Use gemato routines directly instead of calling the CLI tool Michał Górny
@ 2018-02-02 20:42 ` Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 4/9] rsync: Pre-indent the try-finally block for gemato key scope Michał Górny
` (5 subsequent siblings)
7 siblings, 0 replies; 12+ messages in thread
From: Michał Górny @ 2018-02-02 20:42 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Michał Górny
Always verify the Manifest signature if verification is enabled.
Skipping the deep tree verification for unchanged case is reasonable
but we need to make sure the Manifest signature stays valid to catch
the case of the signing key being revoked.
---
pym/portage/sync/modules/rsync/rsync.py | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 39c4066d8..e6e218868 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -292,7 +292,7 @@ class RsyncSync(NewBase):
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
# if synced successfully, verify now
- if exitcode == 0 and not local_state_unchanged and self.verify_metamanifest:
+ if exitcode == 0 and self.verify_metamanifest:
if gemato is None:
writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
level=logging.ERROR, noiselevel=-1)
@@ -315,6 +315,8 @@ class RsyncSync(NewBase):
openpgp_env.refresh_keys()
out.eend(0)
+ # we always verify the Manifest signature, in case
+ # we had to deal with key revocation case
m = gemato.recursiveloader.ManifestRecursiveLoader(
os.path.join(self.repo.location, 'Manifest'),
verify_openpgp=True,
@@ -336,9 +338,12 @@ class RsyncSync(NewBase):
out.einfo('- timestamp: %s UTC' % (
m.openpgp_signature.timestamp))
- out.ebegin('Verifying %s' % (self.repo.location,))
- m.assert_directory_verifies()
- out.eend(0)
+ # if nothing has changed, skip the actual Manifest
+ # verification
+ if not local_state_unchanged:
+ out.ebegin('Verifying %s' % (self.repo.location,))
+ m.assert_directory_verifies()
+ out.eend(0)
except GematoException as e:
writemsg_level("!!! Manifest verification failed:\n%s\n"
% (e,),
--
2.16.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [gentoo-portage-dev] [PATCH v2 4/9] rsync: Pre-indent the try-finally block for gemato key scope
2018-02-02 20:42 [gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 2/9] rsync: Use gemato routines directly instead of calling the CLI tool Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 3/9] rsync: Verify the Manifest signature even if tree is unchanged Michał Górny
@ 2018-02-02 20:42 ` Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 5/9] rsync: Load and update keys early Michał Górny
` (4 subsequent siblings)
7 siblings, 0 replies; 12+ messages in thread
From: Michał Górny @ 2018-02-02 20:42 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Michał Górny
---
pym/portage/sync/modules/rsync/rsync.py | 467 ++++++++++++++++----------------
1 file changed, 235 insertions(+), 232 deletions(-)
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index e6e218868..5c0b53f9e 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -110,247 +110,250 @@ class RsyncSync(NewBase):
level=logging.WARNING, noiselevel=-1)
self.verify_jobs = None
- # Real local timestamp file.
- self.servertimestampfile = os.path.join(
- self.repo.location, "metadata", "timestamp.chk")
-
- content = portage.util.grabfile(self.servertimestampfile)
- timestamp = 0
- if content:
- try:
- timestamp = time.mktime(time.strptime(content[0],
- TIMESTAMP_FORMAT))
- except (OverflowError, ValueError):
- pass
- del content
-
- try:
- self.rsync_initial_timeout = \
- int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
- except ValueError:
- self.rsync_initial_timeout = 15
-
- try:
- maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
- except SystemExit as e:
- raise # Needed else can't exit
- except:
- maxretries = -1 #default number of retries
-
- if syncuri.startswith("file://"):
- self.proto = "file"
- dosyncuri = syncuri[7:]
- unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
- dosyncuri, timestamp, opts)
- self._process_exitcode(exitcode, dosyncuri, out, 1)
- return (exitcode, updatecache_flg)
-
- retries=0
try:
- self.proto, user_name, hostname, port = re.split(
- r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
- syncuri, maxsplit=4)[1:5]
- except ValueError:
- writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
- noiselevel=-1, level=logging.ERROR)
- return (1, False)
+ # Real local timestamp file.
+ self.servertimestampfile = os.path.join(
+ self.repo.location, "metadata", "timestamp.chk")
- self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+ content = portage.util.grabfile(self.servertimestampfile)
+ timestamp = 0
+ if content:
+ try:
+ timestamp = time.mktime(time.strptime(content[0],
+ TIMESTAMP_FORMAT))
+ except (OverflowError, ValueError):
+ pass
+ del content
- if port is None:
- port=""
- if user_name is None:
- user_name=""
- if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
- getaddrinfo_host = hostname
- else:
- # getaddrinfo needs the brackets stripped
- getaddrinfo_host = hostname[1:-1]
- updatecache_flg = False
- all_rsync_opts = set(self.rsync_opts)
- all_rsync_opts.update(self.extra_rsync_opts)
+ try:
+ self.rsync_initial_timeout = \
+ int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
+ except ValueError:
+ self.rsync_initial_timeout = 15
- family = socket.AF_UNSPEC
- if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
- family = socket.AF_INET
- elif socket.has_ipv6 and \
- ("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
- family = socket.AF_INET6
+ try:
+ maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
+ except SystemExit as e:
+ raise # Needed else can't exit
+ except:
+ maxretries = -1 #default number of retries
+
+ if syncuri.startswith("file://"):
+ self.proto = "file"
+ dosyncuri = syncuri[7:]
+ unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
+ dosyncuri, timestamp, opts)
+ self._process_exitcode(exitcode, dosyncuri, out, 1)
+ return (exitcode, updatecache_flg)
+
+ retries=0
+ try:
+ self.proto, user_name, hostname, port = re.split(
+ r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
+ syncuri, maxsplit=4)[1:5]
+ except ValueError:
+ writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
+ noiselevel=-1, level=logging.ERROR)
+ return (1, False)
- addrinfos = None
- uris = []
+ self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
- try:
- addrinfos = getaddrinfo_validate(
- socket.getaddrinfo(getaddrinfo_host, None,
- family, socket.SOCK_STREAM))
- except socket.error as e:
- writemsg_level(
- "!!! getaddrinfo failed for '%s': %s\n"
- % (_unicode_decode(hostname), _unicode(e)),
- noiselevel=-1, level=logging.ERROR)
-
- if addrinfos:
-
- AF_INET = socket.AF_INET
- AF_INET6 = None
- if socket.has_ipv6:
- AF_INET6 = socket.AF_INET6
-
- ips_v4 = []
- ips_v6 = []
-
- for addrinfo in addrinfos:
- if addrinfo[0] == AF_INET:
- ips_v4.append("%s" % addrinfo[4][0])
- elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
- # IPv6 addresses need to be enclosed in square brackets
- ips_v6.append("[%s]" % addrinfo[4][0])
-
- random.shuffle(ips_v4)
- random.shuffle(ips_v6)
-
- # Give priority to the address family that
- # getaddrinfo() returned first.
- if AF_INET6 is not None and addrinfos and \
- addrinfos[0][0] == AF_INET6:
- ips = ips_v6 + ips_v4
- else:
- ips = ips_v4 + ips_v6
-
- for ip in ips:
- uris.append(syncuri.replace(
- "//" + user_name + hostname + port + "/",
- "//" + user_name + ip + port + "/", 1))
-
- if not uris:
- # With some configurations we need to use the plain hostname
- # rather than try to resolve the ip addresses (bug #340817).
- uris.append(syncuri)
-
- # reverse, for use with pop()
- uris.reverse()
- uris_orig = uris[:]
-
- effective_maxretries = maxretries
- if effective_maxretries < 0:
- effective_maxretries = len(uris) - 1
-
- local_state_unchanged = True
- while (1):
- if uris:
- dosyncuri = uris.pop()
- elif maxretries < 0 or retries > maxretries:
- writemsg("!!! Exhausted addresses for %s\n"
- % _unicode_decode(hostname), noiselevel=-1)
- return (1, False)
- else:
- uris.extend(uris_orig)
- dosyncuri = uris.pop()
-
- if (retries==0):
- if "--ask" in opts:
- uq = UserQuery(opts)
- if uq.query("Do you want to sync your Portage tree " + \
- "with the mirror at\n" + blue(dosyncuri) + bold("?"),
- enter_invalid) == "No":
- print()
- print("Quitting.")
- print()
- sys.exit(128 + signal.SIGINT)
- self.logger(self.xterm_titles,
- ">>> Starting rsync with " + dosyncuri)
- if "--quiet" not in opts:
- print(">>> Starting rsync with "+dosyncuri+"...")
- else:
- self.logger(self.xterm_titles,
- ">>> Starting retry %d of %d with %s" % \
- (retries, effective_maxretries, dosyncuri))
- writemsg_stdout(
- "\n\n>>> Starting retry %d of %d with %s\n" % \
- (retries, effective_maxretries, dosyncuri), noiselevel=-1)
-
- if dosyncuri.startswith('ssh://'):
- dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
-
- unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
- dosyncuri, timestamp, opts)
- if not unchanged:
- local_state_unchanged = False
- if is_synced:
- break
-
- retries=retries+1
-
- if maxretries < 0 or retries <= maxretries:
- print(">>> Retrying...")
- else:
- # over retries
- # exit loop
- exitcode = EXCEEDED_MAX_RETRIES
- break
- self._process_exitcode(exitcode, dosyncuri, out, maxretries)
-
- # if synced successfully, verify now
- if exitcode == 0 and self.verify_metamanifest:
- if gemato is None:
- writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
- level=logging.ERROR, noiselevel=-1)
- exitcode = 127
+ if port is None:
+ port=""
+ if user_name is None:
+ user_name=""
+ if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
+ getaddrinfo_host = hostname
else:
- # Use isolated environment if key is specified,
- # system environment otherwise
- if self.repo.sync_openpgp_key_path is not None:
- openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+ # getaddrinfo needs the brackets stripped
+ getaddrinfo_host = hostname[1:-1]
+ updatecache_flg = False
+ all_rsync_opts = set(self.rsync_opts)
+ all_rsync_opts.update(self.extra_rsync_opts)
+
+ family = socket.AF_UNSPEC
+ if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
+ family = socket.AF_INET
+ elif socket.has_ipv6 and \
+ ("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
+ family = socket.AF_INET6
+
+ addrinfos = None
+ uris = []
+
+ try:
+ addrinfos = getaddrinfo_validate(
+ socket.getaddrinfo(getaddrinfo_host, None,
+ family, socket.SOCK_STREAM))
+ except socket.error as e:
+ writemsg_level(
+ "!!! getaddrinfo failed for '%s': %s\n"
+ % (_unicode_decode(hostname), _unicode(e)),
+ noiselevel=-1, level=logging.ERROR)
+
+ if addrinfos:
+
+ AF_INET = socket.AF_INET
+ AF_INET6 = None
+ if socket.has_ipv6:
+ AF_INET6 = socket.AF_INET6
+
+ ips_v4 = []
+ ips_v6 = []
+
+ for addrinfo in addrinfos:
+ if addrinfo[0] == AF_INET:
+ ips_v4.append("%s" % addrinfo[4][0])
+ elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
+ # IPv6 addresses need to be enclosed in square brackets
+ ips_v6.append("[%s]" % addrinfo[4][0])
+
+ random.shuffle(ips_v4)
+ random.shuffle(ips_v6)
+
+ # Give priority to the address family that
+ # getaddrinfo() returned first.
+ if AF_INET6 is not None and addrinfos and \
+ addrinfos[0][0] == AF_INET6:
+ ips = ips_v6 + ips_v4
+ else:
+ ips = ips_v4 + ips_v6
+
+ for ip in ips:
+ uris.append(syncuri.replace(
+ "//" + user_name + hostname + port + "/",
+ "//" + user_name + ip + port + "/", 1))
+
+ if not uris:
+ # With some configurations we need to use the plain hostname
+ # rather than try to resolve the ip addresses (bug #340817).
+ uris.append(syncuri)
+
+ # reverse, for use with pop()
+ uris.reverse()
+ uris_orig = uris[:]
+
+ effective_maxretries = maxretries
+ if effective_maxretries < 0:
+ effective_maxretries = len(uris) - 1
+
+ local_state_unchanged = True
+ while (1):
+ if uris:
+ dosyncuri = uris.pop()
+ elif maxretries < 0 or retries > maxretries:
+ writemsg("!!! Exhausted addresses for %s\n"
+ % _unicode_decode(hostname), noiselevel=-1)
+ return (1, False)
+ else:
+ uris.extend(uris_orig)
+ dosyncuri = uris.pop()
+
+ if (retries==0):
+ if "--ask" in opts:
+ uq = UserQuery(opts)
+ if uq.query("Do you want to sync your Portage tree " + \
+ "with the mirror at\n" + blue(dosyncuri) + bold("?"),
+ enter_invalid) == "No":
+ print()
+ print("Quitting.")
+ print()
+ sys.exit(128 + signal.SIGINT)
+ self.logger(self.xterm_titles,
+ ">>> Starting rsync with " + dosyncuri)
+ if "--quiet" not in opts:
+ print(">>> Starting rsync with "+dosyncuri+"...")
+ else:
+ self.logger(self.xterm_titles,
+ ">>> Starting retry %d of %d with %s" % \
+ (retries, effective_maxretries, dosyncuri))
+ writemsg_stdout(
+ "\n\n>>> Starting retry %d of %d with %s\n" % \
+ (retries, effective_maxretries, dosyncuri), noiselevel=-1)
+
+ if dosyncuri.startswith('ssh://'):
+ dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
+
+ unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
+ dosyncuri, timestamp, opts)
+ if not unchanged:
+ local_state_unchanged = False
+ if is_synced:
+ break
+
+ retries=retries+1
+
+ if maxretries < 0 or retries <= maxretries:
+ print(">>> Retrying...")
else:
- openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+ # over retries
+ # exit loop
+ exitcode = EXCEEDED_MAX_RETRIES
+ break
+ self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+
+ # if synced successfully, verify now
+ if exitcode == 0 and self.verify_metamanifest:
+ if gemato is None:
+ writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
+ level=logging.ERROR, noiselevel=-1)
+ exitcode = 127
+ else:
+ # Use isolated environment if key is specified,
+ # system environment otherwise
+ if self.repo.sync_openpgp_key_path is not None:
+ openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
+ else:
+ openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
+
+ try:
+ with openpgp_env_cls() as openpgp_env:
+ if self.repo.sync_openpgp_key_path is not None:
+ out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+ with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+ openpgp_env.import_key(f)
+ out.ebegin('Refreshing keys from keyserver')
+ openpgp_env.refresh_keys()
+ out.eend(0)
+
+ # we always verify the Manifest signature, in case
+ # we had to deal with key revocation case
+ m = gemato.recursiveloader.ManifestRecursiveLoader(
+ os.path.join(self.repo.location, 'Manifest'),
+ verify_openpgp=True,
+ openpgp_env=openpgp_env,
+ max_jobs=self.verify_jobs)
+ if not m.openpgp_signed:
+ raise RuntimeError('OpenPGP signature not found on Manifest')
+
+ ts = m.find_timestamp()
+ if ts is None:
+ raise RuntimeError('Timestamp not found in Manifest')
+
+ out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+ out.einfo('Valid OpenPGP signature found:')
+ out.einfo('- primary key: %s' % (
+ m.openpgp_signature.primary_key_fingerprint))
+ out.einfo('- subkey: %s' % (
+ m.openpgp_signature.fingerprint))
+ out.einfo('- timestamp: %s UTC' % (
+ m.openpgp_signature.timestamp))
+
+ # if nothing has changed, skip the actual Manifest
+ # verification
+ if not local_state_unchanged:
+ out.ebegin('Verifying %s' % (self.repo.location,))
+ m.assert_directory_verifies()
+ out.eend(0)
+ except GematoException as e:
+ writemsg_level("!!! Manifest verification failed:\n%s\n"
+ % (e,),
+ level=logging.ERROR, noiselevel=-1)
+ exitcode = 1
- try:
- with openpgp_env_cls() as openpgp_env:
- if self.repo.sync_openpgp_key_path is not None:
- out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
- with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
- openpgp_env.import_key(f)
- out.ebegin('Refreshing keys from keyserver')
- openpgp_env.refresh_keys()
- out.eend(0)
-
- # we always verify the Manifest signature, in case
- # we had to deal with key revocation case
- m = gemato.recursiveloader.ManifestRecursiveLoader(
- os.path.join(self.repo.location, 'Manifest'),
- verify_openpgp=True,
- openpgp_env=openpgp_env,
- max_jobs=self.verify_jobs)
- if not m.openpgp_signed:
- raise RuntimeError('OpenPGP signature not found on Manifest')
-
- ts = m.find_timestamp()
- if ts is None:
- raise RuntimeError('Timestamp not found in Manifest')
-
- out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
- out.einfo('Valid OpenPGP signature found:')
- out.einfo('- primary key: %s' % (
- m.openpgp_signature.primary_key_fingerprint))
- out.einfo('- subkey: %s' % (
- m.openpgp_signature.fingerprint))
- out.einfo('- timestamp: %s UTC' % (
- m.openpgp_signature.timestamp))
-
- # if nothing has changed, skip the actual Manifest
- # verification
- if not local_state_unchanged:
- out.ebegin('Verifying %s' % (self.repo.location,))
- m.assert_directory_verifies()
- out.eend(0)
- except GematoException as e:
- writemsg_level("!!! Manifest verification failed:\n%s\n"
- % (e,),
- level=logging.ERROR, noiselevel=-1)
- exitcode = 1
-
- return (exitcode, updatecache_flg)
+ return (exitcode, updatecache_flg)
+ finally:
+ pass
def _process_exitcode(self, exitcode, syncuri, out, maxretries):
--
2.16.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [gentoo-portage-dev] [PATCH v2 5/9] rsync: Load and update keys early
2018-02-02 20:42 [gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs Michał Górny
` (2 preceding siblings ...)
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 4/9] rsync: Pre-indent the try-finally block for gemato key scope Michał Górny
@ 2018-02-02 20:42 ` Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old Michał Górny
` (3 subsequent siblings)
7 siblings, 0 replies; 12+ messages in thread
From: Michał Górny @ 2018-02-02 20:42 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Michał Górny
Load and update keys early to avoid delaying failures post rsync. Any
failure will prevent verification from happening, and presumably most of
the users will prefer fixing it and trying to sync again. For that case,
it is better to perform the task before actual rsync to avoid
unnecessarily rsyncing twice.
---
pym/portage/sync/modules/rsync/rsync.py | 103 ++++++++++++++++++--------------
1 file changed, 57 insertions(+), 46 deletions(-)
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 5c0b53f9e..dc4674548 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -110,7 +110,33 @@ class RsyncSync(NewBase):
level=logging.WARNING, noiselevel=-1)
self.verify_jobs = None
+ openpgp_env = None
+ if self.verify_metamanifest and gemato is not None:
+ # Use isolated environment if key is specified,
+ # system environment otherwise
+ if self.repo.sync_openpgp_key_path is not None:
+ openpgp_env = gemato.openpgp.OpenPGPEnvironment()
+ else:
+ openpgp_env = gemato.openpgp.OpenPGPSystemEnvironment()
+
try:
+ # Load and update the keyring early. If it fails, then verification
+ # will not be performed and the user will have to fix it and try again,
+ # so we may as well bail out before actual rsync happens.
+ if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
+ try:
+ out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+ with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+ openpgp_env.import_key(f)
+ out.ebegin('Refreshing keys from keyserver')
+ openpgp_env.refresh_keys()
+ out.eend(0)
+ except GematoException as e:
+ writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"
+ % (e,),
+ level=logging.ERROR, noiselevel=-1)
+ return (1, False)
+
# Real local timestamp file.
self.servertimestampfile = os.path.join(
self.repo.location, "metadata", "timestamp.chk")
@@ -299,52 +325,36 @@ class RsyncSync(NewBase):
level=logging.ERROR, noiselevel=-1)
exitcode = 127
else:
- # Use isolated environment if key is specified,
- # system environment otherwise
- if self.repo.sync_openpgp_key_path is not None:
- openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment
- else:
- openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment
-
try:
- with openpgp_env_cls() as openpgp_env:
- if self.repo.sync_openpgp_key_path is not None:
- out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
- with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
- openpgp_env.import_key(f)
- out.ebegin('Refreshing keys from keyserver')
- openpgp_env.refresh_keys()
- out.eend(0)
-
- # we always verify the Manifest signature, in case
- # we had to deal with key revocation case
- m = gemato.recursiveloader.ManifestRecursiveLoader(
- os.path.join(self.repo.location, 'Manifest'),
- verify_openpgp=True,
- openpgp_env=openpgp_env,
- max_jobs=self.verify_jobs)
- if not m.openpgp_signed:
- raise RuntimeError('OpenPGP signature not found on Manifest')
-
- ts = m.find_timestamp()
- if ts is None:
- raise RuntimeError('Timestamp not found in Manifest')
-
- out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
- out.einfo('Valid OpenPGP signature found:')
- out.einfo('- primary key: %s' % (
- m.openpgp_signature.primary_key_fingerprint))
- out.einfo('- subkey: %s' % (
- m.openpgp_signature.fingerprint))
- out.einfo('- timestamp: %s UTC' % (
- m.openpgp_signature.timestamp))
-
- # if nothing has changed, skip the actual Manifest
- # verification
- if not local_state_unchanged:
- out.ebegin('Verifying %s' % (self.repo.location,))
- m.assert_directory_verifies()
- out.eend(0)
+ # we always verify the Manifest signature, in case
+ # we had to deal with key revocation case
+ m = gemato.recursiveloader.ManifestRecursiveLoader(
+ os.path.join(self.repo.location, 'Manifest'),
+ verify_openpgp=True,
+ openpgp_env=openpgp_env,
+ max_jobs=self.verify_jobs)
+ if not m.openpgp_signed:
+ raise RuntimeError('OpenPGP signature not found on Manifest')
+
+ ts = m.find_timestamp()
+ if ts is None:
+ raise RuntimeError('Timestamp not found in Manifest')
+
+ out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
+ out.einfo('Valid OpenPGP signature found:')
+ out.einfo('- primary key: %s' % (
+ m.openpgp_signature.primary_key_fingerprint))
+ out.einfo('- subkey: %s' % (
+ m.openpgp_signature.fingerprint))
+ out.einfo('- timestamp: %s UTC' % (
+ m.openpgp_signature.timestamp))
+
+ # if nothing has changed, skip the actual Manifest
+ # verification
+ if not local_state_unchanged:
+ out.ebegin('Verifying %s' % (self.repo.location,))
+ m.assert_directory_verifies()
+ out.eend(0)
except GematoException as e:
writemsg_level("!!! Manifest verification failed:\n%s\n"
% (e,),
@@ -353,7 +363,8 @@ class RsyncSync(NewBase):
return (exitcode, updatecache_flg)
finally:
- pass
+ if openpgp_env is not None:
+ openpgp_env.close()
def _process_exitcode(self, exitcode, syncuri, out, maxretries):
--
2.16.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old
2018-02-02 20:42 [gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs Michał Górny
` (3 preceding siblings ...)
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 5/9] rsync: Load and update keys early Michał Górny
@ 2018-02-02 20:42 ` Michał Górny
2018-02-04 13:48 ` M. J. Everitt
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 7/9] git: Support verifying commit signature post-sync Michał Górny
` (2 subsequent siblings)
7 siblings, 1 reply; 12+ messages in thread
From: Michał Górny @ 2018-02-02 20:42 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Michał Górny
Issue an explicit warning if the Manifest timestamp for Gentoo
repository is 24 hours behind the system clock. This is meant to detect
attacks based on preventing the user from upgrading.
---
cnf/repos.conf | 1 +
man/portage.5 | 4 ++++
pym/portage/sync/modules/rsync/__init__.py | 1 +
pym/portage/sync/modules/rsync/rsync.py | 21 +++++++++++++++++++++
4 files changed, 27 insertions(+)
diff --git a/cnf/repos.conf b/cnf/repos.conf
index 4a40ff4fc..984ecd220 100644
--- a/cnf/repos.conf
+++ b/cnf/repos.conf
@@ -7,6 +7,7 @@ sync-type = rsync
sync-uri = rsync://rsync.gentoo.org/gentoo-portage
auto-sync = yes
sync-rsync-verify-metamanifest = yes
+sync-rsync-verify-max-age = 24
sync-openpgp-key-path = /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg
# for daily squashfs snapshots
diff --git a/man/portage.5 b/man/portage.5
index d4f755f51..778dedfd5 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1086,6 +1086,10 @@ directories if appropriate.
Number of parallel jobs to use when verifying nested Manifests. Defaults
to the apparent number of processors.
.TP
+.B sync\-rsync\-verify\-max\-age
+Warn if repository is older than the specified number of hours. Disabled
+when 0. Defaults to disabled.
+.TP
.B sync\-rsync\-verify\-metamanifest = yes|no
Require the repository to contain a signed MetaManifest and verify
it using \fBapp\-portage/gemato\fR. Defaults to no.
diff --git a/pym/portage/sync/modules/rsync/__init__.py b/pym/portage/sync/modules/rsync/__init__.py
index 27a2548c0..cb80f6d66 100644
--- a/pym/portage/sync/modules/rsync/__init__.py
+++ b/pym/portage/sync/modules/rsync/__init__.py
@@ -29,6 +29,7 @@ module_spec = {
'sync-rsync-extra-opts',
'sync-rsync-vcs-ignore',
'sync-rsync-verify-jobs',
+ 'sync-rsync-verify-max-age',
'sync-rsync-verify-metamanifest',
),
}
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index dc4674548..732298b3f 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -6,6 +6,7 @@ import logging
import time
import signal
import socket
+import datetime
import io
import re
import random
@@ -109,6 +110,20 @@ class RsyncSync(NewBase):
writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
level=logging.WARNING, noiselevel=-1)
self.verify_jobs = None
+ # Support overriding max age.
+ self.max_age = self.repo.module_specific_options.get(
+ 'sync-rsync-verify-max-age', '')
+ if self.max_age:
+ try:
+ self.max_age = int(self.max_age)
+ if self.max_age < 0:
+ raise ValueError(self.max_age)
+ except ValueError:
+ writemsg_level("!!! sync-rsync-max-age not a non-negative integer: %s\n" % (self.max_age,),
+ level=logging.WARNING, noiselevel=-1)
+ self.max_age = 0
+ else:
+ self.max_age = 0
openpgp_env = None
if self.verify_metamanifest and gemato is not None:
@@ -339,6 +354,12 @@ class RsyncSync(NewBase):
ts = m.find_timestamp()
if ts is None:
raise RuntimeError('Timestamp not found in Manifest')
+ if (self.max_age != 0 and
+ (datetime.datetime.utcnow() - ts.ts).hours > self.max_age):
+ out.ewarn('Manifest is over 24 hours old, this is suspicious!')
+ out.ewarn('You may want to try using another mirror and/or reporting this one:')
+ out.ewarn(' %s' % (dosyncuri,))
+ out.ewarn('')
out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
out.einfo('Valid OpenPGP signature found:')
--
2.16.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old Michał Górny
@ 2018-02-04 13:48 ` M. J. Everitt
2018-02-05 18:44 ` Michał Górny
0 siblings, 1 reply; 12+ messages in thread
From: M. J. Everitt @ 2018-02-04 13:48 UTC (permalink / raw
To: gentoo-portage-dev, Michał Górny
[-- Attachment #1.1: Type: text/plain, Size: 957 bytes --]
On 02/02/18 20:42, Michał Górny wrote[excerpted]:
> index 27a2548c0..cb80f6d66 100644
> --- a/pym/portage/sync/modules/rsync/__init__.py
> +++ b/pym/portage/sync/modules/rsync/__init__.py
> @@ -109,6 +110,20 @@ class RsyncSync(NewBase):
> writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
> level=logging.WARNING, noiselevel=-1)
> self.verify_jobs = None
> + # Support overriding max age.
> + self.max_age = self.repo.module_specific_options.get(
> + 'sync-rsync-verify-max-age', '')
> + if self.max_age:
> + try:
> + self.max_age = int(self.max_age)
> + if self.max_age < 0:
> + raise ValueError(self.max_age)
> + except ValueError:
> + writemsg_level("!!! sync-rsync-max-age not a non-negative integer: %s\n" % (self.max_age,),
A beautiful double-negative .. but would read better as "not a positive
integer" or "is a negative integer" ..
MJE
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 819 bytes --]
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old
2018-02-04 13:48 ` M. J. Everitt
@ 2018-02-05 18:44 ` Michał Górny
0 siblings, 0 replies; 12+ messages in thread
From: Michał Górny @ 2018-02-05 18:44 UTC (permalink / raw
To: gentoo-portage-dev
W dniu nie, 04.02.2018 o godzinie 13∶48 +0000, użytkownik M. J. Everitt
napisał:
> On 02/02/18 20:42, Michał Górny wrote[excerpted]:
> > index 27a2548c0..cb80f6d66 100644
> > --- a/pym/portage/sync/modules/rsync/__init__.py
> > +++ b/pym/portage/sync/modules/rsync/__init__.py
> > @@ -109,6 +110,20 @@ class RsyncSync(NewBase):
> > writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
> > level=logging.WARNING, noiselevel=-1)
> > self.verify_jobs = None
> > + # Support overriding max age.
> > + self.max_age = self.repo.module_specific_options.get(
> > + 'sync-rsync-verify-max-age', '')
> > + if self.max_age:
> > + try:
> > + self.max_age = int(self.max_age)
> > + if self.max_age < 0:
> > + raise ValueError(self.max_age)
> > + except ValueError:
> > + writemsg_level("!!! sync-rsync-max-age not a non-negative integer: %s\n" % (self.max_age,),
>
> A beautiful double-negative .. but would read better as "not a positive
> integer" or "is a negative integer" ..
>
Except that neither is correct. Zero is not positive but valid.
And the error also covers the case when it's not an integer at all.
--
Best regards,
Michał Górny
^ permalink raw reply [flat|nested] 12+ messages in thread
* [gentoo-portage-dev] [PATCH v2 7/9] git: Support verifying commit signature post-sync
2018-02-02 20:42 [gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs Michał Górny
` (4 preceding siblings ...)
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 6/9] rsync: Issue an explicit warning if Manifest timestamp is >24hr old Michał Górny
@ 2018-02-02 20:42 ` Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 8/9] git: Support running the verification against sync-openpgp-key-path Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 9/9] max-age fixup Michał Górny
7 siblings, 0 replies; 12+ messages in thread
From: Michał Górny @ 2018-02-02 20:42 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Michał Górny
Add a new sync-git-verify-commit-signature option (defaulting to false)
that verifies the top commit signature after syncing. The verification
is currently done using built-in git routines.
The verification passes if the signature is good or untrusted.
In the latter case, a warning is printed. In any other case,
the verification causes sync to fail and an appropriate error is output.
---
man/portage.5 | 4 +++
pym/portage/sync/modules/git/__init__.py | 3 +-
pym/portage/sync/modules/git/git.py | 48 ++++++++++++++++++++++++++++++--
3 files changed, 52 insertions(+), 3 deletions(-)
diff --git a/man/portage.5 b/man/portage.5
index 778dedfd5..2d5091109 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1007,6 +1007,10 @@ See also example for sync-git-clone-env.
.B sync\-git\-pull\-extra\-opts
Extra options to give to git when updating repository (git pull).
.TP
+.B sync\-git\-verify\-commit\-signature = true|false
+Require the top commit in the repository to contain a good OpenPGP
+signature. Defaults to false.
+.TP
.B sync\-hooks\-only\-on\-change
If set to true, then sync of a given repository will not trigger postsync
hooks unless hooks would have executed for a master repository or the
diff --git a/pym/portage/sync/modules/git/__init__.py b/pym/portage/sync/modules/git/__init__.py
index 2f1d35226..270d97186 100644
--- a/pym/portage/sync/modules/git/__init__.py
+++ b/pym/portage/sync/modules/git/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2017 Gentoo Foundation
+# Copyright 2014-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
doc = """Git plug-in module for portage.
@@ -58,6 +58,7 @@ module_spec = {
'sync-git-env',
'sync-git-pull-env',
'sync-git-pull-extra-opts',
+ 'sync-git-verify-commit-signature',
),
}
}
diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py
index 8b4cab273..7e5ddf3b5 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2017 Gentoo Foundation
+# Copyright 2005-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import logging
@@ -7,7 +7,7 @@ import subprocess
import portage
from portage import os
from portage.util import writemsg_level, shlex_split
-from portage.output import create_color_func
+from portage.output import create_color_func, EOutput
good = create_color_func("GOOD")
bad = create_color_func("BAD")
warn = create_color_func("WARN")
@@ -71,6 +71,7 @@ class GitSync(NewBase):
else:
# default
git_cmd_opts += " --depth 1"
+
if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
@@ -85,6 +86,8 @@ class GitSync(NewBase):
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
+ if not self.verify_head():
+ return (1, False)
return (os.EX_OK, True)
@@ -125,12 +128,53 @@ class GitSync(NewBase):
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
+ if not self.verify_head():
+ return (1, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
+ def verify_head(self):
+ if (self.repo.module_specific_options.get(
+ 'sync-git-verify-commit-signature', 'false') != 'true'):
+ return True
+
+ rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", "-1"]
+ try:
+ status = (portage._unicode_decode(
+ subprocess.check_output(rev_cmd,
+ cwd=portage._unicode_encode(self.repo.location)))
+ .strip())
+ except subprocess.CalledProcessError:
+ return False
+
+ out = EOutput()
+ if status == 'G': # good signature is good
+ out.einfo('Trusted signature found on top commit')
+ return True
+ elif status == 'U': # untrusted
+ out.ewarn('Top commit signature is valid but not trusted')
+ return True
+ else:
+ if status == 'B':
+ expl = 'bad signature'
+ elif status == 'X':
+ expl = 'expired signature'
+ elif status == 'Y':
+ expl = 'expired key'
+ elif status == 'R':
+ expl = 'revoked key'
+ elif status == 'E':
+ expl = 'unable to verify signature (missing key?)'
+ elif status == 'N':
+ expl = 'no signature'
+ else:
+ expl = 'unknown issue'
+ out.eerror('No valid signature found: %s' % (expl,))
+ return False
+
def retrieve_head(self, **kwargs):
'''Get information about the head commit'''
if kwargs:
--
2.16.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [gentoo-portage-dev] [PATCH v2 8/9] git: Support running the verification against sync-openpgp-key-path
2018-02-02 20:42 [gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs Michał Górny
` (5 preceding siblings ...)
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 7/9] git: Support verifying commit signature post-sync Michał Górny
@ 2018-02-02 20:42 ` Michał Górny
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 9/9] max-age fixup Michał Górny
7 siblings, 0 replies; 12+ messages in thread
From: Michał Górny @ 2018-02-02 20:42 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Michał Górny
---
pym/portage/sync/modules/git/git.py | 101 +++++++++++++++++++++++++-----------
1 file changed, 70 insertions(+), 31 deletions(-)
diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py
index 7e5ddf3b5..cec760d00 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -1,6 +1,7 @@
# Copyright 2005-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+import io
import logging
import subprocess
@@ -13,6 +14,12 @@ bad = create_color_func("BAD")
warn = create_color_func("WARN")
from portage.sync.syncbase import NewBase
+try:
+ from gemato.exceptions import GematoException
+ import gemato.openpgp
+except ImportError:
+ gemato = None
+
class GitSync(NewBase):
'''Git sync class'''
@@ -141,39 +148,71 @@ class GitSync(NewBase):
'sync-git-verify-commit-signature', 'false') != 'true'):
return True
- rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", "-1"]
- try:
- status = (portage._unicode_decode(
- subprocess.check_output(rev_cmd,
- cwd=portage._unicode_encode(self.repo.location)))
- .strip())
- except subprocess.CalledProcessError:
- return False
-
- out = EOutput()
- if status == 'G': # good signature is good
- out.einfo('Trusted signature found on top commit')
- return True
- elif status == 'U': # untrusted
- out.ewarn('Top commit signature is valid but not trusted')
- return True
+ if self.repo.sync_openpgp_key_path is not None:
+ if gemato is None:
+ writemsg_level("!!! Verifying against specified key requires gemato-11.0+ installed\n",
+ level=logging.ERROR, noiselevel=-1)
+ return False
+ openpgp_env = gemato.openpgp.OpenPGPEnvironment()
else:
- if status == 'B':
- expl = 'bad signature'
- elif status == 'X':
- expl = 'expired signature'
- elif status == 'Y':
- expl = 'expired key'
- elif status == 'R':
- expl = 'revoked key'
- elif status == 'E':
- expl = 'unable to verify signature (missing key?)'
- elif status == 'N':
- expl = 'no signature'
+ openpgp_env = None
+
+ try:
+ out = EOutput()
+ env = None
+ if openpgp_env is not None:
+ try:
+ out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
+ with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+ openpgp_env.import_key(f)
+ out.ebegin('Refreshing keys from keyserver')
+ openpgp_env.refresh_keys()
+ out.eend(0)
+ except GematoException as e:
+ writemsg_level("!!! Verification impossible due to keyring problem:\n%s\n"
+ % (e,),
+ level=logging.ERROR, noiselevel=-1)
+ return (1, False)
+
+ env = os.environ.copy()
+ env['GNUPGHOME'] = openpgp_env.home
+
+ rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", "-1"]
+ try:
+ status = (portage._unicode_decode(
+ subprocess.check_output(rev_cmd,
+ cwd=portage._unicode_encode(self.repo.location),
+ env=env))
+ .strip())
+ except subprocess.CalledProcessError:
+ return False
+
+ if status == 'G': # good signature is good
+ out.einfo('Trusted signature found on top commit')
+ return True
+ elif status == 'U': # untrusted
+ out.ewarn('Top commit signature is valid but not trusted')
+ return True
else:
- expl = 'unknown issue'
- out.eerror('No valid signature found: %s' % (expl,))
- return False
+ if status == 'B':
+ expl = 'bad signature'
+ elif status == 'X':
+ expl = 'expired signature'
+ elif status == 'Y':
+ expl = 'expired key'
+ elif status == 'R':
+ expl = 'revoked key'
+ elif status == 'E':
+ expl = 'unable to verify signature (missing key?)'
+ elif status == 'N':
+ expl = 'no signature'
+ else:
+ expl = 'unknown issue'
+ out.eerror('No valid signature found: %s' % (expl,))
+ return False
+ finally:
+ if openpgp_env is not None:
+ openpgp_env.close()
def retrieve_head(self, **kwargs):
'''Get information about the head commit'''
--
2.16.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [gentoo-portage-dev] [PATCH v2 9/9] max-age fixup
2018-02-02 20:42 [gentoo-portage-dev] [PATCH v2 1/9] rsync: Verify the value of sync-rsync-verify-jobs Michał Górny
` (6 preceding siblings ...)
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 8/9] git: Support running the verification against sync-openpgp-key-path Michał Górny
@ 2018-02-02 20:42 ` Michał Górny
2018-02-04 13:43 ` Michał Górny
7 siblings, 1 reply; 12+ messages in thread
From: Michał Górny @ 2018-02-02 20:42 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Michał Górny
---
man/portage.5 | 2 +-
pym/portage/sync/modules/rsync/rsync.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/man/portage.5 b/man/portage.5
index 2d5091109..549c51c73 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1091,7 +1091,7 @@ Number of parallel jobs to use when verifying nested Manifests. Defaults
to the apparent number of processors.
.TP
.B sync\-rsync\-verify\-max\-age
-Warn if repository is older than the specified number of hours. Disabled
+Warn if repository is older than the specified number of days. Disabled
when 0. Defaults to disabled.
.TP
.B sync\-rsync\-verify\-metamanifest = yes|no
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 732298b3f..cc121e0fb 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -355,8 +355,8 @@ class RsyncSync(NewBase):
if ts is None:
raise RuntimeError('Timestamp not found in Manifest')
if (self.max_age != 0 and
- (datetime.datetime.utcnow() - ts.ts).hours > self.max_age):
- out.ewarn('Manifest is over 24 hours old, this is suspicious!')
+ (datetime.datetime.utcnow() - ts.ts).days > self.max_age):
+ out.ewarn('Manifest is over %d days old, this is suspicious!' % (self.max_age,))
out.ewarn('You may want to try using another mirror and/or reporting this one:')
out.ewarn(' %s' % (dosyncuri,))
out.ewarn('')
--
2.16.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [gentoo-portage-dev] [PATCH v2 9/9] max-age fixup
2018-02-02 20:42 ` [gentoo-portage-dev] [PATCH v2 9/9] max-age fixup Michał Górny
@ 2018-02-04 13:43 ` Michał Górny
0 siblings, 0 replies; 12+ messages in thread
From: Michał Górny @ 2018-02-04 13:43 UTC (permalink / raw
To: gentoo-portage-dev
W dniu pią, 02.02.2018 o godzinie 21∶42 +0100, użytkownik Michał Górny
napisał:
> ---
> man/portage.5 | 2 +-
> pym/portage/sync/modules/rsync/rsync.py | 4 ++--
> 2 files changed, 3 insertions(+), 3 deletions(-)
>
> diff --git a/man/portage.5 b/man/portage.5
> index 2d5091109..549c51c73 100644
> --- a/man/portage.5
> +++ b/man/portage.5
> @@ -1091,7 +1091,7 @@ Number of parallel jobs to use when verifying nested Manifests. Defaults
> to the apparent number of processors.
> .TP
> .B sync\-rsync\-verify\-max\-age
> -Warn if repository is older than the specified number of hours. Disabled
> +Warn if repository is older than the specified number of days. Disabled
> when 0. Defaults to disabled.
> .TP
> .B sync\-rsync\-verify\-metamanifest = yes|no
> diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
> index 732298b3f..cc121e0fb 100644
> --- a/pym/portage/sync/modules/rsync/rsync.py
> +++ b/pym/portage/sync/modules/rsync/rsync.py
> @@ -355,8 +355,8 @@ class RsyncSync(NewBase):
> if ts is None:
> raise RuntimeError('Timestamp not found in Manifest')
> if (self.max_age != 0 and
> - (datetime.datetime.utcnow() - ts.ts).hours > self.max_age):
> - out.ewarn('Manifest is over 24 hours old, this is suspicious!')
> + (datetime.datetime.utcnow() - ts.ts).days > self.max_age):
> + out.ewarn('Manifest is over %d days old, this is suspicious!' % (self.max_age,))
> out.ewarn('You may want to try using another mirror and/or reporting this one:')
> out.ewarn(' %s' % (dosyncuri,))
> out.ewarn('')
Note: I've merged this into 6/9 now.
--
Best regards,
Michał Górny
^ permalink raw reply [flat|nested] 12+ messages in thread