[gentoo-commits] proj/portage:master commit in: lib/portage/

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     eb81c8081ae53e862f9f0f82c6119f936ca896e7
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Aug  7 18:29:51 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Aug  7 18:34:29 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=eb81c808

action_metadata: refresh eclass cache (bug 663022)

After sync, the eclass cache can contain stale data, therefore
use the update_eclasses method to refresh it.

Bug: https://bugs.gentoo.org/663022

 lib/portage/metadata.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/portage/metadata.py b/lib/portage/metadata.py
index 1abec5a89..ffb7672d3 100644
--- a/lib/portage/metadata.py
+++ b/lib/portage/metadata.py
@@ -50,8 +50,10 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
 			src_db = portdb._create_pregen_cache(path)
 
 		if src_db is not None:
-			porttrees_data.append(TreeData(portdb.auxdb[path],
-				portdb.repositories.get_repo_for_location(path).eclass_db, path, src_db))
+			eclass_db = portdb.repositories.get_repo_for_location(path).eclass_db
+			# Update eclass data which may be stale after sync.
+			eclass_db.update_eclasses()
+			porttrees_data.append(TreeData(portdb.auxdb[path], eclass_db, path, src_db))
 
 	porttrees = [tree_data.path for tree_data in porttrees_data]
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     a9f925aa0c10b728496870b42794783b46882fa8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 23 21:47:41 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Nov 23 22:01:08 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a9f925aa

portage.process.spawn: add cwd parameter

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 75ec299f0..ed1a49247 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -220,7 +220,7 @@ def cleanup():
 	pass
 
 def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
-          uid=None, gid=None, groups=None, umask=None, logfile=None,
+          uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None,
           path_lookup=True, pre_exec=None,
           close_fds=(sys.version_info < (3, 4)), unshare_net=False,
           unshare_ipc=False, unshare_mount=False, unshare_pid=False,
@@ -248,6 +248,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
 	@type groups: List
 	@param umask: An integer representing the umask for the process (see man chmod for umask details)
 	@type umask: Integer
+	@param cwd: Current working directory
+	@type cwd: String
 	@param logfile: name of a file to use for logging purposes
 	@type logfile: String
 	@param path_lookup: If the binary is not fully specified then look for it in PATH
@@ -350,7 +352,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
 		if pid == 0:
 			try:
 				_exec(binary, mycommand, opt_name, fd_pipes,
-					env, gid, groups, uid, umask, pre_exec, close_fds,
+					env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
 					unshare_net, unshare_ipc, unshare_mount, unshare_pid,
 					cgroup)
 			except SystemExit:
@@ -421,7 +423,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
 	# Everything succeeded
 	return 0
 
-def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
+def _exec(binary, mycommand, opt_name, fd_pipes,
+	env, gid, groups, uid, umask, cwd,
 	pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
 	cgroup):
 
@@ -446,6 +449,8 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
 	@type uid: Integer
 	@param umask: an int representing a unix umask (see man chmod for umask details)
 	@type umask: Integer
+	@param cwd: Current working directory
+	@type cwd: String
 	@param pre_exec: A function to be called with no arguments just prior to the exec call.
 	@type pre_exec: callable
 	@param unshare_net: If True, networking will be unshared from the spawned process
@@ -609,6 +614,8 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
 		os.setuid(int(uid))
 	if umask:
 		os.umask(umask)
+	if cwd is not None:
+		os.chdir(cwd)
 	if pre_exec:
 		pre_exec()
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     c8eae4eb00c3c14ec27f4bb8c8983ff6274e478e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Nov 24 23:45:37 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Nov 25 00:02:10 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c8eae4eb

EBUILD_PHASES: add instprep (bug 671824)

Whitelist instprep for collect_ebuild_messages, so that it will
not reject elog messages from the instprep phase like this:

!!! can't process invalid log file: /tmp/portage/$PF/temp/logging/instprep

Fixes: 3f50bfc1ae5b ("Support disabling docompress for binary package builds")
Bug: https://bugs.gentoo.org/671824
Reported-by: Duncan <1i5t5.duncan <AT> cox.net>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/const.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index ca66bc46e..edbfb9f17 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -111,6 +111,7 @@ EBUILD_PHASES            = (
 	"test",
 	"install",
 	"package",
+	"instprep",
 	"preinst",
 	"postinst",
 	"prerm",

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     e9810a30bf044d93c0348d46225ad6b2ae1a45df
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Dec  3 07:47:32 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Dec  4 01:15:03 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9810a30

portage.process.spawn: inherit env by default (bug 672440)

Make child processes inherit the current process's environment by
default, so that behavior is equivalent to the standard library's
subprocess module.

Bug: https://bugs.gentoo.org/672440
Reviewed-by: Brian Dolbec <dolsen <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index ed1a49247..ce3e42a8f 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -219,7 +219,7 @@ spawned_pids = _dummy_list()
 def cleanup():
 	pass
 
-def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
+def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
           uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None,
           path_lookup=True, pre_exec=None,
           close_fds=(sys.version_info < (3, 4)), unshare_net=False,
@@ -230,8 +230,10 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
 	
 	@param mycommand: the command to execute
 	@type mycommand: String or List (Popen style list)
-	@param env: A dict of Key=Value pairs for env variables
-	@type env: Dictionary
+	@param env: If env is not None, it must be a mapping that defines the environment
+		variables for the new process; these are used instead of the default behavior
+		of inheriting the current process's environment.
+	@type env: None or Mapping
 	@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
 	@type opt_name: String
 	@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
@@ -281,6 +283,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
 	if isinstance(mycommand, basestring):
 		mycommand = mycommand.split()
 
+	env = os.environ if env is None else env
+
 	if sys.hexversion < 0x3000000:
 		# Avoid a potential UnicodeEncodeError from os.execve().
 		env_bytes = {}

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     89de8064e8d186a7a001dd1f2f5c486c64f4104f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 31 05:23:29 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Dec 31 05:25:06 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=89de8064

locks: chown lockfiles only if permitted (bug 674090)

Reported-by: Jeroen Roovers <jer <AT> gentoo.org>
Bug: https://bugs.gentoo.org/674090
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index f61e1819a..a4e7ec53f 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -163,7 +163,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
 
 			if not preexisting:
 				try:
-					if os.stat(lockfilename).st_gid != portage_gid:
+					if portage.data.secpass >= 1 and os.stat(lockfilename).st_gid != portage_gid:
 						os.chown(lockfilename, -1, portage_gid)
 				except OSError as e:
 					if e.errno in (errno.ENOENT, errno.ESTALE):
@@ -463,7 +463,7 @@ def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning,
 				if not preexisting:
 					# Don't chown the file if it is preexisting, since we
 					# want to preserve existing permissions in that case.
-					if myfd_st.st_gid != portage_gid:
+					if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid:
 						os.fchown(myfd, -1, portage_gid)
 			except OSError as e:
 				if e.errno not in (errno.ENOENT, errno.ESTALE):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     c2a9850a25b2f32a25b43ef30189cd6657f397ad
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 29 06:56:40 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jan  4 03:04:49 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c2a9850a

process.spawn: validate unshare calls (bug 673900)

In order to prevent failed unshare calls from corrupting the state
of an essential process, validate the relevant unshare call in a
short-lived subprocess. An unshare call is considered valid if it
successfully executes in a short-lived subprocess.

Bug: https://bugs.gentoo.org/673900
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 159 +++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 133 insertions(+), 26 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index ce3e42a8f..7103b6b31 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -6,6 +6,7 @@
 import atexit
 import errno
 import fcntl
+import multiprocessing
 import platform
 import signal
 import socket
@@ -338,11 +339,29 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 		fd_pipes[1] = pw
 		fd_pipes[2] = pw
 
-	# This caches the libc library lookup in the current
-	# process, so that it's only done once rather than
-	# for each child process.
+	# This caches the libc library lookup and _unshare_validator results
+	# in the current process, so that results are cached for use in
+	# child processes.
+	unshare_flags = 0
 	if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
-		find_library("c")
+		# from /usr/include/bits/sched.h
+		CLONE_NEWNS = 0x00020000
+		CLONE_NEWIPC = 0x08000000
+		CLONE_NEWPID = 0x20000000
+		CLONE_NEWNET = 0x40000000
+
+		if unshare_net:
+			unshare_flags |= CLONE_NEWNET
+		if unshare_ipc:
+			unshare_flags |= CLONE_NEWIPC
+		if unshare_mount:
+			# NEWNS = mount namespace
+			unshare_flags |= CLONE_NEWNS
+		if unshare_pid:
+			# we also need mount namespace for slave /proc
+			unshare_flags |= CLONE_NEWPID | CLONE_NEWNS
+
+		_unshare_validate(unshare_flags)
 
 	# Force instantiation of portage.data.userpriv_groups before the
 	# fork, so that the result is cached in the main process.
@@ -358,7 +377,7 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 				_exec(binary, mycommand, opt_name, fd_pipes,
 					env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
 					unshare_net, unshare_ipc, unshare_mount, unshare_pid,
-					cgroup)
+					unshare_flags, cgroup)
 			except SystemExit:
 				raise
 			except Exception as e:
@@ -430,7 +449,7 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 def _exec(binary, mycommand, opt_name, fd_pipes,
 	env, gid, groups, uid, umask, cwd,
 	pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
-	cgroup):
+	unshare_flags, cgroup):
 
 	"""
 	Execute a given binary with options
@@ -466,6 +485,8 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 	@type unshare_mount: Boolean
 	@param unshare_pid: If True, PID ns will be unshared from the spawned process
 	@type unshare_pid: Boolean
+	@param unshare_flags: Flags for the unshare(2) function
+	@type unshare_flags: Integer
 	@param cgroup: CGroup path to bind the process to
 	@type cgroup: String
 	@rtype: None
@@ -527,28 +548,19 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 		if filename is not None:
 			libc = LoadLibrary(filename)
 			if libc is not None:
-				# from /usr/include/bits/sched.h
-				CLONE_NEWNS = 0x00020000
-				CLONE_NEWIPC = 0x08000000
-				CLONE_NEWPID = 0x20000000
-				CLONE_NEWNET = 0x40000000
-
-				flags = 0
-				if unshare_net:
-					flags |= CLONE_NEWNET
-				if unshare_ipc:
-					flags |= CLONE_NEWIPC
-				if unshare_mount:
-					# NEWNS = mount namespace
-					flags |= CLONE_NEWNS
-				if unshare_pid:
-					# we also need mount namespace for slave /proc
-					flags |= CLONE_NEWPID | CLONE_NEWNS
-
 				try:
-					if libc.unshare(flags) != 0:
+					# Since a failed unshare call could corrupt process
+					# state, first validate that the call can succeed.
+					# The parent process should call _unshare_validate
+					# before it forks, so that all child processes can
+					# reuse _unshare_validate results that have been
+					# cached by the parent process.
+					errno_value = _unshare_validate(unshare_flags)
+					if errno_value == 0 and libc.unshare(unshare_flags) != 0:
+						errno_value = ctypes.get_errno()
+					if errno_value != 0:
 						writemsg("Unable to unshare: %s\n" % (
-							errno.errorcode.get(ctypes.get_errno(), '?')),
+							errno.errorcode.get(errno_value, '?')),
 							noiselevel=-1)
 					else:
 						if unshare_pid:
@@ -626,6 +638,101 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 	# And switch to the new process.
 	os.execve(binary, myargs, env)
 
+
+class _unshare_validator(object):
+	"""
+	In order to prevent failed unshare calls from corrupting the state
+	of an essential process, validate the relevant unshare call in a
+	short-lived subprocess. An unshare call is considered valid if it
+	successfully executes in a short-lived subprocess.
+	"""
+
+	def __init__(self):
+		self._results = {}
+
+	def __call__(self, flags):
+		"""
+		Validate unshare with the given flags. Results are cached.
+
+		@rtype: int
+		@returns: errno value, or 0 if no error occurred.
+		"""
+
+		try:
+			return self._results[flags]
+		except KeyError:
+			result = self._results[flags] = self._validate(flags)
+			return result
+
+	@classmethod
+	def _validate(cls, flags):
+		"""
+		Perform validation.
+
+		@param flags: unshare flags
+		@type flags: int
+		@rtype: int
+		@returns: errno value, or 0 if no error occurred.
+		"""
+		filename = find_library("c")
+		if filename is None:
+			return errno.ENOTSUP
+
+		libc = LoadLibrary(filename)
+		if libc is None:
+			return errno.ENOTSUP
+
+		parent_pipe, subproc_pipe = multiprocessing.Pipe(duplex=False)
+
+		proc = multiprocessing.Process(
+			target=cls._run_subproc,
+			args=(subproc_pipe, cls._validate_subproc, (libc.unshare, flags)))
+		proc.start()
+		subproc_pipe.close()
+
+		result = parent_pipe.recv()
+		parent_pipe.close()
+		proc.join()
+
+		return result
+
+	@staticmethod
+	def _run_subproc(subproc_pipe, target, args=(), kwargs={}):
+		"""
+		Call function and send return value to parent process.
+
+		@param subproc_pipe: connection to parent process
+		@type subproc_pipe: multiprocessing.Connection
+		@param target: target is the callable object to be invoked
+		@type target: callable
+		@param args: the argument tuple for the target invocation
+		@type args: tuple
+		@param kwargs: dictionary of keyword arguments for the target invocation
+		@type kwargs: dict
+		"""
+		subproc_pipe.send(target(*args, **kwargs))
+		subproc_pipe.close()
+
+	@staticmethod
+	def _validate_subproc(unshare, flags):
+		"""
+		Perform validation. Calls to this method must be isolated in a
+		subprocess, since the unshare function is called for purposes of
+		validation.
+
+		@param unshare: unshare function
+		@type unshare: callable
+		@param flags: unshare flags
+		@type flags: int
+		@rtype: int
+		@returns: errno value, or 0 if no error occurred.
+		"""
+		return 0 if unshare(flags) == 0 else ctypes.get_errno()
+
+
+_unshare_validate = _unshare_validator()
+
+
 def _setup_pipes(fd_pipes, close_fds=True, inheritable=None):
 	"""Setup pipes for a forked process.
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     16d3d31dbc971ce95d80d6ec0645d6ee92b6baa2
Author:     Zac Medico <zachary.medico <AT> sony <DOT> com>
AuthorDate: Wed Feb  6 22:21:47 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb  7 19:00:08 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=16d3d31d

locks: handle lock file removal on NFS (bug 636798)

Handle cases where a lock file on NFS has been removed by a concurrent
process that held the lock earlier. Since stat is not reliable for
removed files on NFS with the default file attribute cache behavior
('ac' mount option), create a temporary hardlink in order to prove
that the file path exists on the NFS server.

Bug: https://bugs.gentoo.org/636798
Copyright: Sony Interactive Entertainment Inc.
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 78 +++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 65 insertions(+), 13 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index a4e7ec53f..74c2c086a 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -1,5 +1,5 @@
 # portage: Lock management code
-# Copyright 2004-2014 Gentoo Foundation
+# Copyright 2004-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
@@ -20,6 +20,7 @@ from portage.exception import (DirectoryNotFound, FileNotFound,
 	InvalidData, TryAgain, OperationNotPermitted, PermissionDenied,
 	ReadOnlyFileSystem)
 from portage.util import writemsg
+from portage.util.install_mask import _raise_exc
 from portage.localization import _
 
 if sys.hexversion >= 0x3000000:
@@ -148,18 +149,17 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
 		preexisting = os.path.exists(lockfilename)
 		old_mask = os.umask(000)
 		try:
-			try:
-				myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
-			except OSError as e:
-				func_call = "open('%s')" % lockfilename
-				if e.errno == OperationNotPermitted.errno:
-					raise OperationNotPermitted(func_call)
-				elif e.errno == PermissionDenied.errno:
-					raise PermissionDenied(func_call)
-				elif e.errno == ReadOnlyFileSystem.errno:
-					raise ReadOnlyFileSystem(func_call)
+			while True:
+				try:
+					myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
+				except OSError as e:
+					if e.errno in (errno.ENOENT, errno.ESTALE) and os.path.isdir(os.path.dirname(lockfilename)):
+						# Retry required for NFS (see bug 636798).
+						continue
+					else:
+						_raise_exc(e)
 				else:
-					raise
+					break
 
 			if not preexisting:
 				try:
@@ -273,7 +273,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
 
 		
 	if isinstance(lockfilename, basestring) and \
-		myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
+		myfd != HARDLINK_FD and _lockfile_was_removed(myfd, lockfilename):
 		# The file was deleted on us... Keep trying to make one...
 		os.close(myfd)
 		writemsg(_("lockfile recurse\n"), 1)
@@ -298,6 +298,58 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
 	writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
 	return (lockfilename, myfd, unlinkfile, locking_method)
 
+
+def _lockfile_was_removed(lock_fd, lock_path):
+	"""
+	Check if lock_fd still refers to a file located at lock_path, since
+	the file may have been removed by a concurrent process that held the
+	lock earlier. This implementation includes support for NFS, where
+	stat is not reliable for removed files due to the default file
+	attribute cache behavior ('ac' mount option).
+
+	@param lock_fd: an open file descriptor for a lock file
+	@type lock_fd: int
+	@param lock_path: path of lock file
+	@type lock_path: str
+	@rtype: bool
+	@return: True if lock_path exists and corresponds to lock_fd, False otherwise
+	"""
+	try:
+		fstat_st = os.fstat(lock_fd)
+	except OSError as e:
+		if e.errno not in (errno.ENOENT, errno.ESTALE):
+			_raise_exc(e)
+		return True
+
+	# Since stat is not reliable for removed files on NFS with the default
+	# file attribute cache behavior ('ac' mount option), create a temporary
+	# hardlink in order to prove that the file path exists on the NFS server.
+	hardlink_path = hardlock_name(lock_path)
+	try:
+		os.unlink(hardlink_path)
+	except OSError as e:
+		if e.errno not in (errno.ENOENT, errno.ESTALE):
+			_raise_exc(e)
+	try:
+		try:
+			os.link(lock_path, hardlink_path)
+		except OSError as e:
+			if e.errno not in (errno.ENOENT, errno.ESTALE):
+				_raise_exc(e)
+			return True
+
+		hardlink_stat = os.stat(hardlink_path)
+		if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
+			return True
+	finally:
+		try:
+			os.unlink(hardlink_path)
+		except OSError as e:
+			if e.errno not in (errno.ENOENT, errno.ESTALE):
+				_raise_exc(e)
+	return False
+
+
 def _fstat_nlink(fd):
 	"""
 	@param fd: an open file descriptor

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     34532af167cff457c3cccda4ea4249a0bc26481a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 17 22:44:29 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Feb 17 23:02:48 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=34532af1

locks: handle sshfs hardlink inode numbers (bug 678218)

Since hardlinks on sshfs do not have matching inode numbers, detect
this behavior and use a simple stat call to detect if lock_path has
been removed.

Bug: https://bugs.gentoo.org/678218
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 74c2c086a..510925da0 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -340,6 +340,33 @@ def _lockfile_was_removed(lock_fd, lock_path):
 
 		hardlink_stat = os.stat(hardlink_path)
 		if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
+			# Create another hardlink in order to detect whether or not
+			# hardlink inode numbers are expected to match. For example,
+			# inode numbers are not expected to match for sshfs.
+			inode_test = hardlink_path + '-inode-test'
+			try:
+				os.unlink(inode_test)
+			except OSError as e:
+				if e.errno not in (errno.ENOENT, errno.ESTALE):
+					_raise_exc(e)
+			try:
+				os.link(hardlink_path, inode_test)
+			except OSError as e:
+				if e.errno not in (errno.ENOENT, errno.ESTALE):
+					_raise_exc(e)
+				return True
+			else:
+				if not os.path.samefile(hardlink_path, inode_test):
+					# This implies that inode numbers are not expected
+					# to match for this file system, so use a simple
+					# stat call to detect if lock_path has been removed.
+					return not os.path.exists(lock_path)
+			finally:
+				try:
+					os.unlink(inode_test)
+				except OSError as e:
+					if e.errno not in (errno.ENOENT, errno.ESTALE):
+						_raise_exc(e)
 			return True
 	finally:
 		try:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     d192213dab5b55fba9d200daad1cb2dde763136e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 19 07:51:07 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 19 07:54:39 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d192213d

locks: check for removed lockfile only if unlinkfile is True

Check for a removed lockfile only if unlinkfile is True, since
in this case the parent directory must be writable. This fixes
emergelog usage where the parent directory is not writable.

Bug: https://bugs.gentoo.org/678278
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 510925da0..a23d5cb56 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -273,7 +273,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
 
 		
 	if isinstance(lockfilename, basestring) and \
-		myfd != HARDLINK_FD and _lockfile_was_removed(myfd, lockfilename):
+		myfd != HARDLINK_FD and unlinkfile and _lockfile_was_removed(myfd, lockfilename):
 		# The file was deleted on us... Keep trying to make one...
 		os.close(myfd)
 		writemsg(_("lockfile recurse\n"), 1)

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     02922cb3a7eb72e9bf4cec9e30b35aa4adaa51c8
Author:     Zac Medico <zachary.medico <AT> sony <DOT> com>
AuthorDate: Tue Feb 19 23:59:35 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 20 00:38:09 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=02922cb3

lockfile: use loop instead of recursion

Copyright: Sony Interactive Entertainment Inc.
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index a23d5cb56..fff27e55e 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -107,7 +107,34 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
 	If wantnewlockfile is True then this creates a lockfile in the parent
 	directory as the file: '.' + basename + '.portage_lockfile'.
 	"""
+	lock = None
+	while lock is None:
+		lock = _lockfile_iteration(mypath, wantnewlockfile=wantnewlockfile,
+			unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags)
+		if lock is None:
+			writemsg(_("lockfile removed by previous lock holder, retrying\n"), 1)
+	return lock
 
+
+def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False,
+	waiting_msg=None, flags=0):
+	"""
+	Acquire a lock on mypath, without retry. Return None if the lockfile
+	was removed by previous lock holder (caller must retry).
+
+	@param mypath: lock file path
+	@type mypath: str
+	@param wantnewlockfile: use a separate new lock file
+	@type wantnewlockfile: bool
+	@param unlinkfile: remove lock file prior to unlock
+	@type unlinkfile: bool
+	@param waiting_msg: message to show before blocking
+	@type waiting_msg: str
+	@param flags: lock flags (only supports os.O_NONBLOCK)
+	@type flags: int
+	@rtype: bool
+	@return: unlockfile tuple on success, None if retry is needed
+	"""
 	if not mypath:
 		raise InvalidData(_("Empty path given"))
 
@@ -274,12 +301,9 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
 		
 	if isinstance(lockfilename, basestring) and \
 		myfd != HARDLINK_FD and unlinkfile and _lockfile_was_removed(myfd, lockfilename):
-		# The file was deleted on us... Keep trying to make one...
+		# Removed by previous lock holder... Caller will retry...
 		os.close(myfd)
-		writemsg(_("lockfile recurse\n"), 1)
-		lockfilename, myfd, unlinkfile, locking_method = lockfile(
-			mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
-			waiting_msg=waiting_msg, flags=flags)
+		return None
 
 	if myfd != HARDLINK_FD:
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     6b95678e23bf2b68bfd27d7c50777efc38fd4ea3
Author:     Zac Medico <zachary.medico <AT> sony <DOT> com>
AuthorDate: Wed Feb 20 00:17:26 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 20 00:38:09 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6b95678e

lockfile: do not leak file descriptor after _lockfile_was_removed exception

Bug: https://bugs.gentoo.org/678278
Copyright: Sony Interactive Entertainment Inc.
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index fff27e55e..609c8b2dc 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -299,11 +299,18 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False,
 			raise
 
 		
-	if isinstance(lockfilename, basestring) and \
-		myfd != HARDLINK_FD and unlinkfile and _lockfile_was_removed(myfd, lockfilename):
-		# Removed by previous lock holder... Caller will retry...
-		os.close(myfd)
-		return None
+	if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and unlinkfile:
+		try:
+			removed = _lockfile_was_removed(myfd, lockfilename)
+		except Exception:
+			# Do not leak the file descriptor here.
+			os.close(myfd)
+			raise
+		else:
+			if removed:
+				# Removed by previous lock holder... Caller will retry...
+				os.close(myfd)
+				return None
 
 	if myfd != HARDLINK_FD:
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     875a23c347507bb963e89c6f22ff763063c9500b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 27 17:49:37 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 27 18:21:44 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=875a23c3

Format _testing,_deprecated_eapis one per line

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/__init__.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index f361b9dcf..46077ea84 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -458,8 +458,21 @@ def abssymlink(symlink, target=None):
 
 _doebuild_manifest_exempt_depend = 0
 
-_testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", "5-hdepend", "7_pre1", "7"])
-_deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1", "5_pre1", "5_pre2", "6_pre1"])
+_testing_eapis = frozenset([
+	"4-python",
+	"4-slot-abi",
+	"5-progress",
+	"5-hdepend",
+	"7_pre1",
+])
+_deprecated_eapis = frozenset([
+	"4_pre1",
+	"3_pre2",
+	"3_pre1",
+	"5_pre1",
+	"5_pre2",
+	"6_pre1",
+])
 _supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI + 1)] + list(_testing_eapis) + list(_deprecated_eapis))
 
 def _eapi_is_deprecated(eapi):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     ee2e8e8d65713760f85664bcb004e67aed8228d3
Author:     i.Dark_Templar <darktemplar <AT> dark-templar-archives <DOT> net>
AuthorDate: Mon May 27 20:23:18 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May 28 01:45:04 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ee2e8e8d

Don't modify /etc/mtab from temporary namespaces

These records are never removed and just pollute
/etc/mtab if that's a regular file.
And if /etc/mtab isn't a regular file,
then attempts to modify it are pointless.

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 0dba55de3..dfbda75de 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -617,7 +617,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 								os._exit(1)
 							# mount new /proc for our namespace
 							s = subprocess.Popen(['mount',
-								'-t', 'proc', 'proc', '/proc'])
+								'-n', '-t', 'proc', 'proc', '/proc'])
 							mount_ret = s.wait()
 							if mount_ret != 0:
 								writemsg("Unable to mount new /proc: %d\n" % (mount_ret,),

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     590e32d05fce225dfdab6308555f4a7668378d79
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 31 19:51:34 2019 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Thu Aug  1 19:01:21 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=590e32d0

Configure additional addresses on the lo interface for network-sandbox

This works around some strange behavior in glibc's getaddrinfo()
implementation when the AI_ADDRCONFIG flag is set.

For example:

  struct addrinfo *res, hints = { .ai_family = AF_INET, .ai_flags = AI_ADDRCONFIG };
  getaddrinfo("localhost", NULL, &hints, &res);

This returns no results if there are no non-loopback addresses configured.

Bug: https://bugs.gentoo.org/690758
Bug: https://sourceware.org/bugzilla/show_bug.cgi?id=12377#c13
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/process.py | 50 +++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 37 insertions(+), 13 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index dfbda75de..77f7fac02 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -446,6 +446,42 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 	# Everything succeeded
 	return 0
 
+def _configure_loopback_interface():
+	"""
+	Configure the loopback interface.
+	"""
+
+	IFF_UP = 0x1
+	ifreq = struct.pack('16sh', b'lo', IFF_UP)
+	SIOCSIFFLAGS = 0x8914
+
+	sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
+	try:
+		fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
+	except IOError as e:
+		writemsg("Unable to enable loopback interface: %s\n" % e.strerror, noiselevel=-1)
+	sock.close()
+
+	# We add some additional addresses to work around odd behavior in glibc's
+	# getaddrinfo() implementation when the AI_ADDRCONFIG flag is set.
+	#
+	# For example:
+	#
+	#   struct addrinfo *res, hints = { .ai_family = AF_INET, .ai_flags = AI_ADDRCONFIG };
+	#   getaddrinfo("localhost", NULL, &hints, &res);
+	#
+	# This returns no results if there are no non-loopback addresses
+	# configured for a given address family.
+	#
+	# Bug: https://bugs.gentoo.org/690758
+	# Bug: https://sourceware.org/bugzilla/show_bug.cgi?id=12377#c13
+
+	try:
+		subprocess.call(['ip', 'address', 'add', '10.0.0.1/8', 'dev', 'lo'])
+		subprocess.call(['ip', 'address', 'add', 'fd00::1/8', 'dev', 'lo'])
+	except OSError as e:
+		writemsg("Error calling 'ip': %s\n" % e.strerror, noiselevel=-1)
+
 def _exec(binary, mycommand, opt_name, fd_pipes,
 	env, gid, groups, uid, umask, cwd,
 	pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
@@ -624,19 +660,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 									noiselevel=-1)
 								os._exit(1)
 						if unshare_net:
-							# 'up' the loopback
-							IFF_UP = 0x1
-							ifreq = struct.pack('16sh', b'lo', IFF_UP)
-							SIOCSIFFLAGS = 0x8914
-
-							sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
-							try:
-								fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
-							except IOError as e:
-								writemsg("Unable to enable loopback interface: %s\n" % (
-									errno.errorcode.get(e.errno, '?')),
-									noiselevel=-1)
-							sock.close()
+							_configure_loopback_interface()
 				except AttributeError:
 					# unshare() not supported by libc
 					pass

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     3572449357927d40e43c937bc35e26d78eb3b91a
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Fri Aug  2 15:57:39 2019 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Fri Aug  2 20:01:10 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=35724493

process: redirect IPv6 'ip' command output to /dev/null

Bug: https://bugs.gentoo.org/691290
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/process.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 77f7fac02..690421815 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -478,8 +478,10 @@ def _configure_loopback_interface():
 
 	try:
 		subprocess.call(['ip', 'address', 'add', '10.0.0.1/8', 'dev', 'lo'])
-		subprocess.call(['ip', 'address', 'add', 'fd00::1/8', 'dev', 'lo'])
-	except OSError as e:
+		with open(os.devnull, 'wb', 0) as devnull:
+			subprocess.call(['ip', 'address', 'add', 'fd00::1/8', 'dev', 'lo'],
+				stdout=devnull, stderr=devnull)
+	except EnvironmentError as e:
 		writemsg("Error calling 'ip': %s\n" % e.strerror, noiselevel=-1)
 
 def _exec(binary, mycommand, opt_name, fd_pipes,

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     7d05c3e88dd313e3298d74ce7241cbc5b85df54a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Aug  3 18:40:31 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Aug  4 17:41:25 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7d05c3e8

process: add _has_ipv6() function

Add _has_ipv6() function and use it in _configure_loopback_interface()
to decide whether to add an IPv6 address.

Bug: https://bugs.gentoo.org/691290
Reviewed-by: Mike Gilbert <floppym <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 43 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 40 insertions(+), 3 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 690421815..2a2cbd972 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -339,6 +339,9 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 		fd_pipes[1] = pw
 		fd_pipes[2] = pw
 
+	# Cache _has_ipv6() result for use in child processes.
+	_has_ipv6()
+
 	# This caches the libc library lookup and _unshare_validator results
 	# in the current process, so that results are cached for use in
 	# child processes.
@@ -446,6 +449,41 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 	# Everything succeeded
 	return 0
 
+__has_ipv6 = None
+
+def _has_ipv6():
+	"""
+	Test that both userland and kernel support IPv6, by attempting
+	to create a socket and listen on any unused port of the IPv6
+	::1 loopback address.
+
+	@rtype: bool
+	@return: True if IPv6 is supported, False otherwise.
+	"""
+	global __has_ipv6
+
+	if __has_ipv6 is None:
+		if socket.has_ipv6:
+			sock = None
+			try:
+				# With ipv6.disable=0 and ipv6.disable_ipv6=1, socket creation
+				# succeeds, but then the bind call fails with this error:
+				# [Errno 99] Cannot assign requested address.
+				sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+				sock.bind(('::1', 0))
+			except EnvironmentError:
+				__has_ipv6 = False
+			else:
+				__has_ipv6 = True
+			finally:
+				# python2.7 sockets do not support context management protocol
+				if sock is not None:
+					sock.close()
+		else:
+			__has_ipv6 = False
+
+	return __has_ipv6
+
 def _configure_loopback_interface():
 	"""
 	Configure the loopback interface.
@@ -478,9 +516,8 @@ def _configure_loopback_interface():
 
 	try:
 		subprocess.call(['ip', 'address', 'add', '10.0.0.1/8', 'dev', 'lo'])
-		with open(os.devnull, 'wb', 0) as devnull:
-			subprocess.call(['ip', 'address', 'add', 'fd00::1/8', 'dev', 'lo'],
-				stdout=devnull, stderr=devnull)
+		if _has_ipv6():
+			subprocess.call(['ip', 'address', 'add', 'fd00::1/8', 'dev', 'lo'])
 	except EnvironmentError as e:
 		writemsg("Error calling 'ip': %s\n" % e.strerror, noiselevel=-1)
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     53801bfdd8037d1b0631412d56217207e43ff259
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Aug 18 21:49:31 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Aug 18 21:53:18 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=53801bfd

AmbiguousPackageName: initialize self.args for pypy compat

For pypy 7.1 self.args is empty, triggering this IndexError:

Traceback (most recent call last):
  File "<string>", line 53, in <module>
  File "/usr/lib/pypy2.7/site-packages/_emerge/main.py", line 1289, in emerge_main
    return run_action(emerge_config)
  File "/usr/lib/pypy2.7/site-packages/_emerge/actions.py", line 3230, in run_action
    emerge_config.args, spinner)
  File "/usr/lib/pypy2.7/site-packages/_emerge/actions.py", line 2081, in action_uninstall
    for i in e.args[0]:
IndexError: tuple index out of range

Bug: https://bugs.gentoo.org/692412
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/exception.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/portage/exception.py b/lib/portage/exception.py
index aed8beeb9..a1c3c5f17 100644
--- a/lib/portage/exception.py
+++ b/lib/portage/exception.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import signal
@@ -149,6 +149,10 @@ class AmbiguousPackageName(ValueError, PortageException):
 	to the existence of multiple matches in different categories. This inherits
 	from ValueError, for backward compatibility with calling code that already
 	handles ValueError."""
+	def __init__(self, *args, **kwargs):
+		self.args = args
+		super(AmbiguousPackageName, self).__init__(*args, **kwargs)
+
 	def __str__(self):
 		return ValueError.__str__(self)
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     466efb88ab3d72b90aff8c6235f22d7cee99fe15
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Sat Sep  7 06:35:03 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep  7 06:38:25 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=466efb88

grab_updates: use key for file name sort

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/update.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/portage/update.py b/lib/portage/update.py
index 83fc3d2b4..1920d213a 100644
--- a/lib/portage/update.py
+++ b/lib/portage/update.py
@@ -188,10 +188,8 @@ def grab_updates(updpath, prev_mtimes=None):
 	if len(mylist) == 0:
 		return []
 	
-	# update names are mangled to make them sort properly
-	mylist = [myfile[3:]+"-"+myfile[:2] for myfile in mylist]
-	mylist.sort()
-	mylist = [myfile[5:]+"-"+myfile[:4] for myfile in mylist]
+	# sort by (year, quarter)
+	mylist.sort(key=lambda x: (x[3:], x[:2]))
 
 	update_data = []
 	for myfile in mylist:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     5c33a020457308af1b9b3937c5a95aa5be2a888f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 17 02:42:27 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 17 02:52:44 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5c33a020

_lockfile_iteration: eliminate recursion and fd leak after chown failure

Fixes: 02922cb3a7eb ("lockfile: use loop instead of recursion")
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 609c8b2dc..72ac2fc70 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -194,10 +194,8 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False,
 						os.chown(lockfilename, -1, portage_gid)
 				except OSError as e:
 					if e.errno in (errno.ENOENT, errno.ESTALE):
-						return lockfile(mypath,
-							wantnewlockfile=wantnewlockfile,
-							unlinkfile=unlinkfile, waiting_msg=waiting_msg,
-							flags=flags)
+						os.close(myfd)
+						return None
 					else:
 						writemsg("%s: chown('%s', -1, %d)\n" % \
 							(e, lockfilename, portage_gid), noiselevel=-1)

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     07f5c5330d4508990ad52e66af4e18f1f860b2eb
Author:     Ed Baker <edward.baker <AT> intel <DOT> com>
AuthorDate: Fri Nov  8 22:12:27 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Nov 12 22:24:31 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=07f5c533

Keep https_proxy populated for curl downloads

When building Chromium OS behind a proxy, https_proxy needs to be set
for curl. Additional discussion can be found at [1]. Chromium configures
the chroot FETCHCOMMAND at [2].

Example fetch command:
curl --ipv4 -f -y 30 --retry 9 -L \
     --output /var/cache/chromeos-cache/distfiles/host/zlib-1.2.11.tar.gz \
     https://commondatastorage.googleapis.com/chromeos-localmirror/distfiles/zlib-1.2.11.tar.gz

Example error message:
>>> 13:49:14 === Start output for job zlib-1.2.11 ===
zlib-1.2.11: >>> Downloading 'https://commondatastorage.googleapis.com/chromeos-localmirror/distfiles/zlib-1.2.11.tar.gz'
zlib-1.2.11:   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
zlib-1.2.11:                                  Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:--  0:02:10 --:--:--     0
zlib-1.2.11: curl: (7) Failed to connect to commondatastorage.googleapis.com port 443: Connection timed out
zlib-1.2.11: >>> Downloading 'https://commondatastorage.googleapis.com/chromeos-mirror/gentoo/distfiles/zlib-1.2.11.tar.gz'
zlib-1.2.11:   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
zlib-1.2.11:                                  Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:--  0:02:11 --:--:--     0
zlib-1.2.11: curl: (7) Failed to connect to commondatastorage.googleapis.com port 443: Connection timed out
zlib-1.2.11: !!! Couldn't download 'zlib-1.2.11.tar.gz'. Aborting.
zlib-1.2.11:  * Fetch failed for sys-libs/zlib-1.2.11, Log file:
zlib-1.2.11:  *   /var/log/portage/sys-libs:zlib-1.2.11:20191108-204452.log

[1] https://bugs.chromium.org/p/chromium/issues/detail?id=1021751
[2] https://chromium.googlesource.com/chromiumos/platform/crosutils/+/master/sdk_lib/make_conf_util.sh

Closes: https://github.com/gentoo/portage/pull/478
Signed-off-by: Edward Baker <edward.baker <AT> intel.com>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 7bb2b1aa0..71cbdad2e 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -563,7 +563,7 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None,
 		clean_env = {}
 		for k in ('PATH', 'PORTAGE_GRPNAME', 'PORTAGE_REPOSITORIES', 'PORTAGE_USERNAME',
 			'PYTHONPATH', 'SSH_AGENT_PID', 'SSH_AUTH_SOCK', 'TERM',
-			'ftp_proxy', 'http_proxy', 'no_proxy',
+			'ftp_proxy', 'http_proxy', 'https_proxy', 'no_proxy',
 			'__PORTAGE_TEST_HARDLINK_LOCKS'):
 			v = settings.get(k)
 			if v is not None:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     e385eee1614b7b1e296b0e9560ad2d87ace9ea69
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Dec 15 22:11:55 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec 15 22:21:23 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e385eee1

process: display relevant FEATURES when unshare fails

For example, the message will now look like this:

Unable to unshare: EPERM (for FEATURES="ipc-sandbox mount-sandbox network-sandbox pid-sandbox")

Bug: https://bugs.gentoo.org/680456
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index df63ae72f..c1fc4bcf6 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -628,8 +628,19 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 					if errno_value == 0 and libc.unshare(unshare_flags) != 0:
 						errno_value = ctypes.get_errno()
 					if errno_value != 0:
-						writemsg("Unable to unshare: %s\n" % (
-							errno.errorcode.get(errno_value, '?')),
+
+						involved_features = []
+						if unshare_ipc:
+							involved_features.append('ipc-sandbox')
+						if unshare_mount:
+							involved_features.append('mount-sandbox')
+						if unshare_net:
+							involved_features.append('network-sandbox')
+						if unshare_pid:
+							involved_features.append('pid-sandbox')
+
+						writemsg("Unable to unshare: %s (for FEATURES=\"%s\")\n" % (
+							errno.errorcode.get(errno_value, '?'), ' '.join(involved_features)),
 							noiselevel=-1)
 					else:
 						if unshare_pid:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     c3627bb8302c563591249597c7a3ab6ab24b68ea
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb  2 08:51:50 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Feb  2 08:58:08 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c3627bb8

sphinx-build: avoid autodoc ModuleNotFoundError for selinux

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_selinux.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/portage/_selinux.py b/lib/portage/_selinux.py
index 985e96628..49e2e8e58 100644
--- a/lib/portage/_selinux.py
+++ b/lib/portage/_selinux.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 # Don't use the unicode-wrapped os and shutil modules here since
@@ -8,12 +8,15 @@ import shutil
 import sys
 import warnings
 
+try:
+	import selinux
+except ImportError:
+	selinux = None
+
 import portage
 from portage import _encodings
 from portage import _native_string, _unicode_decode
 from portage.localization import _
-portage.proxy.lazyimport.lazyimport(globals(),
-	'selinux')
 
 def copyfile(src, dest):
 	src = _native_string(src, encoding=_encodings['fs'], errors='strict')

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     c4892c36eb1eebbee322796b7acfcd6874cae0c1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb  4 06:26:49 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb  4 06:40:52 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c4892c36

xpak: break_hardlinks: use native copyfile for cow

Enable zero-copy, sparse copy, and reflinks.

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/xpak.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/portage/xpak.py b/lib/portage/xpak.py
index e11f26e6c..c708190b9 100644
--- a/lib/portage/xpak.py
+++ b/lib/portage/xpak.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2014 Gentoo Foundation
+# Copyright 2001-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 
@@ -33,6 +33,7 @@ from portage import normalize_path
 from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
+from portage.util.file_copy import copyfile
 
 def addtolist(mylist, curdir):
 	"""(list, dir) --- Takes an array(list) and appends all files from dir down
@@ -328,7 +329,7 @@ class tbz2(object):
 
 		if break_hardlinks and self.filestat and self.filestat.st_nlink > 1:
 			tmp_fname = "%s.%d" % (self.file, os.getpid())
-			shutil.copyfile(self.file, tmp_fname)
+			copyfile(self.file, tmp_fname)
 			try:
 				portage.util.apply_stat_permissions(self.file, self.filestat)
 			except portage.exception.OperationNotPermitted:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     895a7417f41c4f6617e22dc0ebf001de526fc666
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 25 05:13:43 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar 25 07:25:00 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=895a7417

_lockfile_iteration: index locked files by inode (bug 714480)

Index each locked file by inode so that it's possible to detect when
there's an attempt to lock the same inode twice, since in that case
the lock will not behave as intended with the default fcntl.lockf
function.

Bug: https://bugs.gentoo.org/714480
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 7bb0542dd..2459b12b6 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -84,7 +84,21 @@ def _get_lock_fn():
 	return _lock_fn
 
 
-_open_fds = set()
+_open_fds = {}
+_open_inodes = {}
+
+class _lock_manager(object):
+	__slots__ = ('fd', 'inode_key')
+	def __init__(self, fd, fstat_result):
+		self.fd = fd
+		self.inode_key = (fstat_result.st_dev, fstat_result.st_ino)
+		_open_fds[fd] = self
+		_open_inodes[self.inode_key] = self
+	def close(self):
+		os.close(self.fd)
+		del _open_fds[self.fd]
+		del _open_inodes[self.inode_key]
+
 
 def _close_fds():
 	"""
@@ -93,8 +107,8 @@ def _close_fds():
 	safely after a fork without exec, unlike the _setup_pipes close_fds
 	behavior.
 	"""
-	while _open_fds:
-		os.close(_open_fds.pop())
+	for fd in list(_open_fds.values()):
+		fd.close()
 
 def lockdir(mydir, flags=0):
 	return lockfile(mydir, wantnewlockfile=1, flags=flags)
@@ -296,6 +310,7 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False,
 		else:
 			raise
 
+	fstat_result = None
 	if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and unlinkfile:
 		try:
 			(removed, fstat_result) = _lockfile_was_removed(myfd, lockfilename)
@@ -321,7 +336,7 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False,
 				fcntl.fcntl(myfd, fcntl.F_SETFD,
 					fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
 
-		_open_fds.add(myfd)
+		_lock_manager(myfd, os.fstat(myfd) if fstat_result is None else fstat_result)
 
 	writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
 	return (lockfilename, myfd, unlinkfile, locking_method)
@@ -442,8 +457,7 @@ def unlockfile(mytuple):
 		not os.path.exists(lockfilename):
 		writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 		if myfd is not None:
-			os.close(myfd)
-			_open_fds.remove(myfd)
+			_open_fds[myfd].close()
 		return False
 
 	try:
@@ -453,8 +467,7 @@ def unlockfile(mytuple):
 		locking_method(myfd, fcntl.LOCK_UN)
 	except OSError:
 		if isinstance(lockfilename, basestring):
-			os.close(myfd)
-			_open_fds.remove(myfd)
+			_open_fds[myfd].close()
 		raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 
 	try:
@@ -475,8 +488,7 @@ def unlockfile(mytuple):
 				locking_method(myfd, fcntl.LOCK_UN)
 			else:
 				writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
-				os.close(myfd)
-				_open_fds.remove(myfd)
+				_open_fds[myfd].close()
 				return False
 	except SystemExit:
 		raise
@@ -488,8 +500,7 @@ def unlockfile(mytuple):
 	# fd originally, and the caller might not like having their
 	# open fd closed automatically on them.
 	if isinstance(lockfilename, basestring):
-		os.close(myfd)
-		_open_fds.remove(myfd)
+		_open_fds[myfd].close()
 
 	return True
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     59e9452fda79f69e9b5ab7e44ef918f6fb4d6161
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 25 05:11:06 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar 25 07:24:52 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=59e9452f

_lockfile_was_removed: return fstat result (bug 714480)

Return a tuple that includes the fstat result, which can be used to
detect when there's an attempt to lock the same inode more than once
by the same process, since in that case the lock will not behave as
intended with the default fcntl.lockf function.

Bug: https://bugs.gentoo.org/714480
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 72ac2fc70..7bb0542dd 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -296,10 +296,9 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False,
 		else:
 			raise
 
-		
 	if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and unlinkfile:
 		try:
-			removed = _lockfile_was_removed(myfd, lockfilename)
+			(removed, fstat_result) = _lockfile_was_removed(myfd, lockfilename)
 		except Exception:
 			# Do not leak the file descriptor here.
 			os.close(myfd)
@@ -341,14 +340,15 @@ def _lockfile_was_removed(lock_fd, lock_path):
 	@param lock_path: path of lock file
 	@type lock_path: str
 	@rtype: bool
-	@return: True if lock_path exists and corresponds to lock_fd, False otherwise
+	@return: a tuple of (removed, fstat_result), where removed is True if
+		lock_path does not correspond to lock_fd, and False otherwise
 	"""
 	try:
 		fstat_st = os.fstat(lock_fd)
 	except OSError as e:
 		if e.errno not in (errno.ENOENT, errno.ESTALE):
 			_raise_exc(e)
-		return True
+		return (True, None)
 
 	# Since stat is not reliable for removed files on NFS with the default
 	# file attribute cache behavior ('ac' mount option), create a temporary
@@ -365,7 +365,7 @@ def _lockfile_was_removed(lock_fd, lock_path):
 		except OSError as e:
 			if e.errno not in (errno.ENOENT, errno.ESTALE):
 				_raise_exc(e)
-			return True
+			return (True, None)
 
 		hardlink_stat = os.stat(hardlink_path)
 		if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
@@ -383,13 +383,13 @@ def _lockfile_was_removed(lock_fd, lock_path):
 			except OSError as e:
 				if e.errno not in (errno.ENOENT, errno.ESTALE):
 					_raise_exc(e)
-				return True
+				return (True, None)
 			else:
 				if not os.path.samefile(hardlink_path, inode_test):
 					# This implies that inode numbers are not expected
 					# to match for this file system, so use a simple
 					# stat call to detect if lock_path has been removed.
-					return not os.path.exists(lock_path)
+					return (not os.path.exists(lock_path), fstat_st)
 			finally:
 				try:
 					os.unlink(inode_test)
@@ -403,7 +403,7 @@ def _lockfile_was_removed(lock_fd, lock_path):
 		except OSError as e:
 			if e.errno not in (errno.ENOENT, errno.ESTALE):
 				_raise_exc(e)
-	return False
+	return (False, fstat_st)
 
 
 def _fstat_nlink(fd):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     2b3fc2be07ad897b7df6b77f79616c143a353275
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 25 19:15:23 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar 25 19:16:23 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2b3fc2be

_lockfile_was_removed: always return tuple

Fixes: 59e9452fda79 ("_lockfile_was_removed: return fstat result (bug 714480)")
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 71321e769..2331e6f57 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -419,7 +419,7 @@ def _lockfile_was_removed(lock_fd, lock_path):
 				except OSError as e:
 					if e.errno not in (errno.ENOENT, errno.ESTALE):
 						_raise_exc(e)
-			return True
+			return (True, None)
 	finally:
 		try:
 			os.unlink(hardlink_path)

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Michał Górny]

commit:     28ce410d2aa2eb33d0e61fbf272e1929b734622d
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 27 15:14:48 2020 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sat Mar 28 18:48:24 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=28ce410d

process: Unshare UTS namespace, and set hostname to 'localhost'

Use UTS namespace to override hostname when network-sandbox is enabled.
Set it to 'localhost' as that has a better chance of being present
in /etc/hosts.  This fixes tests in some packages that try to connect
to localhost via hostname obtained using gethostname(), e.g. docker-py,
and suffer resolution problems due to the system hostname not being
defined in /etc/hosts.

Closes: https://github.com/gentoo/portage/pull/539
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/process.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index c1fc4bcf6..590116890 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -348,12 +348,14 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 	if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
 		# from /usr/include/bits/sched.h
 		CLONE_NEWNS = 0x00020000
+		CLONE_NEWUTS = 0x04000000
 		CLONE_NEWIPC = 0x08000000
 		CLONE_NEWPID = 0x20000000
 		CLONE_NEWNET = 0x40000000
 
 		if unshare_net:
-			unshare_flags |= CLONE_NEWNET
+			# UTS namespace to override hostname
+			unshare_flags |= CLONE_NEWNET | CLONE_NEWUTS
 		if unshare_ipc:
 			unshare_flags |= CLONE_NEWIPC
 		if unshare_mount:
@@ -704,6 +706,13 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 									noiselevel=-1)
 								os._exit(1)
 						if unshare_net:
+							# use 'localhost' to avoid hostname resolution problems
+							try:
+								socket.sethostname('localhost')
+							except Exception as e:
+								writemsg("Unable to set hostname: %s (for FEATURES=\"network-sandbox\")\n" % (
+									e,),
+									noiselevel=-1)
 							_configure_loopback_interface()
 				except AttributeError:
 					# unshare() not supported by libc

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     9738a404e876270cbdef2514f66915bce35d7435
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 20 20:47:00 2020 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Mon Apr 20 21:14:25 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9738a404

Import portage.util.netlink.RtNetlink in global scope

Bug: https://bugs.gentoo.org/718578
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 590116890..79052b608 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -27,6 +27,13 @@ from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
 from portage.exception import CommandNotFound
 from portage.util._ctypes import find_library, LoadLibrary, ctypes
 
+try:
+	from portage.util.netlink import RtNetlink
+except ImportError:
+	if platform.system() == "Linux":
+		raise
+	RtNetlink = None
+
 try:
 	import resource
 	max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
@@ -504,8 +511,8 @@ def _configure_loopback_interface():
 	# Bug: https://bugs.gentoo.org/690758
 	# Bug: https://sourceware.org/bugzilla/show_bug.cgi?id=12377#c13
 
-	# Avoid importing this module on systems that may not support netlink sockets.
-	from portage.util.netlink import RtNetlink
+	if RtNetlink is None:
+		return
 
 	try:
 		with RtNetlink() as rtnl:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     def3574d3fe9b944dd83e561462ccc6de6f90ff3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May  7 20:32:03 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May  7 20:34:02 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=def3574d

locks: translate surrogate from uname (bug 721402)

Prevent an error like this when attempting to encode a surrogate:

UnicodeEncodeError: 'utf-8' codec can't encode characters in position 20-22: surrogates not allowed

Tested-by: Vladimir Varlamov <bes.internal <AT> gmail.com>
Bug: https://bugs.gentoo.org/721402
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 2331e6f57..535698dfe 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -1,5 +1,5 @@
 # portage: Lock management code
-# Copyright 2004-2019 Gentoo Authors
+# Copyright 2004-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
@@ -516,7 +516,7 @@ def unlockfile(mytuple):
 def hardlock_name(path):
 	base, tail = os.path.split(path)
 	return os.path.join(base, ".%s.hardlock-%s-%s" %
-		(tail, os.uname()[1], os.getpid()))
+		(tail, portage._decode_argv([os.uname()[1]])[0], os.getpid()))
 
 def hardlink_is_mine(link, lock):
 	try:
@@ -672,7 +672,7 @@ def unhardlink_lockfile(lockfilename, unlinkfile=True):
 		pass
 
 def hardlock_cleanup(path, remove_all_locks=False):
-	myhost = os.uname()[1]
+	myhost = portage._decode_argv([os.uname()[1]])[0]
 	mydl = os.listdir(path)
 
 	results = []

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Michał Górny]

commit:     c35f39af1ae591c9df50da52e83067428a10f80a
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sun May 17 08:55:22 2020 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sun May 17 09:37:45 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c35f39af

process: Provide libc fallback for sethostname() on PyPy

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/process.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 79052b608..ceb454030 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -715,7 +715,14 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 						if unshare_net:
 							# use 'localhost' to avoid hostname resolution problems
 							try:
-								socket.sethostname('localhost')
+								# pypy3 does not implement socket.sethostname()
+								new_hostname = b'localhost'
+								if hasattr(socket, 'sethostname'):
+									socket.sethostname(new_hostname)
+								else:
+									if libc.sethostname(new_hostname, len(new_hostname)) != 0:
+										errno_value = ctypes.get_errno()
+										raise OSError(errno_value, os.strerror(errno_value))
 							except Exception as e:
 								writemsg("Unable to set hostname: %s (for FEATURES=\"network-sandbox\")\n" % (
 									e,),

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     ffc70b10ddab2d9b74b60cf9036af0185ff17066
Author:     Kirill Chibisov <contact <AT> kchibisov <DOT> com>
AuthorDate: Mon Jun  8 17:07:12 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jun  9 00:57:58 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ffc70b10

xtermTitle: support alacritty

alacritty terminfo is included in ncurses-6.2 and so it is
picked up by default on most systems using alacritty terminal emulator,
which causes an issue of portage not updating window title.

Closes: https://github.com/gentoo/portage/pull/555
Signed-off-by: Kirill Chibisov <contact <AT> kchibisov.com>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/output.py b/lib/portage/output.py
index 1070d0ef3..cf799f362 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -234,7 +234,7 @@ def nc_len(mystr):
 	tmp = re.sub(esc_seq + "^m]+m", "", mystr);
 	return len(tmp)
 
-_legal_terms_re = re.compile(r'^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color)')
+_legal_terms_re = re.compile(r'^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color|alacritty)')
 _disable_xtermTitle = None
 _max_xtermTitle_len = 253
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     06c333b65b6dfe59828408c47ca0225f1c182081
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 27 19:42:12 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun 27 19:42:46 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=06c333b6

xtermTitle: support konsole

Bug: https://bugs.gentoo.org/729852
Reported-by: rnddim <AT> gmail.com
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/output.py b/lib/portage/output.py
index cf799f362..6fbb24f4c 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -234,7 +234,7 @@ def nc_len(mystr):
 	tmp = re.sub(esc_seq + "^m]+m", "", mystr);
 	return len(tmp)
 
-_legal_terms_re = re.compile(r'^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color|alacritty)')
+_legal_terms_re = re.compile(r'^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color|alacritty|konsole)')
 _disable_xtermTitle = None
 _max_xtermTitle_len = 253
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     859594ca1c547dac0e1e35c08a86f81ad7023848
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  3 19:05:55 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  3 19:22:09 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=859594ca

lib/portage/exception.py: drop unused-import

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/exception.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/exception.py b/lib/portage/exception.py
index 30ab0c689..1d7ccbe90 100644
--- a/lib/portage/exception.py
+++ b/lib/portage/exception.py
@@ -2,7 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import signal
-from portage import _encodings, _unicode_encode, _unicode_decode
+from portage import _encodings, _unicode_decode
 from portage.localization import _
 
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     728cff4873b3a9638119aa057f3c365c9cd622fe
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  3 19:06:00 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  3 19:22:27 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=728cff48

lib/portage/news.py: drop unused-import

* Drop unused-import
* Update copyright

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/news.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/portage/news.py b/lib/portage/news.py
index 2731ea889..21a7c6a1f 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -1,5 +1,5 @@
 # portage: news management code
-# Copyright 2006-2019 Gentoo Authors
+# Copyright 2006-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
@@ -16,7 +16,6 @@ import io
 import logging
 import os as _os
 import re
-import portage
 from portage import os
 from portage import _encodings
 from portage import _unicode_decode

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     79752101b23e21daae2f7f4099e5b6be5f2cffdd
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  3 19:06:02 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  3 19:22:33 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=79752101

lib/portage/_selinux.py: drop unused-import

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_selinux.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/_selinux.py b/lib/portage/_selinux.py
index aba40c2dd..e3e18c0b8 100644
--- a/lib/portage/_selinux.py
+++ b/lib/portage/_selinux.py
@@ -14,7 +14,7 @@ except ImportError:
 
 import portage
 from portage import _encodings
-from portage import _native_string, _unicode_decode
+from portage import _native_string
 from portage.localization import _
 
 def copyfile(src, dest):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     67f7a6990ef6d6383664491fb4b1604d20022589
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  3 19:06:01 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  3 19:22:30 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=67f7a699

lib/portage/mail.py: drop unused-import

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/mail.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/portage/mail.py b/lib/portage/mail.py
index df7aac52d..6503b4cc9 100644
--- a/lib/portage/mail.py
+++ b/lib/portage/mail.py
@@ -14,7 +14,6 @@ import socket
 import sys
 
 from portage import os
-from portage import _encodings
 from portage import _unicode_decode, _unicode_encode
 from portage.localization import _
 import portage

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     a8301a77eb0f19c18e3212c69fcba6e497e26208
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  3 22:43:18 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  3 23:28:03 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a8301a77

lib/portage/update.py: fix whitespace

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/update.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/update.py b/lib/portage/update.py
index 07e977452..f11b14217 100644
--- a/lib/portage/update.py
+++ b/lib/portage/update.py
@@ -179,7 +179,7 @@ def grab_updates(updpath, prev_mtimes=None):
 	mylist = [myfile for myfile in mylist if len(myfile) == 7 and myfile[1:3] == "Q-"]
 	if len(mylist) == 0:
 		return []
-	
+
 	# sort by (year, quarter)
 	mylist.sort(key=lambda x: (x[3:], x[:2]))
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     207d971d15b84bc54e710ab4b0780b8e1169a685
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  3 22:43:12 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  3 23:28:02 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=207d971d

lib/portage/checksum.py: fix whitespace

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/checksum.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index 8f01f6ac4..d4a75c058 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -1,5 +1,5 @@
 # checksum.py -- core Portage functionality
-# Copyright 1998-2017 Gentoo Foundation
+# Copyright 1998-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import portage
@@ -75,7 +75,7 @@ class _generate_hash_function:
 	def checksum_file(self, filename):
 		"""
 		Run a checksum against a file.
-	
+
 		@param filename: File to run the checksum against
 		@type filename: String
 		@return: The hash and size of the data

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     aefc24179b63c5e2528c12cd964a3556fb27cb78
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Tue Aug  4 00:16:16 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Aug  4 01:08:14 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=aefc2417

lib/portage/output.py: fix unnecessary-semicolon

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/output.py b/lib/portage/output.py
index ab99caf98..098285a52 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -231,7 +231,7 @@ def _parse_color_map(config_root='/', onerror=None):
 		raise
 
 def nc_len(mystr):
-	tmp = re.sub(esc_seq + "^m]+m", "", mystr);
+	tmp = re.sub(esc_seq + "^m]+m", "", mystr)
 	return len(tmp)
 
 _legal_terms_re = re.compile(r'^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color|alacritty|konsole)')

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     a8f0e05d35b0ba2747827ce03dff42682192def8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Sep  9 05:08:20 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Sep 11 19:00:57 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a8f0e05d

selinux: encode os path arguments as UTF-8 (bug 741194)

Encode path arguments as UTF-8, like portage.os wrapper.

Fixes: 6137290b2bb8 ("selinux: python3 unicode paths, bug #430488")
Tested-by: Bob Gleitsmann <rjgleits <AT> bellsouth.net>
Bug: https://bugs.gentoo.org/741194
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_selinux.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/portage/_selinux.py b/lib/portage/_selinux.py
index e3e18c0b8..2423ff8d6 100644
--- a/lib/portage/_selinux.py
+++ b/lib/portage/_selinux.py
@@ -14,7 +14,7 @@ except ImportError:
 
 import portage
 from portage import _encodings
-from portage import _native_string
+from portage import _native_string, _unicode_encode
 from portage.localization import _
 
 def copyfile(src, dest):
@@ -41,7 +41,6 @@ def is_selinux_enabled():
 	return selinux.is_selinux_enabled()
 
 def mkdir(target, refdir):
-	target = _native_string(target, encoding=_encodings['fs'], errors='strict')
 	refdir = _native_string(refdir, encoding=_encodings['fs'], errors='strict')
 	(rc, ctx) = selinux.getfilecon(refdir)
 	if rc < 0:
@@ -51,20 +50,21 @@ def mkdir(target, refdir):
 
 	setfscreate(ctx)
 	try:
-		os.mkdir(target)
+		os.mkdir(_unicode_encode(target, encoding=_encodings['fs'], errors='strict'))
 	finally:
 		setfscreate()
 
 def rename(src, dest):
 	src = _native_string(src, encoding=_encodings['fs'], errors='strict')
-	dest = _native_string(dest, encoding=_encodings['fs'], errors='strict')
 	(rc, ctx) = selinux.lgetfilecon(src)
 	if rc < 0:
 		raise OSError(_("rename: Failed getting context of \"%s\".") % src)
 
 	setfscreate(ctx)
 	try:
-		os.rename(src, dest)
+		os.rename(
+			_unicode_encode(src, encoding=_encodings['fs'], errors='strict'),
+			_unicode_encode(dest, encoding=_encodings['fs'], errors='strict'))
 	finally:
 		setfscreate()
 
@@ -132,8 +132,6 @@ class spawn_wrapper:
 		return self._spawn_func(*args, **kwargs)
 
 def symlink(target, link, reflnk):
-	target = _native_string(target, encoding=_encodings['fs'], errors='strict')
-	link = _native_string(link, encoding=_encodings['fs'], errors='strict')
 	reflnk = _native_string(reflnk, encoding=_encodings['fs'], errors='strict')
 	(rc, ctx) = selinux.lgetfilecon(reflnk)
 	if rc < 0:
@@ -143,6 +141,8 @@ def symlink(target, link, reflnk):
 
 	setfscreate(ctx)
 	try:
-		os.symlink(target, link)
+		os.symlink(
+			_unicode_encode(target, encoding=_encodings['fs'], errors='strict'),
+			_unicode_encode(link, encoding=_encodings['fs'], errors='strict'))
 	finally:
 		setfscreate()

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     ef43dd8ef64ded5f0627b7b8e84ef21262012902
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jan 30 12:44:19 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb  8 04:47:20 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ef43dd8e

portage.getpid: call os.getpid() lazily

Call os.getpid() lazily, which eliminates getpid calls when possible
after os.fork() in the portage.process module.

Bug: https://bugs.gentoo.org/767913
Reviewed-by: Brian Dolbec <dolsen <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 3c9f78497..24c9d8b89 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -375,7 +375,7 @@ _sync_mode = False
 class _ForkWatcher:
 	@staticmethod
 	def hook(_ForkWatcher):
-		_ForkWatcher.current_pid = _os.getpid()
+		_ForkWatcher.current_pid = None
 		# Force instantiation of a new event loop policy as a workaround
 		# for https://bugs.python.org/issue22087.
 		asyncio.set_event_loop_policy(None)
@@ -388,6 +388,8 @@ def getpid():
 	"""
 	Cached version of os.getpid(). ForkProcess updates the cache.
 	"""
+	if _ForkWatcher.current_pid is None:
+		_ForkWatcher.current_pid = _os.getpid()
 	return _ForkWatcher.current_pid
 
 def _get_stdin():

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     3741b4f7d7b901bc7d69bd58ce1b92b0536427bb
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar  6 06:10:58 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar  6 06:14:38 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3741b4f7

_get_lock_fn: split out _test_lock_fn

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 1073343be..163c92482 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -1,5 +1,5 @@
 # portage: Lock management code
-# Copyright 2004-2020 Gentoo Authors
+# Copyright 2004-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
@@ -12,6 +12,7 @@ import multiprocessing
 import sys
 import tempfile
 import time
+import typing
 import warnings
 
 import portage
@@ -44,11 +45,21 @@ def _get_lock_fn():
 	if _lock_fn is not None:
 		return _lock_fn
 
+	if _test_lock_fn(fcntl.lockf):
+		_lock_fn = fcntl.lockf
+		return _lock_fn
+
+	# Fall back to fcntl.flock.
+	_lock_fn = fcntl.flock
+	return _lock_fn
+
+
+def _test_lock_fn(lock_fn: typing.Callable[[int, int], None]) -> bool:
 	def _test_lock(fd, lock_path):
 		os.close(fd)
 		try:
 			with open(lock_path, 'a') as f:
-				fcntl.lockf(f.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
+				lock_fn(f.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
 		except EnvironmentError as e:
 			if e.errno == errno.EAGAIN:
 				# Parent process holds lock, as expected.
@@ -60,7 +71,7 @@ def _get_lock_fn():
 	fd, lock_path = tempfile.mkstemp()
 	try:
 		try:
-			fcntl.lockf(fd, fcntl.LOCK_EX)
+			lock_fn(fd, fcntl.LOCK_EX)
 		except EnvironmentError:
 			pass
 		else:
@@ -69,16 +80,12 @@ def _get_lock_fn():
 			proc.start()
 			proc.join()
 			if proc.exitcode == os.EX_OK:
-				# Use fcntl.lockf because the test passed.
-				_lock_fn = fcntl.lockf
-				return _lock_fn
+				# the test passed
+				return True
 	finally:
 		os.close(fd)
 		os.unlink(lock_path)
-
-	# Fall back to fcntl.flock.
-	_lock_fn = fcntl.flock
-	return _lock_fn
+	return False
 
 
 _open_fds = {}

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     cd2620f41af5910d0ed3f541d2ac43e94a8c11f9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar  6 08:04:55 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar  6 08:17:41 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=cd2620f4

_test_lock_fn: extend lock_fn to return unlock function

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 163c92482..4d0eede50 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -8,6 +8,7 @@ __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
 
 import errno
 import fcntl
+import functools
 import multiprocessing
 import sys
 import tempfile
@@ -45,7 +46,11 @@ def _get_lock_fn():
 	if _lock_fn is not None:
 		return _lock_fn
 
-	if _test_lock_fn(fcntl.lockf):
+	if _test_lock_fn(
+		lambda path, fd, flags: functools.partial(
+			unlockfile, (path, fcntl.lockf(fd, flags), flags, fcntl.lockf)
+		)
+	):
 		_lock_fn = fcntl.lockf
 		return _lock_fn
 
@@ -54,14 +59,14 @@ def _get_lock_fn():
 	return _lock_fn
 
 
-def _test_lock_fn(lock_fn: typing.Callable[[int, int], None]) -> bool:
+def _test_lock_fn(lock_fn: typing.Callable[[str, int, int], typing.Callable[[], None]]) -> bool:
 	def _test_lock(fd, lock_path):
 		os.close(fd)
 		try:
 			with open(lock_path, 'a') as f:
-				lock_fn(f.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
-		except EnvironmentError as e:
-			if e.errno == errno.EAGAIN:
+				lock_fn(lock_path, f.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
+		except (TryAgain, EnvironmentError) as e:
+			if isinstance(e, TryAgain) or e.errno == errno.EAGAIN:
 				# Parent process holds lock, as expected.
 				sys.exit(0)
 
@@ -71,8 +76,8 @@ def _test_lock_fn(lock_fn: typing.Callable[[int, int], None]) -> bool:
 	fd, lock_path = tempfile.mkstemp()
 	try:
 		try:
-			lock_fn(fd, fcntl.LOCK_EX)
-		except EnvironmentError:
+			lock_fn(lock_path, fd, fcntl.LOCK_EX)
+		except (TryAgain, EnvironmentError):
 			pass
 		else:
 			proc = multiprocessing.Process(target=_test_lock,

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     af100c65ebf7fd84307a84819602a934ebb0741c
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  4 19:23:46 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar  6 09:03:28 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=af100c65

Remove outdated mention of Python 2 from comment

Fixes: 5e9fe0f2a ("Eliminate basestring/long/_unicode py3 compat")
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/versions.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/portage/versions.py b/lib/portage/versions.py
index 317683b17..1dc942124 100644
--- a/lib/portage/versions.py
+++ b/lib/portage/versions.py
@@ -341,11 +341,11 @@ def catpkgsplit(mydata, silent=1, eapi=None):
 
 class _pkg_str(str):
 	"""
-	This class represents a cpv. It inherits from str (unicode in python2) and
-	has attributes that cache results for use by functions like catpkgsplit and
-	cpv_getkey which are called frequently (especially in match_from_list).
-	Instances are typically created in dbapi.cp_list() or the Atom contructor,
-	and propagate from there. Generally, code that pickles these objects will
+	This class represents a cpv. It inherits from str and has attributes
+	that cache results for use by functions like catpkgsplit and cpv_getkey
+	which are called frequently (especially in match_from_list).  Instances
+	are typically created in dbapi.cp_list() or the Atom contructor, and
+	propagate from there. Generally, code that pickles these objects will
 	manually convert them to a plain unicode object first.
 
 	Instances of this class will have missing attributes for metadata that

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     9003c5201c6503ddad9237bcffbc6f775567661b
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  4 19:23:45 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar  6 09:01:38 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9003c520

Remove Python 2 workaround

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/__init__.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 24c9d8b89..d33792a77 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -484,11 +484,7 @@ def _eapi_is_deprecated(eapi):
 	return eapi in _deprecated_eapis
 
 def eapi_is_supported(eapi):
-	if not isinstance(eapi, str):
-		# Only call str() when necessary since with python2 it
-		# can trigger UnicodeEncodeError if EAPI is corrupt.
-		eapi = str(eapi)
-	eapi = eapi.strip()
+	eapi = str(eapi).strip()
 
 	return eapi in _supported_eapis
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     67cf9c2b05042de37f36f5b6840c450128a065bd
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  4 19:35:17 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar  6 09:17:52 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=67cf9c2b

Mark EAPIs "4-python" and "5-progress" as deprecated

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/__init__.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index d33792a77..1d202f557 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -465,16 +465,16 @@ def abssymlink(symlink, target=None):
 _doebuild_manifest_exempt_depend = 0
 
 _testing_eapis = frozenset([
-	"4-python",
-	"5-progress",
 ])
 _deprecated_eapis = frozenset([
+	"3_pre1",
+	"3_pre2",
 	"4_pre1",
+	"4-python",
 	"4-slot-abi",
-	"3_pre2",
-	"3_pre1",
 	"5_pre1",
 	"5_pre2",
+	"5-progress",
 	"6_pre1",
 	"7_pre1",
 ])

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     425aa9fb43e948e900478be3f47b799c0da6820f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar  7 13:48:41 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar  7 14:02:40 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=425aa9fb

_test_lock_fn: use _lock_manager and call unlock

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 59 ++++++++++++++++++++++++++--------------------------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 4d0eede50..ddce72f62 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -35,6 +35,28 @@ _quiet = False
 
 
 _lock_fn = None
+_open_fds = {}
+_open_inodes = {}
+
+class _lock_manager:
+	__slots__ = ('fd', 'inode_key')
+	def __init__(self, fd, fstat_result, path):
+		self.fd = fd
+		self.inode_key = (fstat_result.st_dev, fstat_result.st_ino)
+		if self.inode_key in _open_inodes:
+			# This means that the lock is already held by the current
+			# process, so the caller will have to try again. This case
+			# is encountered with the default fcntl.lockf function, and
+			# with the alternative fcntl.flock function TryAgain is
+			# raised earlier.
+			os.close(fd)
+			raise TryAgain(path)
+		_open_fds[fd] = self
+		_open_inodes[self.inode_key] = self
+	def close(self):
+		os.close(self.fd)
+		del _open_fds[self.fd]
+		del _open_inodes[self.inode_key]
 
 
 def _get_lock_fn():
@@ -47,8 +69,8 @@ def _get_lock_fn():
 		return _lock_fn
 
 	if _test_lock_fn(
-		lambda path, fd, flags: functools.partial(
-			unlockfile, (path, fcntl.lockf(fd, flags), flags, fcntl.lockf)
+		lambda path, fd, flags: fcntl.lockf(fd, flags) and functools.partial(
+			unlockfile, (path, fd, flags, fcntl.lockf)
 		)
 	):
 		_lock_fn = fcntl.lockf
@@ -70,16 +92,19 @@ def _test_lock_fn(lock_fn: typing.Callable[[str, int, int], typing.Callable[[],
 				# Parent process holds lock, as expected.
 				sys.exit(0)
 
+
 		# Something went wrong.
 		sys.exit(1)
 
 	fd, lock_path = tempfile.mkstemp()
+	unlock_fn = None
 	try:
 		try:
-			lock_fn(lock_path, fd, fcntl.LOCK_EX)
+			unlock_fn = lock_fn(lock_path, fd, fcntl.LOCK_EX)
 		except (TryAgain, EnvironmentError):
 			pass
 		else:
+			_lock_manager(fd, os.fstat(fd), lock_path)
 			proc = multiprocessing.Process(target=_test_lock,
 				args=(fd, lock_path))
 			proc.start()
@@ -88,35 +113,11 @@ def _test_lock_fn(lock_fn: typing.Callable[[str, int, int], typing.Callable[[],
 				# the test passed
 				return True
 	finally:
-		os.close(fd)
-		os.unlink(lock_path)
+		if unlock_fn is not None:
+			unlock_fn()
 	return False
 
 
-_open_fds = {}
-_open_inodes = {}
-
-class _lock_manager:
-	__slots__ = ('fd', 'inode_key')
-	def __init__(self, fd, fstat_result, path):
-		self.fd = fd
-		self.inode_key = (fstat_result.st_dev, fstat_result.st_ino)
-		if self.inode_key in _open_inodes:
-			# This means that the lock is already held by the current
-			# process, so the caller will have to try again. This case
-			# is encountered with the default fcntl.lockf function, and
-			# with the alternative fcntl.flock function TryAgain is
-			# raised earlier.
-			os.close(fd)
-			raise TryAgain(path)
-		_open_fds[fd] = self
-		_open_inodes[self.inode_key] = self
-	def close(self):
-		os.close(self.fd)
-		del _open_fds[self.fd]
-		del _open_inodes[self.inode_key]
-
-
 def _close_fds():
 	"""
 	This is intended to be called after a fork, in order to close file

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     d8a12c258cbe15b2725c51b1061e8297919b20df
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 11 12:28:15 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Mar 11 12:29:14 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8a12c25

_test_lock_fn: remove temp file (bug 775239)

Bug: https://bugs.gentoo.org/775239
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index ddce72f62..d0218d7bc 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -113,6 +113,10 @@ def _test_lock_fn(lock_fn: typing.Callable[[str, int, int], typing.Callable[[],
 				# the test passed
 				return True
 	finally:
+		try:
+			os.unlink(lock_path)
+		except OSError:
+			pass
 		if unlock_fn is not None:
 			unlock_fn()
 	return False

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     6c1000332491de2afe4d52ee7a380e5b2da2ddb8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 28 02:11:59 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar 28 03:16:56 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c100033

_get_eapi_attrs: cache eapi_* function calls via decorator

Decorate eapi_* functions so that calls use _get_eapi_attrs(eapi)
to cache results.

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/eapi.py | 107 +++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 81 insertions(+), 26 deletions(-)

diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py
index aca571ebd..796184644 100644
--- a/lib/portage/eapi.py
+++ b/lib/portage/eapi.py
@@ -1,7 +1,9 @@
-# Copyright 2010-2018 Gentoo Foundation
+# Copyright 2010-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import collections
+import operator
+import types
 
 from portage import eapi_is_supported
 
@@ -127,15 +129,63 @@ def eapi_has_sysroot(eapi):
 
 _eapi_attrs = collections.namedtuple('_eapi_attrs',
 	'allows_package_provided '
-	'bdepend broot dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC '
-	'exports_PORTDIR exports_ECLASSDIR '
+	'bdepend '
+	'broot '
+	'dots_in_PN dots_in_use_flags '
+	'exports_AA '
+	'exports_EBUILD_PHASE_FUNC '
+	'exports_ECLASSDIR '
+	'exports_KV '
+	'exports_merge_type '
+	'exports_PORTDIR '
+	'exports_replace_vars '
 	'feature_flag_test '
 	'iuse_defaults iuse_effective posixish_locale '
 	'path_variables_end_with_trailing_slash '
+	'prefix '
 	'repo_deps required_use required_use_at_most_one_of slot_operator slot_deps '
 	'src_uri_arrows strong_blocks use_deps use_dep_defaults '
 	'empty_groups_always_true sysroot')
 
+
+_eapi_attr_func_prefixes = (
+	'eapi_allows_',
+	'eapi_has_',
+	'eapi_requires_',
+	'eapi_supports_',
+	'eapi_',
+)
+
+
+def _eapi_func_decorator(func, attr_getter):
+	def wrapper(eapi):
+		return attr_getter(_get_eapi_attrs(eapi))
+	wrapper.func = func
+	wrapper.__doc__ = func.__doc__
+	return wrapper
+
+
+def _decorate_eapi_funcs():
+	"""
+	Decorate eapi_* functions so that they use _get_eapi_attrs(eapi)
+	to cache results.
+	"""
+	decorated = {}
+	for k, v in globals().items():
+		if not (isinstance(v, types.FunctionType) and k.startswith(_eapi_attr_func_prefixes)):
+			continue
+		for prefix in _eapi_attr_func_prefixes:
+			if k.startswith(prefix):
+				attr_name = k[len(prefix):]
+				if hasattr(_eapi_attrs, attr_name):
+					decorated[k] = _eapi_func_decorator(v, operator.attrgetter(attr_name))
+					break
+	globals().update(decorated)
+
+
+_decorate_eapi_funcs()
+
+
 _eapi_attrs_cache = {}
 
 def _get_eapi_attrs(eapi):
@@ -155,31 +205,36 @@ def _get_eapi_attrs(eapi):
 		eapi = None
 
 	eapi_attrs = _eapi_attrs(
-		allows_package_provided=(eapi is None or eapi_allows_package_provided(eapi)),
-		bdepend = (eapi is not None and eapi_has_bdepend(eapi)),
-		broot = (eapi is None or eapi_has_broot(eapi)),
-		dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)),
-		dots_in_use_flags = (eapi is None or eapi_allows_dots_in_use_flags(eapi)),
-		empty_groups_always_true = (eapi is not None and eapi_empty_groups_always_true(eapi)),
-		exports_EBUILD_PHASE_FUNC = (eapi is None or eapi_exports_EBUILD_PHASE_FUNC(eapi)),
-		exports_PORTDIR = (eapi is None or eapi_exports_PORTDIR(eapi)),
-		exports_ECLASSDIR = (eapi is not None and eapi_exports_ECLASSDIR(eapi)),
+		allows_package_provided=(eapi is None or eapi_allows_package_provided.func(eapi)),
+		bdepend = (eapi is not None and eapi_has_bdepend.func(eapi)),
+		broot = (eapi is None or eapi_has_broot.func(eapi)),
+		dots_in_PN = (eapi is None or eapi_allows_dots_in_PN.func(eapi)),
+		dots_in_use_flags = (eapi is None or eapi_allows_dots_in_use_flags.func(eapi)),
+		empty_groups_always_true = (eapi is not None and eapi_empty_groups_always_true.func(eapi)),
+		exports_AA = (eapi is not None and eapi_exports_AA.func(eapi)),
+		exports_EBUILD_PHASE_FUNC = (eapi is None or eapi_exports_EBUILD_PHASE_FUNC.func(eapi)),
+		exports_ECLASSDIR = (eapi is not None and eapi_exports_ECLASSDIR.func(eapi)),
+		exports_KV = (eapi is not None and eapi_exports_KV.func(eapi)),
+		exports_merge_type = (eapi is None or eapi_exports_merge_type.func(eapi)),
+		exports_PORTDIR = (eapi is None or eapi_exports_PORTDIR.func(eapi)),
+		exports_replace_vars = (eapi is None or eapi_exports_replace_vars.func(eapi)),
 		feature_flag_test = False,
-		iuse_defaults = (eapi is None or eapi_has_iuse_defaults(eapi)),
-		iuse_effective = (eapi is not None and eapi_has_iuse_effective(eapi)),
+		iuse_defaults = (eapi is None or eapi_has_iuse_defaults.func(eapi)),
+		iuse_effective = (eapi is not None and eapi_has_iuse_effective.func(eapi)),
 		path_variables_end_with_trailing_slash = (eapi is not None and
-			eapi_path_variables_end_with_trailing_slash(eapi)),
-		posixish_locale = (eapi is not None and eapi_requires_posixish_locale(eapi)),
-		repo_deps = (eapi is None or eapi_has_repo_deps(eapi)),
-		required_use = (eapi is None or eapi_has_required_use(eapi)),
-		required_use_at_most_one_of = (eapi is None or eapi_has_required_use_at_most_one_of(eapi)),
-		slot_deps = (eapi is None or eapi_has_slot_deps(eapi)),
-		slot_operator = (eapi is None or eapi_has_slot_operator(eapi)),
-		src_uri_arrows = (eapi is None or eapi_has_src_uri_arrows(eapi)),
-		strong_blocks = (eapi is None or eapi_has_strong_blocks(eapi)),
-		sysroot = (eapi is None or eapi_has_sysroot(eapi)),
-		use_deps = (eapi is None or eapi_has_use_deps(eapi)),
-		use_dep_defaults = (eapi is None or eapi_has_use_dep_defaults(eapi))
+			eapi_path_variables_end_with_trailing_slash.func(eapi)),
+		posixish_locale = (eapi is not None and eapi_requires_posixish_locale.func(eapi)),
+		prefix = (eapi is None or eapi_supports_prefix.func(eapi)),
+		repo_deps = (eapi is None or eapi_has_repo_deps.func(eapi)),
+		required_use = (eapi is None or eapi_has_required_use.func(eapi)),
+		required_use_at_most_one_of = (eapi is None or eapi_has_required_use_at_most_one_of.func(eapi)),
+		slot_deps = (eapi is None or eapi_has_slot_deps.func(eapi)),
+		slot_operator = (eapi is None or eapi_has_slot_operator.func(eapi)),
+		src_uri_arrows = (eapi is None or eapi_has_src_uri_arrows.func(eapi)),
+		strong_blocks = (eapi is None or eapi_has_strong_blocks.func(eapi)),
+		sysroot = (eapi is None or eapi_has_sysroot.func(eapi)),
+		use_deps = (eapi is None or eapi_has_use_deps.func(eapi)),
+		use_dep_defaults = (eapi is None or eapi_has_use_dep_defaults.func(eapi))
 	)
 
 	_eapi_attrs_cache[orig_eapi] = eapi_attrs

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     09d783eb12ed1f93550d86f1ee2437012ffc7cc2
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Wed May 12 16:10:10 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 24 04:40:21 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=09d783eb

Relax update filenames as permitted for EAPI 8

Bug: https://bugs.gentoo.org/692774
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/update.py | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/lib/portage/update.py b/lib/portage/update.py
index f11b14217..67d3c08b0 100644
--- a/lib/portage/update.py
+++ b/lib/portage/update.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -175,21 +175,16 @@ def grab_updates(updpath, prev_mtimes=None):
 		raise
 	if prev_mtimes is None:
 		prev_mtimes = {}
-	# validate the file name (filter out CVS directory, etc...)
-	mylist = [myfile for myfile in mylist if len(myfile) == 7 and myfile[1:3] == "Q-"]
-	if len(mylist) == 0:
-		return []
-
-	# sort by (year, quarter)
-	mylist.sort(key=lambda x: (x[3:], x[:2]))
 
 	update_data = []
 	for myfile in mylist:
+		if myfile.startswith("."):
+			continue
 		file_path = os.path.join(updpath, myfile)
 		mystat = os.stat(file_path)
-		if update_data or \
-			file_path not in prev_mtimes or \
-			int(prev_mtimes[file_path]) != mystat[stat.ST_MTIME]:
+		if not stat.S_ISREG(mystat.st_mode):
+			continue
+		if int(prev_mtimes.get(file_path, -1)) != mystat[stat.ST_MTIME]:
 			f = io.open(_unicode_encode(file_path,
 				encoding=_encodings['fs'], errors='strict'),
 				mode='r', encoding=_encodings['repo.content'], errors='replace')

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     990e8e2c31b8fbd8c3ff0bdf90fcabf2f473bf73
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Mon May 17 19:55:12 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 24 06:08:20 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=990e8e2c

Enable EAPI 8

Closes: https://github.com/gentoo/portage/pull/718
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index 76fea8293..896771e14 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -210,7 +210,7 @@ SUPPORTED_FEATURES       = frozenset([
 	"xattr",
 ])
 
-EAPI                     = 7
+EAPI                     = 8
 
 HASHING_BLOCKSIZE        = 32768
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Michał Górny]

commit:     97f1f6a42796c91bf1eba628605c4ef641dcc532
Author:     James Stevenson <james.al.stevenson <AT> gmail <DOT> com>
AuthorDate: Sat Sep  4 09:53:46 2021 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sat Sep  4 11:51:28 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=97f1f6a4

Warn if portage home directory is missing

Signed-off-by: James Stevenson <james.al.stevenson <AT> gmail.com>
Bug: https://bugs.gentoo.org/674212
Closes: https://github.com/gentoo/portage/pull/736
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/data.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/portage/data.py b/lib/portage/data.py
index 3887ad32e..21f350934 100644
--- a/lib/portage/data.py
+++ b/lib/portage/data.py
@@ -178,6 +178,15 @@ def _get_global(k):
 				noiselevel=-1)
 			portage_group_warning()
 
+		portage_homedir_ready = os.access(os.path.expanduser("~portage"),
+			os.R_OK | os.W_OK | os.X_OK)
+		if not portage_homedir_ready:
+			writemsg(
+				colorize("BAD",
+				_("portage: unable to utilise 'portage' user home directory."
+					+ " Ensure that it exists with the correct permissions."))
+				+ "\n", noiselevel=-1)
+
 		globals()['portage_gid'] = portage_gid
 		_initialized_globals.add('portage_gid')
 		globals()['portage_uid'] = portage_uid

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Michał Górny]

commit:     3ccfd9a375d34b2c2731064ea4bc9e1a4b8946f3
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Tue Sep  7 07:03:35 2021 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Tue Sep  7 07:03:35 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3ccfd9a3

Revert "Warn if portage home directory is missing"

The check is executed in non-build contexts where the home directory
is irrelevant.

Reverts: 97f1f6a42796c91bf1eba628605c4ef641dcc532
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/data.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/lib/portage/data.py b/lib/portage/data.py
index 21f350934..3887ad32e 100644
--- a/lib/portage/data.py
+++ b/lib/portage/data.py
@@ -178,15 +178,6 @@ def _get_global(k):
 				noiselevel=-1)
 			portage_group_warning()
 
-		portage_homedir_ready = os.access(os.path.expanduser("~portage"),
-			os.R_OK | os.W_OK | os.X_OK)
-		if not portage_homedir_ready:
-			writemsg(
-				colorize("BAD",
-				_("portage: unable to utilise 'portage' user home directory."
-					+ " Ensure that it exists with the correct permissions."))
-				+ "\n", noiselevel=-1)
-
 		globals()['portage_gid'] = portage_gid
 		_initialized_globals.add('portage_gid')
 		globals()['portage_uid'] = portage_uid

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     65d90714ab5dd00fa162695181958749dbddb711
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Mon Sep 20 19:53:05 2021 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Mon Sep 20 19:53:05 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=65d90714

Define PORTAGE_BASE_PATH on a single line

This is required for the mangling code in setup.py to work properly.

Closes: https://bugs.gentoo.org/814113
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/const.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index f6845b806..e46f47892 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -60,9 +60,7 @@ GLOBAL_CONFIG_PATH = "/usr/share/portage/config"
 # NOTE: Use realpath(__file__) so that python module symlinks in site-packages
 # are followed back to the real location of the whole portage installation.
 # NOTE: Please keep PORTAGE_BASE_PATH in one line to help substitutions.
-PORTAGE_BASE_PATH = os.path.join(
-    os.sep, os.sep.join(os.path.realpath(__file__.rstrip("co")).split(os.sep)[:-3])
-)
+PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath(__file__.rstrip("co")).split(os.sep)[:-3]))
 PORTAGE_BIN_PATH = PORTAGE_BASE_PATH + "/bin"
 PORTAGE_PYM_PATH = os.path.realpath(os.path.join(__file__, "../.."))
 LOCALE_DATA_PATH = PORTAGE_BASE_PATH + "/locale"  # FIXME: not used

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     016d5a48d230123f662f3da4ae7d900d365582aa
Author:     Wolfgang E. Sanyer <WolfgangESanyer <AT> gmail <DOT> com>
AuthorDate: Mon Sep 20 18:11:07 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Sep 20 20:05:49 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=016d5a48

Fix installation issue related to black formatting and regex

Bug: https://bugs.gentoo.org/814113
Closes: https://github.com/gentoo/portage/pull/749
Signed-off-by: Wolfgang E. Sanyer <WolfgangESanyer <AT> gmail.com>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/const.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index e46f47892..abe0ef6c6 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -60,7 +60,9 @@ GLOBAL_CONFIG_PATH = "/usr/share/portage/config"
 # NOTE: Use realpath(__file__) so that python module symlinks in site-packages
 # are followed back to the real location of the whole portage installation.
 # NOTE: Please keep PORTAGE_BASE_PATH in one line to help substitutions.
+# fmt:off
 PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath(__file__.rstrip("co")).split(os.sep)[:-3]))
+# fmt:on
 PORTAGE_BIN_PATH = PORTAGE_BASE_PATH + "/bin"
 PORTAGE_PYM_PATH = os.path.realpath(os.path.join(__file__, "../.."))
 LOCALE_DATA_PATH = PORTAGE_BASE_PATH + "/locale"  # FIXME: not used

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     02582b22f05e5506e50907dba4f9ffbc128121f7
Author:     Stanislav Ochotnický <sochotnicky <AT> redhat <DOT> com>
AuthorDate: Fri Sep 24 08:00:00 2021 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Feb  9 11:13:41 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=02582b22

xtermTitle: support foot terminal

Foot supports title updates so let's dynamically update the title when using it.
Closes: https://github.com/gentoo/portage/pull/753

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/output.py b/lib/portage/output.py
index 77375b012..33c477012 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -270,7 +270,7 @@ def nc_len(mystr):
 
 
 _legal_terms_re = re.compile(
-    r"^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color|alacritty|konsole)"
+    r"^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color|alacritty|konsole|foot)"
 )
 _disable_xtermTitle = None
 _max_xtermTitle_len = 253

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Matt Turner]

commit:     e4afca3ed07f0e11f9918e1560074d7e5550be53
Author:     Wolfgang E. Sanyer <WolfgangESanyer <AT> gmail <DOT> com>
AuthorDate: Fri Sep 24 16:09:55 2021 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Tue Mar 15 02:52:10 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e4afca3e

portage.eapi: use tuple instead of str for namedtuple definition

Reviewed-by: Matt Turner <mattst88 <AT> gentoo.org>
Signed-off-by: Wolfgang E. Sanyer <WolfgangESanyer <AT> gmail.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 lib/portage/eapi.py | 52 +++++++++++++++++++++++++++++++++-------------------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py
index adee87d00..18069b04b 100644
--- a/lib/portage/eapi.py
+++ b/lib/portage/eapi.py
@@ -288,25 +288,39 @@ def eapi_has_sysroot(eapi):
 
 _eapi_attrs = collections.namedtuple(
     "_eapi_attrs",
-    "allows_package_provided "
-    "bdepend "
-    "broot "
-    "dots_in_PN dots_in_use_flags "
-    "exports_AA "
-    "exports_EBUILD_PHASE_FUNC "
-    "exports_ECLASSDIR "
-    "exports_KV "
-    "exports_merge_type "
-    "exports_PORTDIR "
-    "exports_replace_vars "
-    "feature_flag_test "
-    "idepend iuse_defaults iuse_effective posixish_locale "
-    "path_variables_end_with_trailing_slash "
-    "prefix "
-    "repo_deps required_use required_use_at_most_one_of "
-    "selective_src_uri_restriction slot_operator slot_deps "
-    "src_uri_arrows strong_blocks use_deps use_dep_defaults "
-    "empty_groups_always_true sysroot",
+    (
+        "allows_package_provided",
+        "bdepend",
+        "broot",
+        "dots_in_PN",
+        "dots_in_use_flags",
+        "exports_AA",
+        "exports_EBUILD_PHASE_FUNC",
+        "exports_ECLASSDIR",
+        "exports_KV",
+        "exports_merge_type",
+        "exports_PORTDIR",
+        "exports_replace_vars",
+        "feature_flag_test",
+        "idepend",
+        "iuse_defaults",
+        "iuse_effective",
+        "posixish_locale",
+        "path_variables_end_with_trailing_slash",
+        "prefix",
+        "repo_deps",
+        "required_use",
+        "required_use_at_most_one_of",
+        "selective_src_uri_restriction",
+        "slot_operator",
+        "slot_deps",
+        "src_uri_arrows",
+        "strong_blocks",
+        "use_deps",
+        "use_dep_defaults",
+        "empty_groups_always_true",
+        "sysroot",
+    ),
 )
 
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     6fb8ee74b3f334a7e2d82ce674a4b98e49f78b06
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Tue Mar 22 07:54:21 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 23:06:43 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6fb8ee74

Use dict expressions instead of looping or dict()

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py     | 10 ++++------
 lib/portage/eclass_cache.py |  5 +----
 lib/portage/getbinpkg.py    | 22 +++++++++++++---------
 3 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index d7e800c47..72d20525a 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -367,9 +367,7 @@ def _perform_md5_merge(x, **kwargs):
 
 
 def perform_all(x, calc_prelink=0):
-    mydict = {}
-    for k in hashfunc_keys:
-        mydict[k] = perform_checksum(x, k, calc_prelink)[0]
+    mydict = {k: perform_checksum(x, k, calc_prelink)[0] for k in hashfunc_keys}
     return mydict
 
 
@@ -458,11 +456,11 @@ def _apply_hash_filter(digests, hash_filter):
                     break
 
     if modified:
-        digests = dict(
-            (k, v)
+        digests = {
+            k: v
             for (k, v) in digests.items()
             if k == "size" or k in verifiable_hash_types
-        )
+        }
 
     return digests
 

diff --git a/lib/portage/eclass_cache.py b/lib/portage/eclass_cache.py
index 2545ed9b2..c4c783168 100644
--- a/lib/portage/eclass_cache.py
+++ b/lib/portage/eclass_cache.py
@@ -169,10 +169,7 @@ class cache:
         return d
 
     def get_eclass_data(self, inherits):
-        ec_dict = {}
-        for x in inherits:
-            ec_dict[x] = self.eclasses[x]
-
+        ec_dict = {x: self.eclasses[x] for x in inherits}
         return ec_dict
 
     @property

diff --git a/lib/portage/getbinpkg.py b/lib/portage/getbinpkg.py
index bf99fd2be..f357a91df 100644
--- a/lib/portage/getbinpkg.py
+++ b/lib/portage/getbinpkg.py
@@ -55,20 +55,24 @@ def make_metadata_dict(data):
     )
 
     myid, _myglob = data
-
-    mydict = {}
-    for k_bytes in portage.xpak.getindex_mem(myid):
-        k = _unicode_decode(
-            k_bytes, encoding=_encodings["repo.content"], errors="replace"
+    metadata = (
+        (
+            k_bytes,
+            _unicode_decode(
+                k_bytes, encoding=_encodings["repo.content"], errors="replace"
+            ),
         )
-        if k not in _all_metadata_keys and k != "CATEGORY":
-            continue
-        v = _unicode_decode(
+        for k_bytes in portage.xpak.getindex_mem(myid)
+    )
+    mydict = {
+        k: _unicode_decode(
             portage.xpak.getitem(data, k_bytes),
             encoding=_encodings["repo.content"],
             errors="replace",
         )
-        mydict[k] = v
+        for k_bytes, k in metadata
+        if k in _all_metadata_keys or k == "CATEGORY"
+    }
 
     return mydict
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     6e316a87828069f19906c5dace6f5215154b52b4
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Tue Mar 22 07:56:35 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 23:06:44 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6e316a87

Use str.join() instead of string concatenation

Joining strings is also faster.

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/_global_updates.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py
index a17ee861d..d7117d36b 100644
--- a/lib/portage/_global_updates.py
+++ b/lib/portage/_global_updates.py
@@ -97,21 +97,18 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
                     )
                     if not quiet:
                         writemsg_stdout(
-                            _(
-                                "  %s='update pass'  %s='binary update'  "
-                                "%s='/var/db update'  %s='/var/db move'\n"
-                                "  %s='/var/db SLOT move'  %s='binary move'  "
-                                "%s='binary SLOT move'\n  %s='update /etc/portage/package.*'\n"
-                            )
-                            % (
-                                bold("."),
-                                bold("*"),
-                                bold("#"),
-                                bold("@"),
-                                bold("s"),
-                                bold("%"),
-                                bold("S"),
-                                bold("p"),
+                            "  ".join(
+                                (
+                                    "",
+                                    f"{bold('.')}='update pass'",
+                                    f"{bold('*')}='binary update'",
+                                    f"{bold('#')}='/var/db update'",
+                                    f"{bold('@')}='/var/db move'\n",
+                                    f"{bold('s')}='/var/db SLOT move'",
+                                    f"{bold('%')}='binary move'"
+                                    f"{bold('S')}='binary SLOT move'\n",
+                                    f"{bold('p')}='update /etc/portage/package.*'\n",
+                                )
                             )
                         )
                 valid_updates, errors = parse_updates(mycontent)

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     7c8d5742d76d65ff3eb203ff269f65817b8b87fc
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Tue Mar 22 07:32:42 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 23:06:39 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c8d5742

Prefer tuples over explicit lists

Tuples offer better memory usage compared to lists due to Python's
ability to optimize with immutability. While these small amount of
changes won't shave off much, the plan is to find more of these which
will arithmetically have significance to the codebase.

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/const.py | 10 +++++-----
 lib/portage/data.py  |  4 ++--
 lib/portage/glsa.py  |  8 ++++----
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index 98eb7e4f2..f0f57067a 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -125,7 +125,7 @@ EBUILD_PHASES = (
     "other",
 )
 SUPPORTED_FEATURES = frozenset(
-    [
+    (
         "assume-digests",
         "binpkg-docompress",
         "binpkg-dostrip",
@@ -217,14 +217,14 @@ SUPPORTED_FEATURES = frozenset(
         "usersync",
         "webrsync-gpg",
         "xattr",
-    ]
+    )
 )
 
 EAPI = 8
 
 HASHING_BLOCKSIZE = 32768
 
-MANIFEST2_HASH_DEFAULTS = frozenset(["BLAKE2B", "SHA512"])
+MANIFEST2_HASH_DEFAULTS = frozenset(("BLAKE2B", "SHA512"))
 MANIFEST2_HASH_DEFAULT = "BLAKE2B"
 
 MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD")
@@ -248,7 +248,7 @@ VCS_DIRS = ("CVS", "RCS", "SCCS", ".bzr", ".git", ".hg", ".svn")
 
 # List of known live eclasses. Keep it in sync with cnf/sets/portage.conf
 LIVE_ECLASSES = frozenset(
-    [
+    (
         "bzr",
         "cvs",
         "darcs",
@@ -257,7 +257,7 @@ LIVE_ECLASSES = frozenset(
         "golang-vcs",
         "mercurial",
         "subversion",
-    ]
+    )
 )
 
 SUPPORTED_BINPKG_FORMATS = ("tar", "rpm")

diff --git a/lib/portage/data.py b/lib/portage/data.py
index 8c58ad0fc..de7d88e5e 100644
--- a/lib/portage/data.py
+++ b/lib/portage/data.py
@@ -83,12 +83,12 @@ def _target_root():
 
 def portage_group_warning():
     warn_prefix = colorize("BAD", "*** WARNING ***  ")
-    mylines = [
+    mylines = (
         "For security reasons, only system administrators should be",
         "allowed in the portage group.  Untrusted users or processes",
         "can potentially exploit the portage group for attacks such as",
         "local privilege escalation.",
-    ]
+    )
     for x in mylines:
         writemsg(warn_prefix, noiselevel=-1)
         writemsg(x, noiselevel=-1)

diff --git a/lib/portage/glsa.py b/lib/portage/glsa.py
index 4b92d52e0..1b68bc0e9 100644
--- a/lib/portage/glsa.py
+++ b/lib/portage/glsa.py
@@ -152,7 +152,7 @@ def getListElements(listnode):
     @rtype:		List of Strings
     @return:	a list that contains the value of the <li> elements
     """
-    if not listnode.nodeName in ["ul", "ol"]:
+    if not listnode.nodeName in ("ul", "ol"):
         raise GlsaFormatException("Invalid function call: listnode is not <ul> or <ol>")
     rValue = [
         getText(li, format="strip")
@@ -191,8 +191,8 @@ def getText(node, format, textfd=None):  # pylint: disable=redefined-builtin
         returnNone = False
     else:
         returnNone = True
-    if format in ["strip", "keep"]:
-        if node.nodeName in ["uri", "mail"]:
+    if format in ("strip", "keep"):
+        if node.nodeName in ("uri", "mail"):
             textfd.write(f"{node.childNodes[0].data}:{node.getAttribute('link')}")
         else:
             for subnode in node.childNodes:
@@ -206,7 +206,7 @@ def getText(node, format, textfd=None):  # pylint: disable=redefined-builtin
                 for p_subnode in subnode.childNodes:
                     if p_subnode.nodeName == "#text":
                         textfd.write(p_subnode.data.strip())
-                    elif p_subnode.nodeName in ["uri", "mail"]:
+                    elif p_subnode.nodeName in ("uri", "mail"):
                         textfd.write(p_subnode.childNodes[0].data)
                         textfd.write(" ( " + p_subnode.getAttribute("link") + " )")
                 textfd.write(NEWLINE_ESCAPE)

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     9b843f5c9d0a5bdd4a57efb5bd5f41c4736d4dcf
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Tue Mar 22 07:46:22 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 23:06:41 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9b843f5c

Consolidate nested conditional branches

Depending how a conditional is wrote, its nested conditional can be
consolidated into its own expression by using the `and` or `or` boolean
operators for the same effect. These kinds of refactors give processors
better chances at branch prediction and would make these conditionals
less expensive than they might have been.

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py      |  5 ++---
 lib/portage/data.py          |  9 ++-------
 lib/portage/dispatch_conf.py | 18 +++++++++++-------
 3 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index 3b5a10d24..d7e800c47 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -427,9 +427,8 @@ class _hash_filter:
         for token in self._tokens:
             if token in matches:
                 return True
-            if token[:1] == "-":
-                if token[1:] in matches:
-                    return False
+            if token[:1] == "-" and token[1:] in matches:
+                return False
         return False
 
 

diff --git a/lib/portage/data.py b/lib/portage/data.py
index de7d88e5e..b73fa8882 100644
--- a/lib/portage/data.py
+++ b/lib/portage/data.py
@@ -158,15 +158,12 @@ def _get_global(k):
                 unprivileged = _unprivileged_mode(eroot_or_parent, eroot_st)
 
         v = 0
-        if uid == 0:
-            v = 2
-        elif unprivileged:
+        if uid == 0 or unprivileged:
             v = 2
         elif _get_global("portage_gid") in os.getgroups():
             v = 1
 
     elif k in ("portage_gid", "portage_uid"):
-
         # Discover the uid and gid of the portage user/group
         keyerror = False
         try:
@@ -357,9 +354,7 @@ def _init(settings):
 
     if "secpass" not in _initialized_globals:
         v = 0
-        if uid == 0:
-            v = 2
-        elif "unprivileged" in settings.features:
+        if uid == 0 or "unprivileged" in settings.features:
             v = 2
         elif portage_gid in os.getgroups():
             v = 1

diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py
index c89caf087..d682a9ad0 100644
--- a/lib/portage/dispatch_conf.py
+++ b/lib/portage/dispatch_conf.py
@@ -238,13 +238,17 @@ def rcs_archive(archive, curconf, newconf, mrgconf):
 
         _archive_copy(mystat, newconf, archive)
 
-        if has_branch:
-            if mrgconf and os.path.isfile(archive) and os.path.isfile(mrgconf):
-                # This puts the results of the merge into mrgconf.
-                ret = os.system(f"rcsmerge -p -r{RCS_BRANCH} '{archive}' > '{mrgconf}'")
-                os.chmod(mrgconf, mystat.st_mode)
-                os.chown(mrgconf, mystat.st_uid, mystat.st_gid)
-        os.rename(archive, archive + ".dist.new")
+        if (
+            has_branch
+            and mrgconf
+            and os.path.isfile(archive)
+            and os.path.isfile(mrgconf)
+        ):
+            # This puts the results of the merge into mrgconf.
+            ret = os.system(f"rcsmerge -p -r{RCS_BRANCH} '{archive}' > '{mrgconf}'")
+            os.chmod(mrgconf, mystat.st_mode)
+            os.chown(mrgconf, mystat.st_uid, mystat.st_gid)
+        os.rename(archive, f"{archive}.dist.new")
 
     return ret
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     13740f43bcc38e787153d9efeabc4f5d878e7af0
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Tue Mar 22 07:22:42 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 23:06:37 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=13740f43

Prefer generator expressions over list comprehensions

List comprehensions do improve the performance compared to using for
loops to build the same lists in Python. However, they are eagerly
evaluated and will create a sequence ahead of time.

The problem is, a fair amount of code would only use these lists as
iterators which does not require them to be sequences. Unless there is a
need to know a list's length, access its items, or alter its size, it's
better to use generators which are lazily evaluated and can be iterated
when necessary. Avoiding the eagerness of list comprehensions will
improve the runtime performance.

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/__init__.py      | 22 +++++++++++++---------
 lib/portage/data.py          | 25 ++++++++++++++-----------
 lib/portage/dispatch_conf.py |  2 +-
 lib/portage/eclass_cache.py  |  2 +-
 lib/portage/getbinpkg.py     | 15 ++++++++-------
 lib/portage/glsa.py          | 14 ++++++++++----
 6 files changed, 47 insertions(+), 33 deletions(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 82e47d0ff..e6dff28ee 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -555,10 +555,15 @@ _deprecated_eapis = frozenset(
         "7_pre1",
     ]
 )
+
+from itertools import chain
+
 _supported_eapis = frozenset(
-    [str(x) for x in range(portage.const.EAPI + 1)]
-    + list(_testing_eapis)
-    + list(_deprecated_eapis)
+    chain(
+        (str(x) for x in range(portage.const.EAPI + 1)),
+        _testing_eapis,
+        _deprecated_eapis,
+    )
 )
 
 
@@ -672,8 +677,7 @@ def create_trees(
         # When ROOT != "/" we only want overrides from the calling
         # environment to apply to the config that's associated
         # with ROOT != "/", so pass a nearly empty dict for the env parameter.
-        clean_env = {}
-        for k in (
+        env_sequence = (
             "PATH",
             "PORTAGE_GRPNAME",
             "PORTAGE_REPOSITORIES",
@@ -687,10 +691,10 @@ def create_trees(
             "https_proxy",
             "no_proxy",
             "__PORTAGE_TEST_HARDLINK_LOCKS",
-        ):
-            v = settings.get(k)
-            if v is not None:
-                clean_env[k] = v
+        )
+        env = ((k, settings.get(k)) for k in env_sequence)
+        clean_env = {k: v for k, v in env if v is not None}
+
         if depcachedir is not None:
             clean_env["PORTAGE_DEPCACHEDIR"] = depcachedir
         settings = config(

diff --git a/lib/portage/data.py b/lib/portage/data.py
index e1457c566..8c58ad0fc 100644
--- a/lib/portage/data.py
+++ b/lib/portage/data.py
@@ -229,27 +229,30 @@ def _get_global(k):
             # Get a list of group IDs for the portage user. Do not use
             # grp.getgrall() since it is known to trigger spurious
             # SIGPIPE problems with nss_ldap.
-            cmd = ["id", "-G", _portage_username]
-
             encoding = portage._encodings["content"]
-            cmd = [
+            cmd = (
                 portage._unicode_encode(x, encoding=encoding, errors="strict")
-                for x in cmd
-            ]
+                for x in ("id", "-G", _portage_username)
+            )
             proc = subprocess.Popen(
                 cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
             )
             myoutput = proc.communicate()[0]
             status = proc.wait()
             if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-                for x in portage._unicode_decode(
-                    myoutput, encoding=encoding, errors="strict"
-                ).split():
+
+                def check(x):
                     try:
-                        v.append(int(x))
+                        return int(x)
                     except ValueError:
-                        pass
-                v = sorted(set(v))
+                        return None
+
+                unicode_decode = portage._unicode_decode(
+                    myoutput, encoding=encoding, errors="strict"
+                )
+                checked_v = (check(x) for x in unicode_decode.split())
+                filtered_v = (x for x in checked_v if x)
+                v = sorted(set(filtered_v))
 
     # Avoid instantiating portage.settings when the desired
     # variable is set in os.environ.

diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py
index 6c6036c4e..c89caf087 100644
--- a/lib/portage/dispatch_conf.py
+++ b/lib/portage/dispatch_conf.py
@@ -37,7 +37,7 @@ def diffstatusoutput(cmd, file1, file2):
     # raise a UnicodeDecodeError which makes the output inaccessible.
     args = shlex_split(cmd % (file1, file2))
 
-    args = [portage._unicode_encode(x, errors="strict") for x in args]
+    args = (portage._unicode_encode(x, errors="strict") for x in args)
     proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     output = portage._unicode_decode(proc.communicate()[0])
     if output and output[-1] == "\n":

diff --git a/lib/portage/eclass_cache.py b/lib/portage/eclass_cache.py
index f1729326d..2545ed9b2 100644
--- a/lib/portage/eclass_cache.py
+++ b/lib/portage/eclass_cache.py
@@ -110,7 +110,7 @@ class cache:
         master_eclasses = {}
         eclass_len = len(".eclass")
         ignored_listdir_errnos = (errno.ENOENT, errno.ENOTDIR)
-        for x in [normalize_path(os.path.join(y, "eclass")) for y in self.porttrees]:
+        for x in (normalize_path(os.path.join(y, "eclass")) for y in self.porttrees):
             try:
                 eclass_filenames = os.listdir(x)
             except OSError as e:

diff --git a/lib/portage/getbinpkg.py b/lib/portage/getbinpkg.py
index 1e89119fb..bf99fd2be 100644
--- a/lib/portage/getbinpkg.py
+++ b/lib/portage/getbinpkg.py
@@ -112,10 +112,12 @@ class ParseLinks(html_parser_HTMLParser):
 
     def handle_starttag(self, tag, attrs):
         if tag == "a":
-            for x in attrs:
-                if x[0] == "href":
-                    if x[1] not in self.PL_anchors:
-                        self.PL_anchors.append(urllib_parse_unquote(x[1]))
+            myarchors = (
+                urllib_parse_unquote(x[1])
+                for x in attrs
+                if x[0] == "href" and x[1] not in self.PL_anchors
+            )
+            self.PL_anchors.extend(myarchors)
 
 
 def create_conn(baseurl, conn=None):
@@ -533,8 +535,7 @@ def file_get(
     from portage.util import varexpand
     from portage.process import spawn
 
-    myfetch = portage.util.shlex_split(fcmd)
-    myfetch = [varexpand(x, mydict=variables) for x in myfetch]
+    myfetch = (varexpand(x, mydict=variables) for x in portage.util.shlex_split(fcmd))
     fd_pipes = {
         0: portage._get_stdin().fileno(),
         1: sys.__stdout__.fileno(),
@@ -986,5 +987,5 @@ class PackageIndex:
             keys = list(metadata)
             keys.sort()
             self._writepkgindex(
-                pkgfile, [(k, metadata[k]) for k in keys if metadata[k]]
+                pkgfile, ((k, metadata[k]) for k in keys if metadata[k])
             )

diff --git a/lib/portage/glsa.py b/lib/portage/glsa.py
index 05de3ade6..4b92d52e0 100644
--- a/lib/portage/glsa.py
+++ b/lib/portage/glsa.py
@@ -126,14 +126,20 @@ def get_glsa_list(myconfig):
         return []
     dirlist = os.listdir(repository)
     prefix = "glsa-"
+    prefix_size = len(prefix)
     suffix = ".xml"
+    suffix_size = len(suffix)
 
-    for f in dirlist:
+    def check(value):
         try:
-            if f[: len(prefix)] == prefix and f[-1 * len(suffix) :] == suffix:
-                rValue.append(f[len(prefix) : -1 * len(suffix)])
+            if value[:prefix_size] == prefix and value[-suffix_size:] == suffix:
+                return value[prefix_size:-suffix_size]
         except IndexError:
-            pass
+            return None
+        return None
+
+    checked_dirlist = (check(f) for f in dirlist)
+    rValue = [f for f in checked_dirlist if f]
     return rValue
 
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     0ad7157bcadf54bf39f28b0157355950b4c04f59
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Mon Mar 28 00:59:30 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Mar 28 01:10:19 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0ad7157b

Revert Portage settings refactor

This partially reverts 18e5a8170c69aecd10f162918de571d85055ae81.

Bug: https://bugs.gentoo.org/836298
Bug: https://bugs.gentoo.org/836299
Closes: https://bugs.gentoo.org/836297
Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/data.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib/portage/data.py b/lib/portage/data.py
index c783d76d0..1ef8d4aef 100644
--- a/lib/portage/data.py
+++ b/lib/portage/data.py
@@ -332,15 +332,17 @@ def _init(settings):
         # from grp.getgrnam() with PyPy
         native_string = platform.python_implementation() == "PyPy"
 
+        v = settings.get("PORTAGE_GRPNAME", "portage")
         if native_string:
-            grpname = settings.get("PORTAGE_GRPNAME", "portage")
-            grpname = portage._native_string(grpname)
-            globals()["_portage_grpname"] = grpname
-            _initialized_globals.add("_portage_grpname")
-            username = settings.get("PORTAGE_USERNAME", "portage")
-            username = portage._native_string(username)
-            globals()["_portage_username"] = username
-            _initialized_globals.add("_portage_username")
+            v = portage._native_string(v)
+        globals()["_portage_grpname"] = v
+        _initialized_globals.add("_portage_grpname")
+
+        v = settings.get("PORTAGE_USERNAME", "portage")
+        if native_string:
+            v = portage._native_string(v)
+        globals()["_portage_username"] = v
+        _initialized_globals.add("_portage_username")
 
     if "secpass" not in _initialized_globals:
         v = 0

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     94dad3d64599f7e2c0c28e90ec4bee883746e385
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Wed Mar 30 22:47:33 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Mar 30 23:11:43 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=94dad3d6

dispatch_conf: fix arguments

Closes: https://bugs.gentoo.org/836447
Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/dispatch_conf.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py
index 6dbb1885e..fc130c9af 100644
--- a/lib/portage/dispatch_conf.py
+++ b/lib/portage/dispatch_conf.py
@@ -28,14 +28,15 @@ RCS_GET = "co"
 _ARCHIVE_ROTATE_MAX = 9
 
 
-def diffstatusoutput(cmd, file1, file2):
+def diffstatusoutput(file1, file2):
     """
     Execute the string cmd in a shell with getstatusoutput() and return a
     2-tuple (status, output).
     """
     # Use Popen to emulate getstatusoutput(), since getstatusoutput() may
     # raise a UnicodeDecodeError which makes the output inaccessible.
-    args = shlex_split(cmd % (file1, file2))
+    cmd = f"diff -aq '{file1}' '{file2}'"
+    args = shlex_split(cmd)
 
     args = (portage._unicode_encode(x, errors="strict") for x in args)
     proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -340,7 +341,7 @@ def file_archive(archive, curconf, newconf, mrgconf):
     # Archive the current config file if it isn't already saved
     if (
         os.path.lexists(archive)
-        and len(diffstatusoutput_mixed(f"diff -aq '{curconf}' '{archive}'")[1]) != 0
+        and len(diffstatusoutput_mixed(curconf, archive)[1]) != 0
     ):
         _file_archive_rotate(archive)
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Matt Turner]

commit:     1b1c4323d73d095f5ee2816201b5780f6a7ea14b
Author:     Wolfgang E. Sanyer <WolfgangESanyer <AT> gmail <DOT> com>
AuthorDate: Thu Feb 24 02:30:24 2022 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Apr  1 20:29:00 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1b1c4323

portage.eapi: use functools @lru_cache decorator

Also, refactor logic in order to improve readability.

Closes: https://github.com/gentoo/portage/pull/754
Signed-off-by: Wolfgang E. Sanyer <WolfgangESanyer <AT> gmail.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 lib/portage/eapi.py | 458 +++++++++++++++++++---------------------------------
 1 file changed, 169 insertions(+), 289 deletions(-)

diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py
index 18069b04b..f8e8573ed 100644
--- a/lib/portage/eapi.py
+++ b/lib/portage/eapi.py
@@ -2,288 +2,167 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import collections
-import operator
-import types
+import logging
+from functools import lru_cache
+from typing import Optional
 
 from portage import eapi_is_supported
 
 
-def eapi_has_iuse_defaults(eapi):
-    return eapi != "0"
+def eapi_has_iuse_defaults(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).iuse_defaults
 
 
-def eapi_has_iuse_effective(eapi):
-    return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi")
+def eapi_has_iuse_effective(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).iuse_effective
 
 
-def eapi_has_slot_deps(eapi):
-    return eapi != "0"
+def eapi_has_slot_deps(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).slot_deps
 
 
-def eapi_has_slot_operator(eapi):
-    return eapi not in ("0", "1", "2", "3", "4", "4-python")
+def eapi_has_slot_operator(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).slot_operator
 
 
-def eapi_has_src_uri_arrows(eapi):
-    return eapi not in ("0", "1")
+def eapi_has_src_uri_arrows(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).src_uri_arrows
 
 
-def eapi_has_selective_src_uri_restriction(eapi):
-    return eapi not in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-        "7",
-    )
+def eapi_has_selective_src_uri_restriction(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).selective_src_uri_restriction
 
 
-def eapi_has_use_deps(eapi):
-    return eapi not in ("0", "1")
+def eapi_has_use_deps(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).use_deps
 
 
-def eapi_has_strong_blocks(eapi):
-    return eapi not in ("0", "1")
+def eapi_has_strong_blocks(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).strong_blocks
 
 
-def eapi_has_src_prepare_and_src_configure(eapi):
+def eapi_has_src_prepare_and_src_configure(eapi: str) -> bool:
     return eapi not in ("0", "1")
 
 
-def eapi_supports_prefix(eapi):
-    return eapi not in ("0", "1", "2")
+def eapi_supports_prefix(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).prefix
 
 
-def eapi_exports_AA(eapi):
-    return eapi in ("0", "1", "2", "3")
+def eapi_exports_AA(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).exports_AA
 
 
-def eapi_exports_KV(eapi):
-    return eapi in ("0", "1", "2", "3")
+def eapi_exports_KV(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).exports_KV
 
 
-def eapi_exports_merge_type(eapi):
-    return eapi not in ("0", "1", "2", "3")
+def eapi_exports_merge_type(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).exports_merge_type
 
 
-def eapi_exports_replace_vars(eapi):
-    return eapi not in ("0", "1", "2", "3")
+def eapi_exports_replace_vars(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).exports_replace_vars
 
 
-def eapi_exports_EBUILD_PHASE_FUNC(eapi):
-    return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi")
+def eapi_exports_EBUILD_PHASE_FUNC(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).exports_EBUILD_PHASE_FUNC
 
 
-def eapi_exports_PORTDIR(eapi):
-    return eapi in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-    )
+def eapi_exports_PORTDIR(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).exports_PORTDIR
 
 
-def eapi_exports_ECLASSDIR(eapi):
-    return eapi in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-    )
+def eapi_exports_ECLASSDIR(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).exports_ECLASSDIR
 
 
-def eapi_exports_REPOSITORY(eapi):
+def eapi_exports_REPOSITORY(eapi: str) -> bool:
     return eapi in ("4-python", "5-progress")
 
 
-def eapi_has_pkg_pretend(eapi):
+def eapi_has_pkg_pretend(eapi: str) -> bool:
     return eapi not in ("0", "1", "2", "3")
 
 
-def eapi_has_implicit_rdepend(eapi):
+def eapi_has_implicit_rdepend(eapi: str) -> bool:
     return eapi in ("0", "1", "2", "3")
 
 
-def eapi_has_dosed_dohard(eapi):
+def eapi_has_dosed_dohard(eapi: str) -> bool:
     return eapi in ("0", "1", "2", "3")
 
 
-def eapi_has_required_use(eapi):
-    return eapi not in ("0", "1", "2", "3")
+def eapi_has_required_use(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).required_use
 
 
-def eapi_has_required_use_at_most_one_of(eapi):
-    return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi")
+def eapi_has_required_use_at_most_one_of(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).required_use_at_most_one_of
 
 
-def eapi_has_use_dep_defaults(eapi):
-    return eapi not in ("0", "1", "2", "3")
+def eapi_has_use_dep_defaults(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).use_dep_defaults
 
 
-def eapi_requires_posixish_locale(eapi):
-    return eapi not in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-    )
+def eapi_requires_posixish_locale(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).posixish_locale
 
 
-def eapi_has_repo_deps(eapi):
-    return eapi in ("4-python", "5-progress")
+def eapi_has_repo_deps(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).repo_deps
 
 
-def eapi_allows_dots_in_PN(eapi):
-    return eapi in ("4-python", "5-progress")
+def eapi_allows_dots_in_PN(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).dots_in_PN
 
 
-def eapi_allows_dots_in_use_flags(eapi):
-    return eapi in ("4-python", "5-progress")
+def eapi_allows_dots_in_use_flags(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).dots_in_use_flags
 
 
-def eapi_supports_stable_use_forcing_and_masking(eapi):
+def eapi_supports_stable_use_forcing_and_masking(eapi: str) -> bool:
     return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi")
 
 
-def eapi_allows_directories_on_profile_level_and_repository_level(eapi):
+def eapi_allows_directories_on_profile_level_and_repository_level(eapi: str) -> bool:
     return eapi not in ("0", "1", "2", "3", "4", "4-slot-abi", "5", "6")
 
 
-def eapi_has_use_aliases(eapi):
+def eapi_has_use_aliases(eapi: str) -> bool:
     return eapi in ("4-python", "5-progress")
 
 
-def eapi_has_automatic_unpack_dependencies(eapi):
+def eapi_has_automatic_unpack_dependencies(eapi: str) -> bool:
     return eapi in ("5-progress",)
 
 
-def eapi_allows_package_provided(eapi):
-    return eapi in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-    )
+def eapi_allows_package_provided(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).allows_package_provided
 
 
-def eapi_has_bdepend(eapi):
-    return eapi not in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-    )
+def eapi_has_bdepend(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).bdepend
 
 
-def eapi_has_idepend(eapi):
-    return eapi not in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-        "7",
-    )
+def eapi_has_idepend(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).idepend
 
 
-def eapi_empty_groups_always_true(eapi):
-    return eapi in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-    )
+def eapi_empty_groups_always_true(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).empty_groups_always_true
 
 
-def eapi_path_variables_end_with_trailing_slash(eapi):
-    return eapi in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-    )
+def eapi_path_variables_end_with_trailing_slash(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).path_variables_end_with_trailing_slash
 
 
-def eapi_has_broot(eapi):
-    return eapi not in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-    )
+def eapi_has_broot(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).broot
 
 
-def eapi_has_sysroot(eapi):
-    return eapi not in (
-        "0",
-        "1",
-        "2",
-        "3",
-        "4",
-        "4-python",
-        "4-slot-abi",
-        "5",
-        "5-progress",
-        "6",
-    )
+def eapi_has_sysroot(eapi: str) -> bool:
+    return _get_eapi_attrs(eapi).sysroot
 
 
 _eapi_attrs = collections.namedtuple(
@@ -324,53 +203,39 @@ _eapi_attrs = collections.namedtuple(
 )
 
 
-_eapi_attr_func_prefixes = (
-    "eapi_allows_",
-    "eapi_has_",
-    "eapi_requires_",
-    "eapi_supports_",
-    "eapi_",
-)
-
-
-def _eapi_func_decorator(func, attr_getter):
-    def wrapper(eapi):
-        return attr_getter(_get_eapi_attrs(eapi))
-
-    wrapper.func = func
-    wrapper.__doc__ = func.__doc__
-    return wrapper
-
+class Eapi:
+    ALL_EAPIS = (
+        "0",
+        "1",
+        "2",
+        "3",
+        "4",
+        "4-python",
+        "4-slot-abi",
+        "5",
+        "5-progress",
+        "6",
+        "7",
+        "8",
+    )
 
-def _decorate_eapi_funcs():
-    """
-    Decorate eapi_* functions so that they use _get_eapi_attrs(eapi)
-    to cache results.
-    """
-    decorated = {}
-    for k, v in globals().items():
-        if not (
-            isinstance(v, types.FunctionType) and k.startswith(_eapi_attr_func_prefixes)
-        ):
-            continue
-        for prefix in _eapi_attr_func_prefixes:
-            if k.startswith(prefix):
-                attr_name = k[len(prefix) :]
-                if hasattr(_eapi_attrs, attr_name):
-                    decorated[k] = _eapi_func_decorator(
-                        v, operator.attrgetter(attr_name)
-                    )
-                    break
-    globals().update(decorated)
+    _eapi_val: int = -1
 
+    def __init__(self, eapi_string: str):
+        if not eapi_string in self.ALL_EAPIS:
+            raise ValueError(f"'{eapi_string}' not recognized as a valid EAPI")
 
-_decorate_eapi_funcs()
+        self._eapi_val = int(eapi_string.partition("-")[0])
 
+    def __ge__(self, other: "Eapi") -> bool:
+        return self._eapi_val >= other._eapi_val
 
-_eapi_attrs_cache = {}
+    def __le__(self, other: "Eapi") -> bool:
+        return self._eapi_val <= other._eapi_val
 
 
-def _get_eapi_attrs(eapi):
+@lru_cache(32)
+def _get_eapi_attrs(eapi_str: Optional[str]) -> _eapi_attrs:
     """
     When eapi is None then validation is not as strict, since we want the
     same to work for multiple EAPIs that may have slightly different rules.
@@ -378,59 +243,74 @@ def _get_eapi_attrs(eapi):
     be helpful for handling of corrupt EAPI metadata in essential functions
     such as pkgsplit.
     """
-    eapi_attrs = _eapi_attrs_cache.get(eapi)
-    if eapi_attrs is not None:
-        return eapi_attrs
-
-    orig_eapi = eapi
-    if eapi is not None and not eapi_is_supported(eapi):
-        eapi = None
-
-    eapi_attrs = _eapi_attrs(
-        allows_package_provided=(
-            eapi is None or eapi_allows_package_provided.func(eapi)
-        ),
-        bdepend=(eapi is not None and eapi_has_bdepend.func(eapi)),
-        broot=(eapi is None or eapi_has_broot.func(eapi)),
-        dots_in_PN=(eapi is None or eapi_allows_dots_in_PN.func(eapi)),
-        dots_in_use_flags=(eapi is None or eapi_allows_dots_in_use_flags.func(eapi)),
-        empty_groups_always_true=(
-            eapi is not None and eapi_empty_groups_always_true.func(eapi)
-        ),
-        exports_AA=(eapi is not None and eapi_exports_AA.func(eapi)),
-        exports_EBUILD_PHASE_FUNC=(
-            eapi is None or eapi_exports_EBUILD_PHASE_FUNC.func(eapi)
-        ),
-        exports_ECLASSDIR=(eapi is not None and eapi_exports_ECLASSDIR.func(eapi)),
-        exports_KV=(eapi is not None and eapi_exports_KV.func(eapi)),
-        exports_merge_type=(eapi is None or eapi_exports_merge_type.func(eapi)),
-        exports_PORTDIR=(eapi is None or eapi_exports_PORTDIR.func(eapi)),
-        exports_replace_vars=(eapi is None or eapi_exports_replace_vars.func(eapi)),
-        feature_flag_test=False,
-        idepend=(eapi is not None and eapi_has_idepend.func(eapi)),
-        iuse_defaults=(eapi is None or eapi_has_iuse_defaults.func(eapi)),
-        iuse_effective=(eapi is not None and eapi_has_iuse_effective.func(eapi)),
-        path_variables_end_with_trailing_slash=(
-            eapi is not None and eapi_path_variables_end_with_trailing_slash.func(eapi)
-        ),
-        posixish_locale=(eapi is not None and eapi_requires_posixish_locale.func(eapi)),
-        prefix=(eapi is None or eapi_supports_prefix.func(eapi)),
-        repo_deps=(eapi is None or eapi_has_repo_deps.func(eapi)),
-        required_use=(eapi is None or eapi_has_required_use.func(eapi)),
-        required_use_at_most_one_of=(
-            eapi is None or eapi_has_required_use_at_most_one_of.func(eapi)
-        ),
-        selective_src_uri_restriction=(
-            eapi is None or eapi_has_selective_src_uri_restriction.func(eapi)
-        ),
-        slot_deps=(eapi is None or eapi_has_slot_deps.func(eapi)),
-        slot_operator=(eapi is None or eapi_has_slot_operator.func(eapi)),
-        src_uri_arrows=(eapi is None or eapi_has_src_uri_arrows.func(eapi)),
-        strong_blocks=(eapi is None or eapi_has_strong_blocks.func(eapi)),
-        sysroot=(eapi is None or eapi_has_sysroot.func(eapi)),
-        use_deps=(eapi is None or eapi_has_use_deps.func(eapi)),
-        use_dep_defaults=(eapi is None or eapi_has_use_dep_defaults.func(eapi)),
-    )
-
-    _eapi_attrs_cache[orig_eapi] = eapi_attrs
-    return eapi_attrs
+    logging.info("cache info: {}".format(_get_eapi_attrs.cache_info()))
+    if eapi_str is None or not eapi_is_supported(eapi_str):
+        return _eapi_attrs(
+            allows_package_provided=True,
+            bdepend=False,
+            broot=True,
+            dots_in_PN=True,
+            dots_in_use_flags=True,
+            empty_groups_always_true=False,
+            exports_AA=False,
+            exports_EBUILD_PHASE_FUNC=True,
+            exports_ECLASSDIR=False,
+            exports_KV=False,
+            exports_merge_type=True,
+            exports_PORTDIR=True,
+            exports_replace_vars=True,
+            feature_flag_test=False,
+            idepend=False,
+            iuse_defaults=True,
+            iuse_effective=False,
+            path_variables_end_with_trailing_slash=False,
+            posixish_locale=False,
+            prefix=True,
+            repo_deps=True,
+            required_use=True,
+            required_use_at_most_one_of=True,
+            selective_src_uri_restriction=True,
+            slot_deps=True,
+            slot_operator=True,
+            src_uri_arrows=True,
+            strong_blocks=True,
+            sysroot=True,
+            use_deps=True,
+            use_dep_defaults=True,
+        )
+    else:
+        eapi_special = ("4-python", "5-progress")
+        eapi = Eapi(eapi_str)
+        return _eapi_attrs(
+            allows_package_provided=eapi <= Eapi("6"),
+            bdepend=eapi >= Eapi("7"),
+            broot=eapi >= Eapi("7"),
+            dots_in_PN=eapi_str in eapi_special,
+            dots_in_use_flags=eapi_str in eapi_special,
+            empty_groups_always_true=eapi <= Eapi("6"),
+            exports_AA=eapi <= Eapi("3"),
+            exports_EBUILD_PHASE_FUNC=eapi >= Eapi("5"),
+            exports_ECLASSDIR=eapi <= Eapi("6"),
+            exports_KV=eapi <= Eapi("3"),
+            exports_merge_type=eapi >= Eapi("4"),
+            exports_PORTDIR=eapi <= Eapi("6"),
+            exports_replace_vars=eapi >= Eapi("4"),
+            feature_flag_test=False,
+            idepend=eapi >= Eapi("8"),
+            iuse_defaults=eapi >= Eapi("1"),
+            iuse_effective=eapi >= Eapi("5"),
+            path_variables_end_with_trailing_slash=eapi <= Eapi("6"),
+            posixish_locale=eapi >= Eapi("6"),
+            prefix=eapi >= Eapi("3"),
+            repo_deps=eapi_str in eapi_special,
+            required_use=eapi >= Eapi("4"),
+            required_use_at_most_one_of=eapi >= Eapi("5"),
+            selective_src_uri_restriction=eapi >= Eapi("8"),
+            slot_deps=eapi >= Eapi("1"),
+            slot_operator=eapi >= Eapi("5"),
+            src_uri_arrows=eapi >= Eapi("2"),
+            strong_blocks=eapi >= Eapi("2"),
+            sysroot=eapi >= Eapi("7"),
+            use_deps=eapi >= Eapi("2"),
+            use_dep_defaults=eapi >= Eapi("4"),
+        )

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     231d7d26c9076a8a88eddbf048437d46ed46a7ed
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:04:55 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:32 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=231d7d26

Use String Interpolation where possible

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 17 ++++++++--------
 lib/portage/metadata.py | 19 +++++++++---------
 lib/portage/module.py   | 53 +++++++++++++++++++------------------------------
 lib/portage/news.py     | 31 +++++++++++++----------------
 4 files changed, 52 insertions(+), 68 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index 0b4fad76c..5472e8fb1 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -352,7 +352,7 @@ class Manifest:
                     # non-empty for all currently known use cases.
                     write_atomic(
                         self.getFullname(),
-                        "".join("%s\n" % str(myentry) for myentry in myentries),
+                        "".join(f"{myentry}\n" for myentry in myentries),
                     )
                     self._apply_max_mtime(preserved_stats, myentries)
                     rval = True
@@ -447,8 +447,7 @@ class Manifest:
                     # unless this repo is being prepared for distribution
                     # via rsync.
                     writemsg_level(
-                        "!!! utime('%s', (%s, %s)): %s\n"
-                        % (path, max_mtime, max_mtime, e),
+                        f"!!! utime('{path}', ({max_mtime}, {max_mtime})): {e}\n",
                         level=logging.WARNING,
                         noiselevel=-1,
                     )
@@ -590,12 +589,12 @@ class Manifest:
             return None
         pf = filename[:-7]
         ps = portage.versions._pkgsplit(pf)
-        cpv = "%s/%s" % (cat, pf)
+        cpv = f"{cat}/{pf}"
         if not ps:
-            raise PortagePackageException(_("Invalid package name: '%s'") % cpv)
+            raise PortagePackageException(_(f"Invalid package name: '{cpv}'"))
         if ps[0] != pn:
             raise PortagePackageException(
-                _("Package name does not " "match directory name: '%s'") % cpv
+                _(f"Package name does not match directory name: '{cpv}'")
             )
         return cpv
 
@@ -635,7 +634,7 @@ class Manifest:
             else:
                 continue
             self.fhashdict[mytype][f] = perform_multiple_checksums(
-                self.pkgdir + f, self.hashes
+                f"{self.pkgdir}{f}", self.hashes
             )
         recursive_files = []
 
@@ -693,7 +692,7 @@ class Manifest:
         except FileNotFound as e:
             if not ignoreMissing:
                 raise
-            return False, _("File Not Found: '%s'") % str(e)
+            return False, _(f"File Not Found: '{e}'")
 
     def checkCpvHashes(
         self, cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=False
@@ -704,7 +703,7 @@ class Manifest:
             self.checkTypeHashes("AUX", ignoreMissingFiles=False)
             if checkMiscfiles:
                 self.checkTypeHashes("MISC", ignoreMissingFiles=False)
-            ebuildname = "%s.ebuild" % self._catsplit(cpv)[1]
+            ebuildname = f"{self._catsplit(cpv)[1]}.ebuild"
             self.checkFileHashes("EBUILD", ebuildname, ignoreMissing=False)
         if checkDistfiles or onlyDistfiles:
             for f in self._getCpvDistfiles(cpv):

diff --git a/lib/portage/metadata.py b/lib/portage/metadata.py
index 0bd2bcce4..357917051 100644
--- a/lib/portage/metadata.py
+++ b/lib/portage/metadata.py
@@ -38,12 +38,11 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
         "/var",
     ]:
         print(
-            "!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY "
-            + "ROOT DIRECTORY ON YOUR SYSTEM.",
-            file=sys.stderr,
-        )
-        print(
-            "!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir,
+            (
+                "!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY "
+                "ROOT DIRECTORY ON YOUR SYSTEM.\n"
+                f"!!! This is ALMOST CERTAINLY NOT what you want: '{cachedir}'",
+            ),
             file=sys.stderr,
         )
         sys.exit(73)
@@ -123,7 +122,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
 
             src_chf = tree_data.src_db.validation_chf
             dest_chf = tree_data.dest_db.validation_chf
-            dest_chf_key = "_%s_" % dest_chf
+            dest_chf_key = f"_{dest_chf}_"
             dest_chf_getter = operator.attrgetter(dest_chf)
 
             for cpv in portdb.cp_list(cp, mytree=tree_data.path):
@@ -219,8 +218,10 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
             dead_nodes = set(tree_data.dest_db)
         except CacheError as e:
             writemsg_level(
-                "Error listing cache entries for "
-                + "'%s': %s, continuing...\n" % (tree_data.path, e),
+                (
+                    "Error listing cache entries for "
+                    f"'{tree_data.path}': {e}, continuing...\n"
+                ),
                 level=logging.ERROR,
                 noiselevel=-1,
             )

diff --git a/lib/portage/module.py b/lib/portage/module.py
index a30d509ee..61c85aa47 100644
--- a/lib/portage/module.py
+++ b/lib/portage/module.py
@@ -53,12 +53,13 @@ class Module:
                 kid["module_name"] = ".".join([mod_name, kid["sourcefile"]])
             except KeyError:
                 kid["module_name"] = ".".join([mod_name, self.name])
-                msg = (
-                    "%s module's module_spec is old, missing attribute: "
-                    "'sourcefile'.  Backward compatibility may be "
-                    "removed in the future.\nFile: %s\n"
+                writemsg(
+                    _(
+                        f"{self.name} module's module_spec is old, missing attribute: "
+                        "'sourcefile'.  Backward compatibility may be "
+                        f"removed in the future.\nFile: {self._module.__file__}\n"
+                    )
                 )
-                writemsg(_(msg) % (self.name, self._module.__file__))
             kid["is_imported"] = False
             self.kids[kidname] = kid
             self.kids_names.append(kidname)
@@ -67,8 +68,10 @@ class Module:
     def get_class(self, name):
         if not name or name not in self.kids_names:
             raise InvalidModuleName(
-                "Module name '%s' is invalid or not" % name
-                + "part of the module '%s'" % self.name
+                (
+                    f"Module name '{name}' is invalid or not"
+                    f"part of the module '{self.name}'"
+                )
             )
         kid = self.kids[name]
         if kid["is_imported"]:
@@ -149,9 +152,7 @@ class Modules:
         if modname and modname in self.module_names:
             mod = self._modules[modname]["parent"].get_class(modname)
         else:
-            raise InvalidModuleName(
-                "Module name '%s' is invalid or not" % modname + "found"
-            )
+            raise InvalidModuleName(f"Module name '{modname}' is invalid or not found")
         return mod
 
     def get_description(self, modname):
@@ -165,9 +166,7 @@ class Modules:
         if modname and modname in self.module_names:
             mod = self._modules[modname]["description"]
         else:
-            raise InvalidModuleName(
-                "Module name '%s' is invalid or not" % modname + "found"
-            )
+            raise InvalidModuleName(f"Module name '{modname}' is invalid or not found")
         return mod
 
     def get_functions(self, modname):
@@ -181,9 +180,7 @@ class Modules:
         if modname and modname in self.module_names:
             mod = self._modules[modname]["functions"]
         else:
-            raise InvalidModuleName(
-                "Module name '%s' is invalid or not" % modname + "found"
-            )
+            raise InvalidModuleName(f"Module name '{modname}' is invalid or not found")
         return mod
 
     def get_func_descriptions(self, modname):
@@ -197,9 +194,7 @@ class Modules:
         if modname and modname in self.module_names:
             desc = self._modules[modname]["func_desc"]
         else:
-            raise InvalidModuleName(
-                "Module name '%s' is invalid or not" % modname + "found"
-            )
+            raise InvalidModuleName(f"Module name '{modname}' is invalid or not found")
         return desc
 
     def get_opt_descriptions(self, modname):
@@ -213,9 +208,7 @@ class Modules:
         if modname and modname in self.module_names:
             desc = self._modules[modname].get("opt_desc")
         else:
-            raise InvalidModuleName(
-                "Module name '%s' is invalid or not found" % modname
-            )
+            raise InvalidModuleName(f"Module name '{modname}' is invalid or not found")
         return desc
 
     def get_spec(self, modname, var):
@@ -231,22 +224,16 @@ class Modules:
         if modname and modname in self.module_names:
             value = self._modules[modname].get(var, None)
         else:
-            raise InvalidModuleName(
-                "Module name '%s' is invalid or not found" % modname
-            )
+            raise InvalidModuleName(f"Module name '{modname}' is invalid or not found")
         return value
 
     def _check_compat(self, module):
         if self.compat_versions:
             if not module.module_spec["version"] in self.compat_versions:
                 raise ModuleVersionError(
-                    "Error loading '%s' plugin module: %s, version: %s\n"
-                    "Module is not compatible with the current application version\n"
-                    "Compatible module API versions are: %s"
-                    % (
-                        self._namepath,
-                        module.module_spec["name"],
-                        module.module_spec["version"],
-                        self.compat_versions,
+                    (
+                        f"Error loading '{self._namepath}' plugin module: {module.module_spec['name']}, version: {module.module_spec['version']}\n"
+                        "Module is not compatible with the current application version\n"
+                        f"Compatible module API versions are: {self.compat_versions}"
                     )
                 )

diff --git a/lib/portage/news.py b/lib/portage/news.py
index ce61f8490..9ef6efde0 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -91,15 +91,15 @@ class NewsManager:
         self._profile_path = profile_path
 
     def _unread_filename(self, repoid):
-        return os.path.join(self.unread_path, "news-%s.unread" % repoid)
+        return os.path.join(self.unread_path, f"news-{repoid}.unread")
 
     def _skip_filename(self, repoid):
-        return os.path.join(self.unread_path, "news-%s.skip" % repoid)
+        return os.path.join(self.unread_path, f"news-{repoid}.skip")
 
     def _news_dir(self, repoid):
         repo_path = self.portdb.getRepositoryPath(repoid)
         if repo_path is None:
-            raise AssertionError(_("Invalid repoID: %s") % repoid)
+            raise AssertionError(_(f"Invalid repoID: {repoid}"))
         return os.path.join(repo_path, self.news_path)
 
     def updateItems(self, repoid):
@@ -164,7 +164,7 @@ class NewsManager:
                 if itemid in skip:
                     continue
                 filename = os.path.join(
-                    news_dir, itemid, itemid + "." + self.language_id + ".txt"
+                    news_dir, itemid, f"{itemid}.{self.language_id}.txt"
                 )
                 if not os.path.isfile(filename):
                     continue
@@ -178,9 +178,7 @@ class NewsManager:
                     skip.add(item.name)
 
             if unread != unread_orig:
-                write_atomic(
-                    unread_filename, "".join("%s\n" % x for x in sorted(unread))
-                )
+                write_atomic(unread_filename, "".join(f"{x}\n" for x in sorted(unread)))
                 apply_secpass_permissions(
                     unread_filename,
                     uid=self._uid,
@@ -190,7 +188,7 @@ class NewsManager:
                 )
 
             if skip != skip_orig:
-                write_atomic(skip_filename, "".join("%s\n" % x for x in sorted(skip)))
+                write_atomic(skip_filename, "".join(f"{x}\n" for x in sorted(skip)))
                 apply_secpass_permissions(
                     skip_filename,
                     uid=self._uid,
@@ -348,12 +346,12 @@ class NewsItem:
 
         if invalids:
             self._valid = False
-            msg = []
-            msg.append(_("Invalid news item: %s") % (self.path,))
-            for lineno, line in invalids:
-                msg.append(_("  line %d: %s") % (lineno, line))
+            msg = [
+                _(f"Invalid news item: {self.path}"),
+                *(_(f"  line {lineno}: {line}") for lineno, line in invalids),
+            ]
             writemsg_level(
-                "".join("!!! %s\n" % x for x in msg), level=logging.ERROR, noiselevel=-1
+                "".join(f"!!! {x}\n" for x in msg), level=logging.ERROR, noiselevel=-1
             )
 
         self._parsed = True
@@ -473,7 +471,7 @@ def count_unread_news(portdb, vardb, repos=None, update=True):
             # NOTE: The NewsManager typically handles permission errors by
             # returning silently, so PermissionDenied won't necessarily be
             # raised even if we do trigger a permission error above.
-            msg = "Permission denied: '%s'\n" % (e,)
+            msg = f"Permission denied: '{e}'\n"
             if msg in permission_msgs:
                 pass
             else:
@@ -498,9 +496,8 @@ def display_news_notifications(news_counts):
                 newsReaderDisplay = True
                 print()
             print(colorize("WARN", " * IMPORTANT:"), end=" ")
-            print("%s news items need reading for repository '%s'." % (count, repo))
+            print(f"{count} news items need reading for repository '{repo}'.")
 
     if newsReaderDisplay:
         print(colorize("WARN", " *"), end=" ")
-        print("Use " + colorize("GOOD", "eselect news read") + " to view new items.")
-        print()
+        print(f"Use {colorize('GOOD', 'eselect news read')} to view new items.\n")

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     9e24d0143450628f334cdb62e579efafd1bfd2ba
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:13:57 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:34 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e24d014

Simplify with declarative programming

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 144 ++++++++++++++++++++++++------------------------
 lib/portage/metadata.py |  15 ++---
 lib/portage/module.py   |  23 +++++---
 lib/portage/news.py     |  14 ++---
 lib/portage/output.py   |  13 ++---
 5 files changed, 105 insertions(+), 104 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index 5472e8fb1..ff166faa8 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -3,6 +3,7 @@
 
 import errno
 import io
+import itertools
 import logging
 import re
 import stat
@@ -107,9 +108,8 @@ class Manifest2Entry(ManifestEntry):
         myhashkeys = list(self.hashes)
         myhashkeys.remove("size")
         myhashkeys.sort()
-        for h in myhashkeys:
-            myline += " " + h + " " + str(self.hashes[h])
-        return myline
+        with_hashes = " ".join(f"{h} {self.hashes[h]}" for h in myhashkeys)
+        return f"{myline} {with_hashes}"
 
     def __eq__(self, other):
         if (
@@ -162,7 +162,6 @@ class Manifest:
             find_invalid_path_char = _find_invalid_path_char
         self._find_invalid_path_char = find_invalid_path_char
         self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
-        self.fhashdict = {}
         self.hashes = set()
         self.required_hashes = set()
 
@@ -182,8 +181,8 @@ class Manifest:
         self.required_hashes.update(required_hashes)
         self.required_hashes.intersection_update(self.hashes)
 
-        for t in MANIFEST2_IDENTIFIERS:
-            self.fhashdict[t] = {}
+        self.fhashdict = {t: {} for t in MANIFEST2_IDENTIFIERS}
+
         if not from_scratch:
             self._read()
         if fetchlist_dict != None:
@@ -206,9 +205,9 @@ class Manifest:
 
     def getDigests(self):
         """Compability function for old digest/manifest code, returns dict of filename:{hashfunction:hashvalue}"""
-        rval = {}
-        for t in MANIFEST2_IDENTIFIERS:
-            rval.update(self.fhashdict[t])
+        rval = {
+            k: v for t in MANIFEST2_IDENTIFIERS for k, v in self.fhashdict[t].items()
+        }
         return rval
 
     def getTypeDigests(self, ftype):
@@ -269,18 +268,16 @@ class Manifest:
 
     def _getDigestData(self, distlist):
         """create a hash dict for a specific list of files"""
-        myhashdict = {}
-        for myname in distlist:
-            for mytype in self.fhashdict:
-                if myname in self.fhashdict[mytype]:
-                    myhashdict.setdefault(mytype, {})
-                    myhashdict[mytype].setdefault(myname, {})
-                    myhashdict[mytype][myname].update(self.fhashdict[mytype][myname])
+        myhashdict = {
+            mytype: {myname: self.fhashdict[mytype][myname]}
+            for myname in distlist
+            for mytype in self.fhashdict
+            if myname in self.fhashdict[mytype]
+        }
         return myhashdict
 
     def _createManifestEntries(self):
-        valid_hashes = set(get_valid_checksum_keys())
-        valid_hashes.add("size")
+        valid_hashes = set(itertools.chain(get_valid_checksum_keys(), ("size")))
         mytypes = list(self.fhashdict)
         mytypes.sort()
         for t in mytypes:
@@ -296,13 +293,22 @@ class Manifest:
                 yield myentry
 
     def checkIntegrity(self):
-        for t in self.fhashdict:
-            for f in self.fhashdict[t]:
-                diff = self.required_hashes.difference(set(self.fhashdict[t][f]))
-                if diff:
-                    raise MissingParameter(
-                        _("Missing %s checksum(s): %s %s") % (" ".join(diff), t, f)
+        manifest_data = (
+            (
+                self.required_hashes.difference(set(self.fhashdict[mytype][myfile])),
+                mytype,
+                myfile,
+            )
+            for mytype in self.fhashdict
+            for myfile in self.fhashdict[mytype]
+        )
+        for needed_hashes, its_type, its_file in manifest_data:
+            if needed_hashes:
+                raise MissingParameter(
+                    _(
+                        f"Missing {' '.join(needed_hashes)} checksum(s): {its_type} {its_file}"
                     )
+                )
 
     def write(self, sign=False, force=False):
         """Write Manifest instance to disk, optionally signing it. Returns
@@ -488,10 +494,8 @@ class Manifest:
 
     def findFile(self, fname):
         """Return entrytype of the given file if present in Manifest or None if not present"""
-        for t in MANIFEST2_IDENTIFIERS:
-            if fname in self.fhashdict[t]:
-                return t
-        return None
+        found_entries = (t for t in MANIFEST2_IDENTIFIERS if fname in self.fhashdict[t])
+        return next(found_entries, None)
 
     def create(
         self,
@@ -529,18 +533,19 @@ class Manifest:
             find_invalid_path_char=self._find_invalid_path_char,
             strict_misc_digests=self.strict_misc_digests,
         )
-        pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
-        cat = self._pkgdir_category()
 
-        pkgdir = self.pkgdir
+        update_pkgdir = self._update_thick_pkgdir
         if self.thin:
-            cpvlist = self._update_thin_pkgdir(cat, pn, pkgdir)
-        else:
-            cpvlist = self._update_thick_pkgdir(cat, pn, pkgdir)
+            update_pkgdir = self._update_thin_pkgdir
 
-        distlist = set()
-        for cpv in cpvlist:
-            distlist.update(self._getCpvDistfiles(cpv))
+        cpvlist = update_pkgdir(
+            self._pkgdir_category(),
+            os.path.basename(self.pkgdir.rstrip(os.path.sep)),
+            self.pkgdir,
+        )
+        distlist = set(
+            distfile for cpv in cpvlist for distfile in self._getCpvDistfiles(cpv)
+        )
 
         if requiredDistfiles is None:
             # This allows us to force removal of stale digests for the
@@ -550,9 +555,7 @@ class Manifest:
             # repoman passes in an empty list, which implies that all distfiles
             # are required.
             requiredDistfiles = distlist.copy()
-        required_hash_types = set()
-        required_hash_types.add("size")
-        required_hash_types.update(self.required_hashes)
+        required_hash_types = set(itertools.chain(self.required_hashes, ("size")))
         for f in distlist:
             fname = os.path.join(self.distdir, f)
             mystat = None
@@ -599,25 +602,28 @@ class Manifest:
         return cpv
 
     def _update_thin_pkgdir(self, cat, pn, pkgdir):
-        for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
-            break
-        cpvlist = []
-        for f in pkgdir_files:
+        _, _, pkgdir_files = next(os.walk(pkgdir), (None, None, None))
+
+        def _process_for_cpv(filename):
             try:
-                f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
+                filename = _unicode_decode(
+                    filename, encoding=_encodings["fs"], errors="strict"
+                )
             except UnicodeDecodeError:
-                continue
-            if f[:1] == ".":
-                continue
-            pf = self._is_cpv(cat, pn, f)
+                return None
+            if filename.startswith("."):
+                return None
+            pf = self._is_cpv(cat, pn, filename)
             if pf is not None:
-                cpvlist.append(pf)
+                return pf
+
+        processed = (_process_for_cpv(filename) for filename in pkgdir_files)
+        cpvlist = [pf for pf in processed if pf]
         return cpvlist
 
     def _update_thick_pkgdir(self, cat, pn, pkgdir):
+        _, _, pkgdir_files = next(os.walk(pkgdir), (None, None, None))
         cpvlist = []
-        for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
-            break
         for f in pkgdir_files:
             try:
                 f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
@@ -714,9 +720,7 @@ class Manifest:
         return self.fetchlist_dict[cpv]
 
     def getDistfilesSize(self, fetchlist):
-        total_bytes = 0
-        for f in fetchlist:
-            total_bytes += int(self.fhashdict["DIST"][f]["size"])
+        total_bytes = sum(int(self.fhashdict["DIST"][f]["size"]) for f in fetchlist)
         return total_bytes
 
     def updateFileHashes(
@@ -784,28 +788,26 @@ class Manifest:
 
     def getVersions(self):
         """Returns a list of manifest versions present in the manifest file."""
-        rVal = []
         mfname = self.getFullname()
         if not os.path.exists(mfname):
-            return rVal
-        myfile = io.open(
+            return []
+        with io.open(
             _unicode_encode(mfname, encoding=_encodings["fs"], errors="strict"),
             mode="r",
             encoding=_encodings["repo.content"],
             errors="replace",
-        )
-        lines = myfile.readlines()
-        myfile.close()
-        for l in lines:
-            mysplit = l.split()
-            if (
-                len(mysplit) > 4
-                and mysplit[0] in MANIFEST2_IDENTIFIERS
-                and ((len(mysplit) - 3) % 2) == 0
-                and not 2 in rVal
-            ):
-                rVal.append(2)
-        return rVal
+        ) as myfile:
+            line_splits = (line.split() for line in myfile.readlines())
+            validation = (
+                True
+                for line_split in line_splits
+                if len(line_split) > 4
+                and line_split[0] in MANIFEST2_IDENTIFIERS
+                and (len(line_split) - 3) % 2 == 0
+            )
+            if any(validation):
+                return [2]
+        return []
 
     def _catsplit(self, pkg_key):
         """Split a category and package, returning a list of [cat, pkg].

diff --git a/lib/portage/metadata.py b/lib/portage/metadata.py
index 357917051..869c10bb3 100644
--- a/lib/portage/metadata.py
+++ b/lib/portage/metadata.py
@@ -75,8 +75,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
             eclass_db.update_eclasses()
             porttrees_data.append(TreeData(portdb.auxdb[path], eclass_db, path, src_db))
 
-    porttrees = [tree_data.path for tree_data in porttrees_data]
-
     quiet = (
         settings.get("TERM") == "dumb" or "--quiet" in myopts or not sys.stdout.isatty()
     )
@@ -100,7 +98,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
     # Temporarily override portdb.porttrees so portdb.cp_all()
     # will only return the relevant subset.
     portdb_porttrees = portdb.porttrees
-    portdb.porttrees = porttrees
+    portdb.porttrees = (tree_data.path for tree_data in porttrees_data)
     try:
         cp_all = portdb.cp_all()
     finally:
@@ -119,7 +117,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
 
     for cp in cp_all:
         for tree_data in porttrees_data:
-
             src_chf = tree_data.src_db.validation_chf
             dest_chf = tree_data.dest_db.validation_chf
             dest_chf_key = f"_{dest_chf}_"
@@ -190,11 +187,11 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
                         # We don't want to skip the write unless we're really
                         # sure that the existing cache is identical, so don't
                         # trust _mtime_ and _eclasses_ alone.
-                        for k in auxdbkeys:
-                            if dest.get(k, "") != src.get(k, ""):
-                                dest = None
-                                break
-
+                        cache_is_identical = (
+                            True for k in auxdbkeys if dest.get(k, "") != src.get(k, "")
+                        )
+                        if any(cache_is_identical):
+                            dest = None
                 if dest is not None:
                     # The existing data is valid and identical,
                     # so there's no need to overwrite it.

diff --git a/lib/portage/module.py b/lib/portage/module.py
index 61c85aa47..8e63cd545 100644
--- a/lib/portage/module.py
+++ b/lib/portage/module.py
@@ -110,19 +110,27 @@ class Modules:
         @rtype: dictionary of module_plugins
         """
         module_dir = self._module_path
-        importables = []
         names = os.listdir(module_dir)
-        for entry in names:
-            # skip any __init__ or __pycache__ files or directories
-            if entry.startswith("__"):
-                continue
+
+        def _a_real_module(entry):
             try:
                 # test for statinfo to ensure it should a real module
                 # it will bail if it errors
                 os.lstat(os.path.join(module_dir, entry, "__init__.py"))
-                importables.append(entry)
             except EnvironmentError:
-                pass
+                return False
+            return True
+
+        # The importables list cannot be a generator.
+        # If it was a generator, it would be consumed by self.parents.extend()
+        # and the following for loop wouldn't have anything to iterate with.
+        importables = [
+            entry
+            for entry in names
+            if not entry.startswith("__") and _a_real_module(entry)
+        ]
+        self.parents.extend(importables)
+
         kids = {}
         for entry in importables:
             new_module = Module(entry, self._namepath)
@@ -131,7 +139,6 @@ class Modules:
                 kid = new_module.kids[module_name]
                 kid["parent"] = new_module
                 kids[kid["name"]] = kid
-            self.parents.append(entry)
         return kids
 
     def get_module_names(self):

diff --git a/lib/portage/news.py b/lib/portage/news.py
index 9ef6efde0..9f373d3d7 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -280,14 +280,12 @@ class NewsItem:
 
         kwargs = {"vardb": vardb, "config": config, "profile": profile}
 
-        all_match = True
-        for values in self.restrictions.values():
-            any_match = False
-            for restriction in values:
-                if restriction.checkRestriction(**kwargs):
-                    any_match = True
-            if not any_match:
-                all_match = False
+        all_match = all(
+            True
+            for values in self.restrictions.values()
+            for restriction in values
+            if restriction.checkRestriction(**kwargs)
+        )
 
         return all_match
 

diff --git a/lib/portage/output.py b/lib/portage/output.py
index 33c477012..e20046fc5 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -5,6 +5,7 @@ __docformat__ = "epytext"
 
 import errno
 import io
+import itertools
 import re
 import subprocess
 import sys
@@ -74,16 +75,12 @@ codes["bg_darkyellow"] = codes["bg_brown"]
 
 
 def color(fg, bg="default", attr=["normal"]):
-    mystr = codes[fg]
-    for x in [bg] + attr:
-        mystr += codes[x]
-    return mystr
+    myansicodechain = itertools.chain((codes[fg]), (codes[x] for x in [bg, *attr]))
+    return "".join(myansicodechain)
 
 
-ansi_codes = []
-for x in range(30, 38):
-    ansi_codes.append("%im" % x)
-    ansi_codes.append("%i;01m" % x)
+ansi_codes = [y for x in range(30, 38) for y in (f"{x}m", f"{x};01m")]
+
 
 rgb_ansi_colors = [
     "0x000000",

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     eeaeb27707890e3f96baaf13493f95e6368f5f13
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:20:36 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:37 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=eeaeb277

Simplify by inlining a single use method

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index 4eb6dc18c..f0fe1fbca 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -182,11 +182,16 @@ class Manifest:
         self.fhashdict = {t: {} for t in MANIFEST2_IDENTIFIERS}
 
         if not from_scratch:
-            self._read()
-        if fetchlist_dict != None:
-            self.fetchlist_dict = fetchlist_dict
-        else:
-            self.fetchlist_dict = {}
+            # Parse Manifest file for this instance
+            try:
+                self._readManifest(self.getFullname(), myhashdict=self.fhashdict)
+            except FileNotFound:
+                pass
+
+        self.fetchlist_dict = {}
+        if fetchlist_dict:
+            self.fetchlist_dict.update(fetchlist_dict)
+
         self.distdir = distdir
         self.thin = thin
         if thin:
@@ -232,13 +237,6 @@ class Manifest:
             else:
                 raise
 
-    def _read(self):
-        """Parse Manifest file for this instance"""
-        try:
-            self._readManifest(self.getFullname(), myhashdict=self.fhashdict)
-        except FileNotFound:
-            pass
-
     def _parseManifestLines(self, mylines):
         """Parse manifest lines and return a list of manifest entries."""
         for myline in mylines:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     bf66319d572fbe00bea0c65001da57ee5f9c154a
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:27:30 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:41 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf66319d

Replace obscure lambdas with closures for clarity

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index 3555e08de..4dca536e4 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -395,30 +395,30 @@ class Manifest:
         # it always rounds down. Note that stat_result.st_mtime will round
         # up from 0.999999999 to 1.0 when precision is lost during conversion
         # from nanosecond resolution to float.
-        max_mtime = None
-        _update_max = (
-            lambda st: max_mtime
-            if max_mtime is not None and max_mtime > st[stat.ST_MTIME]
-            else st[stat.ST_MTIME]
-        )
-        _stat = (
-            lambda path: preserved_stats[path]
-            if path in preserved_stats
-            else os.stat(path)
-        )
 
+        def _update_max(max_mtime, st):
+            stat_mtime = st[stat.ST_MTIME]
+            if max_mtime:
+                return max(max_mtime, stat_mtime)
+
+        def _stat(path):
+            if path in preserved_stats:
+                return preserved_stats[path]
+            else:
+                return os.stat(path)
+
+        max_mtime = None
         for stat_result in preserved_stats.values():
-            max_mtime = _update_max(stat_result)
+            max_mtime = _update_max(max_mtime, stat_result)
 
         for entry in entries:
             if entry.type == "DIST":
                 continue
-            abs_path = (
-                os.path.join(self.pkgdir, "files", entry.name)
-                if entry.type == "AUX"
-                else os.path.join(self.pkgdir, entry.name)
-            )
-            max_mtime = _update_max(_stat(abs_path))
+            files = ""
+            if entry.type == "AUX":
+                files = "files"
+            abs_path = os.path.join(self.pkgdir, files, entry.name)
+            max_mtime = _update_max(max_mtime, _stat(abs_path))
 
         if not self.thin:
             # Account for changes to all relevant nested directories.
@@ -435,7 +435,7 @@ class Manifest:
                     # report such problems).
                     pass
                 else:
-                    max_mtime = _update_max(_stat(parent_dir))
+                    max_mtime = _update_max(max_mtime, _stat(parent_dir))
 
         if max_mtime is not None:
             for path in preserved_stats:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     579092cb02fc15a7047c3c3c2c28d6aae6ea2aeb
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:23:49 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:39 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=579092cb

Inline key-value insertions as dict expressions with key-value pairs

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py |  3 +--
 lib/portage/output.py   | 58 ++++++++++++++++++++++++-------------------------
 2 files changed, 29 insertions(+), 32 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index f0fe1fbca..3555e08de 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -317,8 +317,7 @@ class Manifest:
         try:
             myentries = list(self._createManifestEntries())
             update_manifest = True
-            preserved_stats = {}
-            preserved_stats[self.pkgdir.rstrip(os.sep)] = os.stat(self.pkgdir)
+            preserved_stats = {self.pkgdir.rstrip(os.sep): os.stat(self.pkgdir)}
             if myentries and not force:
                 try:
                     f = io.open(

diff --git a/lib/portage/output.py b/lib/portage/output.py
index e20046fc5..565f782ff 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -38,40 +38,38 @@ dotitles = 1
 _styles = {}
 """Maps style class to tuple of attribute names."""
 
-codes = {}
 """Maps attribute name to ansi code."""
 
 esc_seq = "\x1b["
 
-codes["normal"] = esc_seq + "0m"
-codes["reset"] = esc_seq + "39;49;00m"
-
-codes["bold"] = esc_seq + "01m"
-codes["faint"] = esc_seq + "02m"
-codes["standout"] = esc_seq + "03m"
-codes["underline"] = esc_seq + "04m"
-codes["blink"] = esc_seq + "05m"
-codes["overline"] = esc_seq + "06m"
-codes["reverse"] = esc_seq + "07m"
-codes["invisible"] = esc_seq + "08m"
-
-codes["no-attr"] = esc_seq + "22m"
-codes["no-standout"] = esc_seq + "23m"
-codes["no-underline"] = esc_seq + "24m"
-codes["no-blink"] = esc_seq + "25m"
-codes["no-overline"] = esc_seq + "26m"
-codes["no-reverse"] = esc_seq + "27m"
-
-codes["bg_black"] = esc_seq + "40m"
-codes["bg_darkred"] = esc_seq + "41m"
-codes["bg_darkgreen"] = esc_seq + "42m"
-codes["bg_brown"] = esc_seq + "43m"
-codes["bg_darkblue"] = esc_seq + "44m"
-codes["bg_purple"] = esc_seq + "45m"
-codes["bg_teal"] = esc_seq + "46m"
-codes["bg_lightgray"] = esc_seq + "47m"
-codes["bg_default"] = esc_seq + "49m"
-codes["bg_darkyellow"] = codes["bg_brown"]
+codes = {
+    "normal": f"{esc_seq}0m",
+    "reset": f"{esc_seq}39;49;00m",
+    "bold": f"{esc_seq}01m",
+    "faint": f"{esc_seq}02m",
+    "standout": f"{esc_seq}03m",
+    "underline": f"{esc_seq}04m",
+    "blink": f"{esc_seq}05m",
+    "overline": f"{esc_seq}06m",
+    "reverse": f"{esc_seq}07m",
+    "invisible": f"{esc_seq}08m",
+    "no-attr": f"{esc_seq}22m",
+    "no-standout": f"{esc_seq}23m",
+    "no-underline": f"{esc_seq}24m",
+    "no-blink": f"{esc_seq}25m",
+    "no-overline": f"{esc_seq}26m",
+    "no-reverse": f"{esc_seq}27m",
+    "bg_black": f"{esc_seq}40m",
+    "bg_darkred": f"{esc_seq}41m",
+    "bg_darkgreen": f"{esc_seq}42m",
+    "bg_brown": f"{esc_seq}43m",
+    "bg_darkblue": f"{esc_seq}44m",
+    "bg_purple": f"{esc_seq}45m",
+    "bg_teal": f"{esc_seq}46m",
+    "bg_lightgray": f"{esc_seq}47m",
+    "bg_default": f"{esc_seq}49m",
+    "bg_darkyellow": f"{esc_seq}43m",
+}
 
 
 def color(fg, bg="default", attr=["normal"]):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     64d84ce2d9a333e83e2a5fba5e7ec95f936959e7
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:32:42 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:45 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=64d84ce2

Miscellaneous refactors and cleanups

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Closes: https://github.com/gentoo/portage/pull/798
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 89 +++++++++++++++++++++++++------------------------
 lib/portage/news.py     |  9 +++--
 lib/portage/output.py   |  2 +-
 3 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index 655eabf68..eb3695669 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -83,13 +83,13 @@ def parseManifest2(line):
     if not isinstance(line, str):
         line = " ".join(line)
     myentry = None
-    match = _manifest_re.match(line)
-    if match is not None:
-        tokens = match.group(3).split()
+    matched = _manifest_re.match(line)
+    if matched:
+        tokens = matched.group(3).split()
         hashes = dict(zip(tokens[1::2], tokens[2::2]))
         hashes["size"] = int(tokens[0])
         myentry = Manifest2Entry(
-            type=match.group(1), name=match.group(2), hashes=hashes
+            type=matched.group(1), name=matched.group(2), hashes=hashes
         )
     return myentry
 
@@ -274,19 +274,21 @@ class Manifest:
 
     def _createManifestEntries(self):
         valid_hashes = set(itertools.chain(get_valid_checksum_keys(), ("size")))
-        mytypes = list(self.fhashdict)
-        mytypes.sort()
-        for t in mytypes:
-            myfiles = list(self.fhashdict[t])
-            myfiles.sort()
-            for f in myfiles:
-                myentry = Manifest2Entry(
-                    type=t, name=f, hashes=self.fhashdict[t][f].copy()
+        mytypes = sorted(self.fhashdict)
+        for mytype in mytypes:
+            myfiles = sorted(self.fhashdict[mytype])
+            for myfile in myfiles:
+                remainings = set(self.fhashdict[mytype][myfile]).intersection(
+                    valid_hashes
+                )
+                yield Manifest2Entry(
+                    type=mytype,
+                    name=myfile,
+                    hashes={
+                        remaining: self.fhashdict[mytype][myfile][remaining]
+                        for remaining in remainings
+                    },
                 )
-                for h in list(myentry.hashes):
-                    if h not in valid_hashes:
-                        del myentry.hashes[h]
-                yield myentry
 
     def checkIntegrity(self):
         manifest_data = (
@@ -320,7 +322,7 @@ class Manifest:
             preserved_stats = {self.pkgdir.rstrip(os.sep): os.stat(self.pkgdir)}
             if myentries and not force:
                 try:
-                    f = io.open(
+                    with io.open(
                         _unicode_encode(
                             self.getFullname(),
                             encoding=_encodings["fs"],
@@ -329,16 +331,15 @@ class Manifest:
                         mode="r",
                         encoding=_encodings["repo.content"],
                         errors="replace",
-                    )
-                    oldentries = list(self._parseManifestLines(f))
-                    preserved_stats[self.getFullname()] = os.fstat(f.fileno())
-                    f.close()
-                    if len(oldentries) == len(myentries):
-                        update_manifest = False
-                        for i in range(len(oldentries)):
-                            if oldentries[i] != myentries[i]:
-                                update_manifest = True
-                                break
+                    ) as f:
+                        oldentries = list(self._parseManifestLines(f))
+                        preserved_stats[self.getFullname()] = os.fstat(f.fileno())
+                        if len(oldentries) == len(myentries):
+                            update_manifest = False
+                            for oldentry, myentry in zip(oldentries, myentries):
+                                if oldentry != myentry:
+                                    update_manifest = True
+                                    break
                 except (IOError, OSError) as e:
                     if e.errno == errno.ENOENT:
                         pass
@@ -463,16 +464,17 @@ class Manifest:
 
     def addFile(self, ftype, fname, hashdict=None, ignoreMissing=False):
         """Add entry to Manifest optionally using hashdict to avoid recalculation of hashes"""
-        if ftype == "AUX" and not fname.startswith("files/"):
-            fname = os.path.join("files", fname)
-        if not os.path.exists(self.pkgdir + fname) and not ignoreMissing:
+        if ftype == "AUX":
+            if not fname.startswith("files/"):
+                fname = os.path.join("files", fname)
+            if fname.startswith("files"):
+                fname = fname[6:]
+        if not os.path.exists(f"{self.pkgdir}{fname}") and not ignoreMissing:
             raise FileNotFound(fname)
-        if not ftype in MANIFEST2_IDENTIFIERS:
+        if ftype not in MANIFEST2_IDENTIFIERS:
             raise InvalidDataType(ftype)
-        if ftype == "AUX" and fname.startswith("files"):
-            fname = fname[6:]
         self.fhashdict[ftype][fname] = {}
-        if hashdict != None:
+        if hashdict is not None:
             self.fhashdict[ftype][fname].update(hashdict)
         if self.required_hashes.difference(set(self.fhashdict[ftype][fname])):
             self.updateFileHashes(
@@ -497,7 +499,7 @@ class Manifest:
         checkExisting=False,
         assumeDistHashesSometimes=False,
         assumeDistHashesAlways=False,
-        requiredDistfiles=[],
+        requiredDistfiles=None,
     ):
         """Recreate this Manifest from scratch.  This will not use any
         existing checksums unless assumeDistHashesSometimes or
@@ -511,10 +513,9 @@ class Manifest:
             return
         if checkExisting:
             self.checkAllHashes()
+        distfilehashes = {}
         if assumeDistHashesSometimes or assumeDistHashesAlways:
-            distfilehashes = self.fhashdict["DIST"]
-        else:
-            distfilehashes = {}
+            distfilehashes.update(self.fhashdict["DIST"])
         self.__init__(
             self.pkgdir,
             distdir=self.distdir,
@@ -624,7 +625,7 @@ class Manifest:
                 f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
             except UnicodeDecodeError:
                 continue
-            if f[:1] == ".":
+            if f.startswith("."):
                 continue
             pf = self._is_cpv(cat, pn, f)
             if pf is not None:
@@ -640,7 +641,7 @@ class Manifest:
         recursive_files = []
 
         pkgdir = self.pkgdir
-        cut_len = len(os.path.join(pkgdir, "files") + os.sep)
+        cut_len = len(os.path.join(pkgdir, "files", os.sep))
         for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
             for f in files:
                 try:
@@ -662,12 +663,12 @@ class Manifest:
 
     def _getAbsname(self, ftype, fname):
         if ftype == "DIST":
-            absname = os.path.join(self.distdir, fname)
+            abspath = (self.distdir, fname)
         elif ftype == "AUX":
-            absname = os.path.join(self.pkgdir, "files", fname)
+            abspath = (self.pkgdir, "files", fname)
         else:
-            absname = os.path.join(self.pkgdir, fname)
-        return absname
+            abspath = (self.pkgdir, fname)
+        return os.path.join(*abspath)
 
     def checkAllHashes(self, ignoreMissingFiles=False):
         for t in MANIFEST2_IDENTIFIERS:

diff --git a/lib/portage/news.py b/lib/portage/news.py
index 801edb68c..132e050f2 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -80,7 +80,7 @@ class NewsManager:
         portdir = portdb.repositories.mainRepoLocation()
         profiles_base = None
         if portdir is not None:
-            profiles_base = os.path.join(portdir, "profiles") + os.path.sep
+            profiles_base = os.path.join(portdir, "profiles", os.path.sep)
         profile_path = None
         if profiles_base is not None and portdb.settings.profile_path:
             profile_path = normalize_path(
@@ -295,14 +295,13 @@ class NewsItem:
         return self._valid
 
     def parse(self):
-        f = io.open(
+        with io.open(
             _unicode_encode(self.path, encoding=_encodings["fs"], errors="strict"),
             mode="r",
             encoding=_encodings["content"],
             errors="replace",
-        )
-        lines = f.readlines()
-        f.close()
+        ) as f:
+            lines = f.readlines()
         self.restrictions = {}
         invalids = []
         news_format = None

diff --git a/lib/portage/output.py b/lib/portage/output.py
index 565f782ff..c7922038e 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -35,8 +35,8 @@ from portage.localization import _
 havecolor = 1
 dotitles = 1
 
-_styles = {}
 """Maps style class to tuple of attribute names."""
+_styles = {}
 
 """Maps attribute name to ansi code."""
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     5ac2a8f08101501d0416ad9d79e277c363cea85e
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:29:10 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:43 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5ac2a8f0

Enhance the update hash methods for simplification

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 85 +++++++++++++++++++++++++++----------------------
 1 file changed, 47 insertions(+), 38 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index 4dca536e4..655eabf68 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -718,63 +718,72 @@ class Manifest:
         total_bytes = sum(int(self.fhashdict["DIST"][f]["size"]) for f in fetchlist)
         return total_bytes
 
-    def updateFileHashes(
-        self, ftype, fname, checkExisting=True, ignoreMissing=True, reuseExisting=False
+    def updateAllFileHashes(
+        self, ftype, fnames, checkExisting=True, ignoreMissing=True, reuseExisting=False
     ):
-        """Regenerate hashes for the given file"""
-        if checkExisting:
-            self.checkFileHashes(ftype, fname, ignoreMissing=ignoreMissing)
-        if not ignoreMissing and fname not in self.fhashdict[ftype]:
-            raise FileNotInManifestException(fname)
-        if fname not in self.fhashdict[ftype]:
-            self.fhashdict[ftype][fname] = {}
-        myhashkeys = list(self.hashes)
-        if reuseExisting:
-            for k in [h for h in self.fhashdict[ftype][fname] if h in myhashkeys]:
-                myhashkeys.remove(k)
-        myhashes = perform_multiple_checksums(
-            self._getAbsname(ftype, fname), myhashkeys
-        )
-        self.fhashdict[ftype][fname].update(myhashes)
+        """Regenerate hashes from a list of files"""
+        for fname in fnames:
+            if checkExisting:
+                self.checkFileHashes(ftype, fname, ignoreMissing=ignoreMissing)
+            if not ignoreMissing and fname not in self.fhashdict[ftype]:
+                raise FileNotInManifestException(fname)
+            if fname not in self.fhashdict[ftype]:
+                self.fhashdict[ftype][fname] = {}
+            myhashkeys = self.hashes
+            if reuseExisting:
+                myhashkeys = myhashkeys.difference(self.fhashdict[ftype][fname])
+            myhashes = perform_multiple_checksums(
+                self._getAbsname(ftype, fname), myhashkeys
+            )
+            self.fhashdict[ftype][fname].update(myhashes)
 
-    def updateTypeHashes(self, idtype, checkExisting=False, ignoreMissingFiles=True):
-        """Regenerate all hashes for all files of the given type"""
-        for fname in self.fhashdict[idtype]:
-            self.updateFileHashes(idtype, fname, checkExisting)
+    def updateAllTypeHashes(
+        self, idtypes, checkExisting=False, ignoreMissingFiles=True
+    ):
+        """Regenerate all hashes for all files from a list of types"""
+        for idtype in idtypes:
+            self.updateAllFileHashes(
+                ftype=idtype, fnames=self.fhashdict[idtype], checkExisting=checkExisting
+            )
 
     def updateAllHashes(self, checkExisting=False, ignoreMissingFiles=True):
         """Regenerate all hashes for all files in this Manifest."""
-        for idtype in MANIFEST2_IDENTIFIERS:
-            self.updateTypeHashes(
-                idtype,
-                checkExisting=checkExisting,
-                ignoreMissingFiles=ignoreMissingFiles,
-            )
+        self.updateTypeHashes(
+            idtypes=MANIFEST2_IDENTIFIERS,
+            checkExisting=checkExisting,
+            ignoreMissingFiles=ignoreMissingFiles,
+        )
 
     def updateCpvHashes(self, cpv, ignoreMissingFiles=True):
         """Regenerate all hashes associated to the given cpv (includes all AUX and MISC
         files)."""
-        self.updateTypeHashes("AUX", ignoreMissingFiles=ignoreMissingFiles)
-        self.updateTypeHashes("MISC", ignoreMissingFiles=ignoreMissingFiles)
-        ebuildname = "%s.ebuild" % self._catsplit(cpv)[1]
-        self.updateFileHashes(
-            "EBUILD", ebuildname, ignoreMissingFiles=ignoreMissingFiles
+        self.updateAllTypeHashes(
+            idtypes=("AUX", "MISC"),
+            ignoreMissingFiles=ignoreMissingFiles,
+        )
+        self.updateAllFileHashes(
+            ftype="EBUILD",
+            fnames=(f"{self._catsplit(cpv)[1]}.ebuild",),
+            ignoreMissingFiles=ignoreMissingFiles,
+        )
+        self.updateAllFileHashes(
+            ftype="DIST",
+            fnames=self._getCpvDistfiles(cpv),
+            ignoreMissingFiles=ignoreMissingFiles,
         )
-        for f in self._getCpvDistfiles(cpv):
-            self.updateFileHashes("DIST", f, ignoreMissingFiles=ignoreMissingFiles)
 
     def updateHashesGuessType(self, fname, *args, **kwargs):
         """Regenerate hashes for the given file (guesses the type and then
         calls updateFileHashes)."""
         mytype = self.guessType(fname)
-        if mytype == "AUX":
-            fname = fname[len("files" + os.sep) :]
-        elif mytype is None:
+        if mytype is None:
             return
+        elif mytype == "AUX":
+            fname = fname[len(f"files{os.sep}") :]
         myrealtype = self.findFile(fname)
         if myrealtype is not None:
             mytype = myrealtype
-        return self.updateFileHashes(mytype, fname, *args, **kwargs)
+        return self.updateAllFileHashes(ftype=mytype, fnames=(fname,), *args, **kwargs)
 
     def getFileData(self, ftype, fname, key):
         """Return the value of a specific (type,filename,key) triple, mainly useful

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     a387219c4bdc1510e7958193203fcd29acf6c173
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:16:57 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:36 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a387219c

Return boolean expressions instead of branching

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 14 ++++++--------
 lib/portage/news.py     | 22 ++++++++--------------
 2 files changed, 14 insertions(+), 22 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index ff166faa8..4eb6dc18c 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -112,14 +112,12 @@ class Manifest2Entry(ManifestEntry):
         return f"{myline} {with_hashes}"
 
     def __eq__(self, other):
-        if (
-            not isinstance(other, Manifest2Entry)
-            or self.type != other.type
-            or self.name != other.name
-            or self.hashes != other.hashes
-        ):
-            return False
-        return True
+        return (
+            isinstance(other, Manifest2Entry)
+            and self.type == other.type
+            and self.name == other.name
+            and self.hashes == other.hashes
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)

diff --git a/lib/portage/news.py b/lib/portage/news.py
index 9f373d3d7..801edb68c 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -382,13 +382,12 @@ class DisplayProfileRestriction(DisplayRestriction):
         self.format = news_format
 
     def isValid(self):
-        if fnmatch.fnmatch(self.format, "1.*") and "*" in self.profile:
-            return False
-        if fnmatch.fnmatch(self.format, "2.*") and not _valid_profile_RE.match(
-            self.profile
-        ):
-            return False
-        return True
+        return (
+            not fnmatch.fnmatch(self.format, "1.*")
+            or "*" not in self.profile
+            and not fnmatch.fnmatch(self.format, "2.*")
+            or _valid_profile_RE.match(self.profile)
+        )
 
     def checkRestriction(self, **kwargs):
         if fnmatch.fnmatch(self.format, "2.*") and self.profile.endswith("/*"):
@@ -407,9 +406,7 @@ class DisplayKeywordRestriction(DisplayRestriction):
         self.format = news_format
 
     def checkRestriction(self, **kwargs):
-        if kwargs["config"].get("ARCH", "") == self.keyword:
-            return True
-        return False
+        return kwargs["config"].get("ARCH", "") == self.keyword
 
 
 class DisplayInstalledRestriction(DisplayRestriction):
@@ -430,10 +427,7 @@ class DisplayInstalledRestriction(DisplayRestriction):
         return isvalidatom(self.atom)
 
     def checkRestriction(self, **kwargs):
-        vdb = kwargs["vardb"]
-        if vdb.match(self.atom):
-            return True
-        return False
+        return kwargs["vardb"].match(self.atom)
 
 
 def count_unread_news(portdb, vardb, repos=None, update=True):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     b4ca7c5802aafab54ca163c8e59ac3d2222e3084
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  9 01:42:32 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Apr  9 04:32:12 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b4ca7c58

manifest: fix thick manifest generation

Closes: https://bugs.gentoo.org/837326
Closes: https://bugs.gentoo.org/837323
Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index eb3695669..a5b26c130 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -641,7 +641,7 @@ class Manifest:
         recursive_files = []
 
         pkgdir = self.pkgdir
-        cut_len = len(os.path.join(pkgdir, "files", os.sep))
+        cut_len = len(os.path.join(pkgdir, f"files{os.sep}"))
         for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
             for f in files:
                 try:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     3d63920030ec24996d6678ca0d7bce24abab6383
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 10 14:36:26 2022 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Mon Apr 11 12:11:19 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d639200

dispatch-conf: fix diffstatusoutput_mixed() usage

This function should accept a command placeholder and two arguments.

The command placeholder is used in 2 places:
1. User config: dispatch-conf.conf
2. Inline in portage.dispatch_conf.file_archive()

We need to maintain the command placeholder format becaused of usage 1.

Fixes: 7c8875c38b187fbd2e0fa5fc39bbb38c1588d0fb
Bug: https://bugs.gentoo.org/836447
Bug: https://bugs.gentoo.org/837656
Closes: https://github.com/gentoo/portage/pull/805
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/dispatch_conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py
index 6dbb1885e..ec2bc84d3 100644
--- a/lib/portage/dispatch_conf.py
+++ b/lib/portage/dispatch_conf.py
@@ -340,7 +340,7 @@ def file_archive(archive, curconf, newconf, mrgconf):
     # Archive the current config file if it isn't already saved
     if (
         os.path.lexists(archive)
-        and len(diffstatusoutput_mixed(f"diff -aq '{curconf}' '{archive}'")[1]) != 0
+        and len(diffstatusoutput_mixed("diff -aq '%s' '%s'", curconf, archive)[1]) != 0
     ):
         _file_archive_rotate(archive)
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     51df6a3a490c46bcb40939be0691f59e2edbbd7b
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 10 14:29:09 2022 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Mon Apr 11 12:11:05 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=51df6a3a

Revert "dispatch_conf: fix arguments"

This is a step in the wrong direction, and causes another failure.

Reverts: 94dad3d64599f7e2c0c28e90ec4bee883746e385.
Bug: https://bugs.gentoo.org/836447
Bug: https://bugs.gentoo.org/837656
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/dispatch_conf.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py
index fc130c9af..6dbb1885e 100644
--- a/lib/portage/dispatch_conf.py
+++ b/lib/portage/dispatch_conf.py
@@ -28,15 +28,14 @@ RCS_GET = "co"
 _ARCHIVE_ROTATE_MAX = 9
 
 
-def diffstatusoutput(file1, file2):
+def diffstatusoutput(cmd, file1, file2):
     """
     Execute the string cmd in a shell with getstatusoutput() and return a
     2-tuple (status, output).
     """
     # Use Popen to emulate getstatusoutput(), since getstatusoutput() may
     # raise a UnicodeDecodeError which makes the output inaccessible.
-    cmd = f"diff -aq '{file1}' '{file2}'"
-    args = shlex_split(cmd)
+    args = shlex_split(cmd % (file1, file2))
 
     args = (portage._unicode_encode(x, errors="strict") for x in args)
     proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -341,7 +340,7 @@ def file_archive(archive, curconf, newconf, mrgconf):
     # Archive the current config file if it isn't already saved
     if (
         os.path.lexists(archive)
-        and len(diffstatusoutput_mixed(curconf, archive)[1]) != 0
+        and len(diffstatusoutput_mixed(f"diff -aq '{curconf}' '{archive}'")[1]) != 0
     ):
         _file_archive_rotate(archive)
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     2feeb57c0a260d308b55c8be943c73fc1fa44910
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  5 20:20:18 2022 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  5 20:24:49 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2feeb57c

Manifest: Fix ("size",) tuple

Fixes: 9e24d0143450 ("Simplify with declarative programming")
Bug: https://bugs.gentoo.org/849815
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/manifest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index a5b26c130..65dda211b 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -273,7 +273,7 @@ class Manifest:
         return myhashdict
 
     def _createManifestEntries(self):
-        valid_hashes = set(itertools.chain(get_valid_checksum_keys(), ("size")))
+        valid_hashes = set(itertools.chain(get_valid_checksum_keys(), ("size",)))
         mytypes = sorted(self.fhashdict)
         for mytype in mytypes:
             myfiles = sorted(self.fhashdict[mytype])
@@ -551,7 +551,7 @@ class Manifest:
             # repoman passes in an empty list, which implies that all distfiles
             # are required.
             requiredDistfiles = distlist.copy()
-        required_hash_types = set(itertools.chain(self.required_hashes, ("size")))
+        required_hash_types = set(itertools.chain(self.required_hashes, ("size",)))
         for f in distlist:
             fname = os.path.join(self.distdir, f)
             mystat = None

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     1a25cc20a662d5dbcbaa18b4f10093f95428b1d3
Author:     Rin Cat (鈴猫) <syu.os <AT> protonmail <DOT> com>
AuthorDate: Sun Jul  3 21:10:29 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Jul  5 22:56:05 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1a25cc20

Fix _check_pre_quickpkg_files image prefix

Closes: https://bugs.gentoo.org/851276
Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/847
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index fc111e44a..0622feb70 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1847,11 +1847,12 @@ class gpkg:
             image_total_size,
         )
 
-    def _check_pre_quickpkg_files(self, contents, root):
+    def _check_pre_quickpkg_files(self, contents, root, image_prefix="image"):
         """
         Check the pre quickpkg files size and path, return the longest
         path length, largest single file size, and total files size.
         """
+        image_prefix_length = len(image_prefix) + 1
         root_dir = os.path.join(
             normalize_path(
                 _unicode_decode(root, encoding=_encodings["fs"], errors="strict")
@@ -1881,6 +1882,7 @@ class gpkg:
             prefix_length = (
                 len(_unicode_encode(d, encoding=_encodings["fs"], errors="strict"))
                 - root_dir_length
+                + image_prefix_length
             )
             image_max_prefix_length = max(image_max_prefix_length, prefix_length)
 
@@ -1892,6 +1894,7 @@ class gpkg:
             path_length = (
                 len(_unicode_encode(path, encoding=_encodings["fs"], errors="strict"))
                 - root_dir_length
+                + image_prefix_length
             )
 
             file_stat = os.lstat(path)

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     d75a411aaec59a203b98246711377807a2d548d0
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Sat Jul  9 20:13:23 2022 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Sat Jul  9 20:26:18 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d75a411a

portage.eapi: remove debug logging statement

This ensures we don't accidentally break portage API consumers that fail
to configure the root logger before calling the portage API.

Bug: https://bugs.gentoo.org/838406
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/eapi.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py
index e3c951453..2c1701870 100644
--- a/lib/portage/eapi.py
+++ b/lib/portage/eapi.py
@@ -2,7 +2,6 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import collections
-import logging
 from functools import lru_cache
 from typing import Optional
 
@@ -219,7 +218,6 @@ def _get_eapi_attrs(eapi_str: Optional[str]) -> _eapi_attrs:
     be helpful for handling of corrupt EAPI metadata in essential functions
     such as pkgsplit.
     """
-    logging.info("cache info: {}".format(_get_eapi_attrs.cache_info()))
     if eapi_str is None or not eapi_is_supported(eapi_str):
         return _eapi_attrs(
             allows_package_provided=True,

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     1ffaa70544f34e93df24c0a175105a900bf272bf
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 11 04:50:24 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Jul 11 23:01:04 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1ffaa705

news: fix isRelevant check

Manifested as all news items being shown, even if
(very) irrelevant to the running system (e.g.
different arch, packages not installed, ...).

I think the distinction here is that with the previous state,
we'd end up with _only_ Trues, or nothing (an element
would be omitted), whereas this commit means we end
up considering a possible mixed sequence.

Closes: https://bugs.gentoo.org/857669
Fixes: 9e24d0143450628f334cdb62e579efafd1bfd2ba
Reported-by: kurly
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/news.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/portage/news.py b/lib/portage/news.py
index 132e050f2..6b2996c31 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -281,10 +281,9 @@ class NewsItem:
         kwargs = {"vardb": vardb, "config": config, "profile": profile}
 
         all_match = all(
-            True
+            restriction.checkRestriction(**kwargs)
             for values in self.restrictions.values()
             for restriction in values
-            if restriction.checkRestriction(**kwargs)
         )
 
         return all_match

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     b3ddadeb9bc0d1cfadcde8c7abfc2cb2d7f545fc
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 15 10:31:59 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Jul 18 18:47:37 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b3ddadeb

portage: drop obsolete fixpackages warning

18e5a8170c69aecd10f162918de571d85055ae81 exposed this but
the support for fixpackages actually got removed
in 7ab4d5722a828167dd1bf946b26cfa80808f59fc.

The "if True" construct was just intended as a way
of stubbing out the old stuff. Let's just drop the
cruft instead.

This fixes giving a completely harmless "fixpackages error"
which is meaningless.

See: https://forums.gentoo.org/viewtopic-t-1152818.html
Bug: https://bugs.gentoo.org/858407
Signed-off-by: Sam James <sam <AT> gentoo.org>
Closes: https://github.com/gentoo/portage/pull/856
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/_global_updates.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py
index 59e0b263e..05c13920f 100644
--- a/lib/portage/_global_updates.py
+++ b/lib/portage/_global_updates.py
@@ -261,13 +261,4 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
         # what follows.
         writemsg_stdout("\n\n")
 
-        if bindb and bindb.cpv_all():
-            writemsg_stdout(
-                _(
-                    " ** Skipping packages. Run 'fixpackages' or set it in FEATURES to fix the tbz2's in the packages directory.\n"
-                )
-            )
-            writemsg_stdout(bold(_("Note: This can take a very long time.")))
-            writemsg_stdout("\n")
-
     return retupd

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     12ebaf02dda97b5e088e661bd3dd5ccd0a10b6fa
Author:     Miezhiko <Miezhiko <AT> gmail <DOT> com>
AuthorDate: Tue Jul 19 07:10:24 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Jul 19 21:39:35 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=12ebaf02

checksum.py: discard can't be called on frozenset

Closes: https://github.com/gentoo/portage/pull/860
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index b4644da04..03a963a3e 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -441,7 +441,7 @@ def _apply_hash_filter(digests, hash_filter):
     @type hash_filter: callable
     """
 
-    verifiable_hash_types = frozenset(digests).intersection(hashfunc_keys)
+    verifiable_hash_types = set(digests).intersection(hashfunc_keys)
     modified = False
     if len(verifiable_hash_types) > 1:
         verifiable_hash_types.discard("size")

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     88cdf59280dafb5925d1444863b09d656c6ef46e
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  1 17:33:49 2022 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Mon Aug  1 17:33:49 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=88cdf592

portage.eapi: remove reference to logging module

Fixes: 205d41608e55f78b620f8c3a0cb204c88df041ee
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/eapi.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py
index 969cf9027..2c1701870 100644
--- a/lib/portage/eapi.py
+++ b/lib/portage/eapi.py
@@ -7,8 +7,6 @@ from typing import Optional
 
 from portage import eapi_is_supported
 
-logger = logging.getLogger(__name__)
-
 
 def eapi_has_iuse_defaults(eapi: str) -> bool:
     return _get_eapi_attrs(eapi).iuse_defaults

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     b9ef74a27bb847e8afe9fec5e6689d7934aeb16b
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 28 01:23:46 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Aug  1 22:38:11 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9ef74a2

portage: output: update functions.sh path

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/output.py b/lib/portage/output.py
index c7922038e..439cc4fac 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -134,7 +134,7 @@ codes["0xAAAA00"] = codes["brown"]
 codes["darkyellow"] = codes["0xAAAA00"]
 
 
-# Colors from /etc/init.d/functions.sh
+# Colors from /lib/gentoo/functions.sh
 _styles["BAD"] = ("red",)
 _styles["BRACKET"] = ("blue",)
 _styles["ERR"] = ("red",)

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     62f35411d92d487339549a6ee1353ad3026c6579
Author:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 18 15:01:31 2022 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Sun Sep 18 17:30:07 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=62f35411

Revert changes to portage.checksum._apply_hash_filter

The modified code may discard the file size from the output, which is
never desired.

Also, it calls remove on the object being iterated over, which is
problematic.

Reverts: 18e5a8170c69aecd10f162918de571d85055ae81
Reported-by: Marek Behún <kabel <AT> kernel.org>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/checksum.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index 03a963a3e..85cc36af3 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -442,10 +442,10 @@ def _apply_hash_filter(digests, hash_filter):
     """
 
     verifiable_hash_types = set(digests).intersection(hashfunc_keys)
+    verifiable_hash_types.discard("size")
     modified = False
     if len(verifiable_hash_types) > 1:
-        verifiable_hash_types.discard("size")
-        for k in verifiable_hash_types:
+        for k in list(verifiable_hash_types):
             if not hash_filter(k):
                 modified = True
                 verifiable_hash_types.remove(k)
@@ -453,7 +453,11 @@ def _apply_hash_filter(digests, hash_filter):
                     break
 
     if modified:
-        digests = {k: v for k, v in digests.items() if k in verifiable_hash_types}
+        digests = {
+            k: v
+            for k, v in digests.items()
+            if k == "size" or k in verifiable_hash_types
+        }
 
     return digests
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     355b6598225e3ffa921d2fb6646539bcbcc694a7
Author:     Sheng Yu <syu.os <AT> protonmail <DOT> com>
AuthorDate: Thu Sep 15 15:52:31 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Sep 20 03:39:23 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=355b6598

GPKG quickpkg allow missing files

Bug: https://bugs.gentoo.org/870229
Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/900
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 5f8e19341..c0a80208f 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -33,7 +33,7 @@ from portage.exception import (
     MissingSignature,
     InvalidSignature,
 )
-from portage.output import colorize
+from portage.output import colorize, EOutput
 from portage.util._urlopen import urlopen
 from portage.util import writemsg
 from portage.util import shlex_split, varexpand
@@ -1271,6 +1271,7 @@ class gpkg:
         Will compress the given files to image with root,
         ignoring all other files.
         """
+        eout = EOutput()
 
         protect_file = io.BytesIO(
             b"# empty file because --include-config=n when `quickpkg` was used\n"
@@ -1284,7 +1285,7 @@ class gpkg:
 
         # Get pre image info
         container_tar_format, image_tar_format = self._get_tar_format_from_stats(
-            *self._check_pre_quickpkg_files(contents, root_dir)
+            *self._check_pre_quickpkg_files(contents, root_dir, ignore_missing=True)
         )
 
         # Long CPV
@@ -1341,6 +1342,7 @@ class gpkg:
                     except OSError as e:
                         if e.errno != errno.ENOENT:
                             raise
+                        eout.ewarn(f'Missing file from local system: "{path}"')
                         del e
                         continue
                     contents_type = contents[path][0]
@@ -1984,7 +1986,9 @@ class gpkg:
             image_total_size,
         )
 
-    def _check_pre_quickpkg_files(self, contents, root, image_prefix="image"):
+    def _check_pre_quickpkg_files(
+        self, contents, root, image_prefix="image", ignore_missing=False
+    ):
         """
         Check the pre quickpkg files size and path, return the longest
         path length, largest single file size, and total files size.
@@ -2034,6 +2038,12 @@ class gpkg:
                 + image_prefix_length
             )
 
+            if not os.path.exists(path):
+                if ignore_missing:
+                    continue
+                else:
+                    raise FileNotFound(path)
+
             file_stat = os.lstat(path)
 
             if os.path.islink(path):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     f8cb1ef31318d63f183c237f9a129efe77d91f98
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 20 03:50:24 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Sep 20 19:45:51 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f8cb1ef3

getbinpkg: fix command list with --getbinpkgonly

Observable with say, PORTAGE_BINHOST="rsync://../" set.

Bug: https://bugs.gentoo.org/866197
Fixes: 13740f43bcc38e787153d9efeabc4f5d878e7af0
Thanks-to: Sheng Yu <syu.os <AT> protonmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>
Closes: https://github.com/gentoo/portage/pull/904
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/getbinpkg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/getbinpkg.py b/lib/portage/getbinpkg.py
index ea9ee1d0a..799f5b171 100644
--- a/lib/portage/getbinpkg.py
+++ b/lib/portage/getbinpkg.py
@@ -529,7 +529,7 @@ def file_get(
     from portage.util import varexpand
     from portage.process import spawn
 
-    myfetch = (varexpand(x, mydict=variables) for x in portage.util.shlex_split(fcmd))
+    myfetch = [varexpand(x, mydict=variables) for x in portage.util.shlex_split(fcmd)]
     fd_pipes = {
         0: portage._get_stdin().fileno(),
         1: sys.__stdout__.fileno(),

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     73514c310754def1f8cb932f00296e008b9d4437
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Sep 26 17:40:33 2022 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Sep 26 17:44:33 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=73514c31

data.py: fix PORTAGE_USERNAME default (bug 873088)

Fix the PORTAGE_USERNAME default for the case where the portage.settings
attribute exists and PORTAGE_USERNAME is unset.

Fixes: 18e5a8170c69aecd10f162918de571d85055ae81
Bug: https://bugs.gentoo.org/873088
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/data.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/data.py b/lib/portage/data.py
index 1ef8d4aef..8aa66df7f 100644
--- a/lib/portage/data.py
+++ b/lib/portage/data.py
@@ -259,7 +259,7 @@ def _get_global(k):
         if env_key in os.environ:
             v = os.environ[env_key]
         elif hasattr(portage, "settings"):
-            v = portage.settings.get(env_key)
+            v = portage.settings.get(env_key, v)
         else:
             # The config class has equivalent code, but we also need to
             # do it here if _disable_legacy_globals() has been called.

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     d4df27b8897e7c5bb508d8a2be3f3c2cb82628e8
Author:     Dima S <dimonade <AT> protonmail <DOT> com>
AuthorDate: Sat Sep 24 18:01:26 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Sep 28 23:56:08 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d4df27b8

portage: news: reformatted for PEP8; add docstrings; add type annotation

Signed-off-by: Dima S <dimonade <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/908
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/news.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/portage/news.py b/lib/portage/news.py
index 6b2996c31..7964f74d3 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -474,20 +474,23 @@ def count_unread_news(portdb, vardb, repos=None, update=True):
     return news_counts
 
 
-def display_news_notifications(news_counts):
+def display_news_notifications(news_counts: dict):
     """
     Display a notification for unread news items, using a dictionary mapping
     repos to integer counts, like that returned from count_unread_news().
+
+    @param news_count: mapping of repos to integer counts of unread news items
+    @type news_count: dict
     """
-    newsReaderDisplay = False
+    news_reader_display = False
     for repo, count in news_counts.items():
         if count > 0:
-            if not newsReaderDisplay:
-                newsReaderDisplay = True
+            if not news_reader_display:
+                news_reader_display = True
                 print()
             print(colorize("WARN", " * IMPORTANT:"), end=" ")
             print(f"{count} news items need reading for repository '{repo}'.")
 
-    if newsReaderDisplay:
+    if news_reader_display:
         print(colorize("WARN", " *"), end=" ")
         print(f"Use {colorize('GOOD', 'eselect news read')} to view new items.\n")

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     9ec692a666ff8ddd30cbf12dbe66867c5084ac10
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Thu Sep 29 06:37:19 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Thu Sep 29 20:45:40 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9ec692a6

lib: portage: gpkg: fix typo in comment

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index ff2195854..7e1b98370 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1778,7 +1778,7 @@ class gpkg:
 
     def _get_binary_cmd(self, compression, mode):
         """
-        get command list form portage and try match compressor
+        get command list from portage and try match compressor
         """
         if compression not in _compressors:
             raise InvalidCompressionMethod(compression)

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     526c4f7b662646a90add74f64c318875d1ffb4e0
Author:     Vitaliy Perekhovy <feriman <AT> gmail <DOT> com>
AuthorDate: Thu Sep 29 12:25:16 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Thu Sep 29 21:37:20 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=526c4f7b

_global_updates.py: fix syntax error

Due to missed comma operator, indentation between
%='binary move' and S='binary SLOT move' is gone.

Signed-off-by: Vitaliy Perekhovy <feriman <AT> gmail.com>
Closes: https://github.com/gentoo/portage/pull/917
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/_global_updates.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py
index 05c13920f..feb87c3d1 100644
--- a/lib/portage/_global_updates.py
+++ b/lib/portage/_global_updates.py
@@ -105,7 +105,7 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
                                     f"{bold('#')}='/var/db update'",
                                     f"{bold('@')}='/var/db move'\n",
                                     f"{bold('s')}='/var/db SLOT move'",
-                                    f"{bold('%')}='binary move'"
+                                    f"{bold('%')}='binary move'",
                                     f"{bold('S')}='binary SLOT move'\n",
                                     f"{bold('p')}='update /etc/portage/package.*'\n",
                                 )

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     5f256ab11e5510f98d7951ac9770948e00c36ed5
Author:     Sheng Yu <syu.os <AT> protonmail <DOT> com>
AuthorDate: Wed Nov  2 19:05:29 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Nov  2 22:58:15 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5f256ab1

Unconditionally update first level of dir name in binpkgs

Enabling / disabling binpkg-multi-instance while doing metadata move /
update may cause binary package structure mismatch.

This will unconditional replace the first level name (cpv) in the
package.

Closes: https://bugs.gentoo.org/877271
Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/929
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 36 +++++++++++++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 5a8c16186..3d2aa4223 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1048,6 +1048,14 @@ class gpkg:
             self._add_signature(checksum_info, image_tarinfo, container)
 
         self._add_manifest(container)
+
+        # Check if all directories are the same in the container
+        prefix = os.path.commonpath(container.getnames())
+        if not prefix:
+            raise InvalidBinaryPackageFormat(
+                f"gpkg file structure mismatch in {self.gpkg_file}"
+            )
+
         container.close()
 
     def decompress(self, decompress_dir):
@@ -1132,9 +1140,9 @@ class gpkg:
                         os.path.join(self.prefix, file_name_old)
                     )
                     new_data_tarinfo = copy(old_data_tarinfo)
-                    new_data_tarinfo.name = new_data_tarinfo.name.replace(
-                        old_basename, new_basename, 1
-                    )
+                    new_file_path = list(os.path.split(new_data_tarinfo.name))
+                    new_file_path[0] = new_basename
+                    new_data_tarinfo.name = os.path.join(*new_file_path)
                     container.addfile(
                         new_data_tarinfo, container_old.extractfile(old_data_tarinfo)
                     )
@@ -1142,6 +1150,13 @@ class gpkg:
 
             self._add_manifest(container)
 
+            # Check if all directories are the same in the container
+            prefix = os.path.commonpath(container.getnames())
+            if not prefix:
+                raise InvalidBinaryPackageFormat(
+                    f"gpkg file structure mismatch in {self.gpkg_file}"
+                )
+
         shutil.move(tmp_gpkg_file_name, self.gpkg_file)
 
     def update_signature(self, keep_current_signature=False):
@@ -1222,6 +1237,13 @@ class gpkg:
 
             self._add_manifest(container)
 
+            # Check if all directories are the same in the container
+            prefix = os.path.commonpath(container.getnames())
+            if not prefix:
+                raise InvalidBinaryPackageFormat(
+                    f"gpkg file structure mismatch in {self.gpkg_file}"
+                )
+
         shutil.move(tmp_gpkg_file_name, self.gpkg_file)
 
     def _add_metadata(self, container, metadata, compression_cmd):
@@ -1438,6 +1460,14 @@ class gpkg:
             self._add_signature(checksum_info, image_tarinfo, container)
 
         self._add_manifest(container)
+
+        # Check if all directories are the same in the container
+        prefix = os.path.commonpath(container.getnames())
+        if not prefix:
+            raise InvalidBinaryPackageFormat(
+                f"gpkg file structure mismatch in {self.gpkg_file}"
+            )
+
         container.close()
 
     def _record_checksum(self, checksum_info, tarinfo):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     b9e30bb50996c11f2b046a9c6eb47ba8dc6331e6
Author:     Sheng Yu <syu.os <AT> protonmail <DOT> com>
AuthorDate: Wed Nov  2 17:52:18 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Nov  2 22:58:49 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9e30bb5

Do not redirect stderr from gpkg subprocess

Closes: https://bugs.gentoo.org/877215
Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/927
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 3d2aa4223..561fa4450 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -92,6 +92,7 @@ class tar_stream_writer:
         gid              # drop root group to gid
         """
         self.checksum_helper = checksum_helper
+        self.cmd = cmd
         self.closed = False
         self.container = container
         self.killed = False
@@ -124,7 +125,6 @@ class tar_stream_writer:
                     cmd,
                     stdin=subprocess.PIPE,
                     stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
                     user=self.uid,
                     group=self.gid,
                 )
@@ -133,7 +133,6 @@ class tar_stream_writer:
                     cmd,
                     stdin=subprocess.PIPE,
                     stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
                     preexec_fn=self._drop_privileges,
                 )
 
@@ -195,10 +194,10 @@ class tar_stream_writer:
                 buffer = self.proc.stdout.read(HASHING_BLOCKSIZE)
                 if not buffer:
                     self.proc.stdout.close()
-                    self.proc.stderr.close()
                     return
             except BrokenPipeError:
                 self.proc.stdout.close()
+                writemsg(colorize("BAD", f"GPKG subprocess failed: {self.cmd} \n"))
                 if not self.killed:
                     # Do not raise error if killed by portage
                     raise CompressorOperationFailed("PIPE broken")
@@ -220,6 +219,7 @@ class tar_stream_writer:
                 self.proc.stdin.write(data)
             except BrokenPipeError:
                 self.error = True
+                writemsg(colorize("BAD", f"GPKG subprocess failed: {self.cmd} \n"))
                 raise CompressorOperationFailed("PIPE broken")
         else:
             # Write to container
@@ -319,7 +319,6 @@ class tar_stream_reader:
                     cmd,
                     stdin=subprocess.PIPE,
                     stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
                     user=self.uid,
                     group=self.gid,
                 )
@@ -328,7 +327,6 @@ class tar_stream_reader:
                     cmd,
                     stdin=subprocess.PIPE,
                     stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
                     preexec_fn=self._drop_privileges,
                 )
             self.read_io = self.proc.stdout
@@ -374,6 +372,7 @@ class tar_stream_reader:
                     break
         except BrokenPipeError:
             if self.killed is False:
+                writemsg(colorize("BAD", f"GPKG subprocess failed: {self.cmd} \n"))
                 raise CompressorOperationFailed("PIPE broken")
 
     def _drop_privileges(self):
@@ -431,14 +430,11 @@ class tar_stream_reader:
             self.thread.join()
             try:
                 if self.proc.wait() != os.EX_OK:
-                    if not self.proc.stderr.closed:
-                        stderr = self.proc.stderr.read().decode()
                     if not self.killed:
-                        writemsg(colorize("BAD", f"!!!\n{stderr}"))
+                        writemsg(colorize("BAD", f"GPKG external program failed."))
                         raise CompressorOperationFailed("decompression failed")
             finally:
                 self.proc.stdout.close()
-                self.proc.stderr.close()
 
 
 class checksum_helper:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     f1d98b6dc36ff2b47c36427c9938999320352eb4
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  2 04:47:11 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Jan  2 05:25:51 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f1d98b6d

news: fix value of profiles_base

This fixes matching profile paths in Display-If-Profile in news items.

bad: kwargs['profile']='var/db/repos/gentoo/profiles/default/linux/amd64/17.1/hardened', self.profile='default/linux/amd64/17.1/hardened'
good: kwargs['profile']='default/linux/amd64/17.1/hardened', self.profile='default/linux/amd64/17.1/hardened'

os.path.join() treats paths differently based on the components given:
'os.path.join(portdir, "profiles", os.path.sep)' passes 3 different paths,
whereas before 64d84ce2d9a333e83e2a5fba5e7ec95f936959e7, and now, we concat.
profiles and os.path.sep first so that further splitting isn't carried out.

Thanks to kurly for noticing a recent news item wasn't showing up.

Fixes: 64d84ce2d9a333e83e2a5fba5e7ec95f936959e7
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/news.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/news.py b/lib/portage/news.py
index 51d493cb7..14401814d 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -79,7 +79,7 @@ class NewsManager:
         portdir = portdb.repositories.mainRepoLocation()
         profiles_base = None
         if portdir is not None:
-            profiles_base = os.path.join(portdir, "profiles", os.path.sep)
+            profiles_base = os.path.join(portdir, ("profiles" + os.path.sep))
         profile_path = None
         if profiles_base is not None and portdb.settings.profile_path:
             profile_path = normalize_path(

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     6e1c42a89fe1ae7a5915c6eb8e3e27b47aecf93c
Author:     Sheng Yu <syu.os <AT> protonmail <DOT> com>
AuthorDate: Fri Feb 10 07:34:49 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 01:23:14 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6e1c42a8

Catch all Exceptions in binpkg identify

We got many different exceptions here, so have to catch all.

Bug: https://bugs.gentoo.org/893638
Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/binpkg.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index 001a2e277..2078e3ca5 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -50,8 +50,13 @@ def get_binpkg_format(binpkg_path, check_file=False, remote=False):
             except tarfile.TarError:
                 pass
 
-    except OSError:
+    except Exception as err:
+        # We got many different exceptions here, so have to catch all of them.
         file_format = None
+        writemsg(
+            colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}"),
+        )
+        raise InvalidBinaryPackageFormat(f"Error reading binpkg '{binpkg_path}': {err}")
 
     if file_format is None:
         raise InvalidBinaryPackageFormat(

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     fb4ba0ef70095bd6f17e198674e16d60284ab2dc
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 27 05:10:23 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Feb 27 06:15:40 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=fb4ba0ef

portage: news: trivial type annotations

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/news.py | 59 +++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 41 insertions(+), 18 deletions(-)

diff --git a/lib/portage/news.py b/lib/portage/news.py
index f81debe97..68f0c72d3 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -14,11 +14,17 @@ __all__ = [
 ]
 
 from collections import OrderedDict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 import fnmatch
 import logging
 import os as _os
 import re
+
+if TYPE_CHECKING:
+    import portage.dbapi.vartree
+    import portage.package.ebuild.config
+
 from portage import os
 from portage import _encodings
 from portage import _unicode_decode
@@ -58,7 +64,14 @@ class NewsManager:
 
     """
 
-    def __init__(self, portdb, vardb, news_path, unread_path, language_id="en"):
+    def __init__(
+        self,
+        portdb: "portage.dbapi.porttree.portdbapi",
+        vardb: "portage.dbapi.vartree.vardbapi",
+        news_path: str,
+        unread_path: str,
+        language_id: str = "en",
+    ):
         self.news_path = news_path
         self.unread_path = unread_path
         self.language_id = language_id
@@ -89,19 +102,19 @@ class NewsManager:
                 profile_path = profile_path[len(profiles_base) :]
         self._profile_path = profile_path
 
-    def _unread_filename(self, repoid):
+    def _unread_filename(self, repoid: str) -> str:
         return os.path.join(self.unread_path, f"news-{repoid}.unread")
 
-    def _skip_filename(self, repoid):
+    def _skip_filename(self, repoid: str) -> str:
         return os.path.join(self.unread_path, f"news-{repoid}.skip")
 
-    def _news_dir(self, repoid):
+    def _news_dir(self, repoid: str) -> str:
         repo_path = self.portdb.getRepositoryPath(repoid)
         if repo_path is None:
             raise AssertionError(_(f"Invalid repoID: {repoid}"))
         return os.path.join(repo_path, self.news_path)
 
-    def updateItems(self, repoid):
+    def updateItems(self, repoid: str) -> None:
         """
         Figure out which news items from NEWS_PATH are both unread and relevant to
         the user (according to the GLEP 42 standards of relevancy).  Then add these
@@ -199,7 +212,7 @@ class NewsManager:
         finally:
             unlockfile(unread_lock)
 
-    def getUnreadItems(self, repoid, update=False):
+    def getUnreadItems(self, repoid: str, update: bool = False) -> int:
         """
         Determine if there are unread relevant items in news.repoid.unread.
         If there are unread items return their number.
@@ -249,7 +262,7 @@ class NewsItem:
     Creation of a news item involves passing in the path to the particular news item.
     """
 
-    def __init__(self, path, name):
+    def __init__(self, path: str, name: str):
         """
         For a given news item we only want if it path is a file.
         """
@@ -258,7 +271,12 @@ class NewsItem:
         self._parsed = False
         self._valid = True
 
-    def isRelevant(self, vardb, config, profile):
+    def isRelevant(
+        self,
+        vardb: "portage.dbapi.vartree.vardbapi",
+        config: "portage.package.ebuild.config.config",
+        profile: Optional[str],
+    ) -> bool:
         """
         This function takes a dict of keyword arguments; one should pass in any
         objects need to do to lookups (like what keywords we are on, what profile,
@@ -292,12 +310,12 @@ class NewsItem:
 
         return all_match
 
-    def isValid(self):
+    def isValid(self) -> bool:
         if not self._parsed:
             self.parse()
         return self._valid
 
-    def parse(self):
+    def parse(self) -> None:
         with open(
             _unicode_encode(self.path, encoding=_encodings["fs"], errors="strict"),
             encoding=_encodings["content"],
@@ -365,7 +383,7 @@ class DisplayRestriction:
     are met, then it is displayed
     """
 
-    def isValid(self):
+    def isValid(self) -> bool:
         return True
 
     def checkRestriction(self, **kwargs):
@@ -382,7 +400,7 @@ class DisplayProfileRestriction(DisplayRestriction):
         self.profile = profile
         self.format = news_format
 
-    def isValid(self):
+    def isValid(self) -> bool:
         return (
             not fnmatch.fnmatch(self.format, "1.*")
             or "*" not in self.profile
@@ -390,7 +408,7 @@ class DisplayProfileRestriction(DisplayRestriction):
             or _valid_profile_RE.match(self.profile)
         )
 
-    def checkRestriction(self, **kwargs):
+    def checkRestriction(self, **kwargs) -> bool:
         if fnmatch.fnmatch(self.format, "2.*") and self.profile.endswith("/*"):
             return kwargs["profile"].startswith(self.profile[:-1])
         return kwargs["profile"] == self.profile
@@ -406,7 +424,7 @@ class DisplayKeywordRestriction(DisplayRestriction):
         self.keyword = keyword
         self.format = news_format
 
-    def checkRestriction(self, **kwargs):
+    def checkRestriction(self, **kwargs) -> bool:
         return kwargs["config"].get("ARCH", "") == self.keyword
 
 
@@ -420,18 +438,23 @@ class DisplayInstalledRestriction(DisplayRestriction):
         self.atom = atom
         self.format = news_format
 
-    def isValid(self):
+    def isValid(self) -> bool:
         if fnmatch.fnmatch(self.format, "1.*"):
             return isvalidatom(self.atom, eapi="0")
         if fnmatch.fnmatch(self.format, "2.*"):
             return isvalidatom(self.atom, eapi="5")
         return isvalidatom(self.atom)
 
-    def checkRestriction(self, **kwargs):
+    def checkRestriction(self, **kwargs) -> bool:
         return kwargs["vardb"].match(self.atom)
 
 
-def count_unread_news(portdb, vardb, repos=None, update=True):
+def count_unread_news(
+    portdb: "portage.dbapi.porttree.portdbapi",
+    vardb: "portage.dbapi.vartree.vardbapi",
+    repos: Optional[List[Any]] = None,
+    update: bool = True,
+) -> Dict[str, int]:
     """
     Returns a dictionary mapping repos to integer counts of unread news items.
     By default, this will scan all repos and check for new items that have
@@ -477,7 +500,7 @@ def count_unread_news(portdb, vardb, repos=None, update=True):
     return news_counts
 
 
-def display_news_notifications(news_counts: dict):
+def display_news_notifications(news_counts: Dict[Any, int]) -> None:
     """
     Display a notification for unread news items, using a dictionary mapping
     repos to integer counts, like that returned from count_unread_news().

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     2d76645750ec5ee12a2eadbdc866ed0e4310e443
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 20 03:30:35 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Mar 21 02:30:24 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2d766457

checksum: update WHIRLPOOL comment for provider preference

Since 45f2da735a473d8b800861f97d7be1ba7850187f, we use a bundled C implementation
before falling back to the bundled pure Python impl.

Bug: https://bugs.gentoo.org/885909
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index 1797f268a..11b83a1fc 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -26,7 +26,7 @@ from portage.localization import _
 # SHA256: hashlib
 # SHA512: hashlib
 # RMD160: hashlib, pycrypto, mhash
-# WHIRLPOOL: hashlib, mhash, bundled
+# WHIRLPOOL: hashlib, mhash, bundled (C), bundled (Python)
 # BLAKE2B (512): hashlib, pycrypto
 # BLAKE2S (512): hashlib,pycrypto
 # SHA3_256: hashlib, pycrypto

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     85b71d3379575cb8a5f2f79f3fc534d82c3bbf09
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 20 03:34:08 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Mar 21 02:30:24 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=85b71d33

checksum: drop pycrypto/pycryptodome fallback for BLAKE2/SHA3

Included with >= Python 3.6 and we weren't testing these codepaths anyway.

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py | 55 ++++---------------------------------------------
 1 file changed, 4 insertions(+), 51 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index 11b83a1fc..96f3939c9 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -27,10 +27,10 @@ from portage.localization import _
 # SHA512: hashlib
 # RMD160: hashlib, pycrypto, mhash
 # WHIRLPOOL: hashlib, mhash, bundled (C), bundled (Python)
-# BLAKE2B (512): hashlib, pycrypto
-# BLAKE2S (512): hashlib,pycrypto
-# SHA3_256: hashlib, pycrypto
-# SHA3_512: hashlib, pycrypto
+# BLAKE2B (512): hashlib
+# BLAKE2S (512): hashlib
+# SHA3_256: hashlib
+# SHA3_512: hashlib
 
 
 # Dict of all available hash functions
@@ -138,53 +138,6 @@ if "RMD160" not in hashfunc_map:
     except ImportError:
         pass
 
-# The following hashes were added in pycryptodome (pycrypto fork)
-if "BLAKE2B" not in hashfunc_map:
-    try:
-        from Crypto.Hash import BLAKE2b
-
-        blake2bhash_ = getattr(BLAKE2b, "new", None)
-        if blake2bhash_ is not None:
-            _generate_hash_function(
-                "BLAKE2B",
-                functools.partial(blake2bhash_, digest_bytes=64),
-                origin="pycrypto",
-            )
-    except ImportError:
-        pass
-
-if "BLAKE2S" not in hashfunc_map:
-    try:
-        from Crypto.Hash import BLAKE2s
-
-        blake2shash_ = getattr(BLAKE2s, "new", None)
-        if blake2shash_ is not None:
-            _generate_hash_function(
-                "BLAKE2S",
-                functools.partial(blake2shash_, digest_bytes=32),
-                origin="pycrypto",
-            )
-    except ImportError:
-        pass
-
-if "SHA3_256" not in hashfunc_map:
-    try:
-        from Crypto.Hash import SHA3_256
-
-        sha3_256hash_ = getattr(SHA3_256, "new", None)
-        if sha3_256hash_ is not None:
-            _generate_hash_function("SHA3_256", sha3_256hash_, origin="pycrypto")
-    except ImportError:
-        pass
-
-if "SHA3_512" not in hashfunc_map:
-    try:
-        from Crypto.Hash import SHA3_512
-
-        sha3_512hash_ = getattr(SHA3_512, "new", None)
-        if sha3_512hash_ is not None:
-            _generate_hash_function("SHA3_512", sha3_512hash_, origin="pycrypto")
-    except ImportError:
         pass
 
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     d8f9c763a180b5b014b333791dd7235cbf923d31
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 20 03:24:39 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Mar 21 02:30:23 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8f9c763

checksum: drop pygcrypt support

No longer in Gentoo and the codepath for this has been disabled since 2017(!)
because of https://bugs.gentoo.org/615620.

The last release on pypi is also from April 2017.

Bug: https://bugs.gentoo.org/615620
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py | 46 ----------------------------------------------
 1 file changed, 46 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index 307dd4429..fdb8387a7 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -148,52 +148,6 @@ if "SHA3_256" not in hashfunc_map or "SHA3_512" not in hashfunc_map:
         pass
 
 
-# Support pygcrypt as fallback using optimized routines from libgcrypt
-# (GnuPG).
-gcrypt_algos = frozenset(
-    ("RMD160", "WHIRLPOOL", "SHA3_256", "SHA3_512")
-)
-# Note: currently disabled due to resource exhaustion bugs in pygcrypt.
-# Please do not reenable until upstream has a fix.
-# https://bugs.gentoo.org/615620
-if False:
-    # if gcrypt_algos.difference(hashfunc_map):
-    try:
-        import binascii
-        import pygcrypt.hashcontext
-
-        class GCryptHashWrapper:
-            def __init__(self, algo):
-                self._obj = pygcrypt.hashcontext.HashContext(algo=algo, secure=False)
-
-            def update(self, data):
-                self._obj.write(data)
-
-            def hexdigest(self):
-                return binascii.b2a_hex(self._obj.read()).decode()
-
-        name_mapping = {
-            "RMD160": "ripemd160",
-            "WHIRLPOOL": "whirlpool",
-            "SHA3_256": "sha3-256",
-            "SHA3_512": "sha3-512",
-        }
-
-        for local_name, gcry_name in name_mapping.items():
-            try:
-                pygcrypt.hashcontext.HashContext(algo=gcry_name)
-            except Exception:  # yes, it throws Exception...
-                pass
-            else:
-                _generate_hash_function(
-                    local_name,
-                    functools.partial(GCryptHashWrapper, gcry_name),
-                    origin="pygcrypt",
-                )
-    except ImportError:
-        pass
-
-
 # Use pycrypto when available, prefer it over the internal fallbacks
 # Check for 'new' attributes, since they can be missing if the module
 # is broken somehow.

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     b0e43d6bc73cb63642404e661b073d6183fb9aa8
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 20 03:37:26 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Mar 21 02:30:24 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b0e43d6b

checksum: add further comments on existing fallback logic for RIPEMD/WHIRLPOOL

Bug: https://github.com/python/cpython/issues/91257
Bug: https://github.com/python/cpython/issues/92876
Bug: https://bugs.gentoo.org/846389
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index 96f3939c9..3b9796b32 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -99,10 +99,15 @@ class _generate_hash_function:
 # Define hash functions, try to use the best module available. Preferred
 # modules should go first, latter ones should check if the hashes aren't
 # already defined.
-
-
 # Use hashlib if available and prefer it over pycrypto and internal fallbacks.
+#
 # Need special handling for RMD160/WHIRLPOOL as they may not always be provided by hashlib.
+# We keep fallbacks for RMD160/WHIRLPOOL for now as newer OpenSSLs don't expose them
+# by default.
+# See also
+# - https://github.com/python/cpython/issues/91257
+# - https://github.com/python/cpython/issues/92876
+# - https://bugs.gentoo.org/846389
 _generate_hash_function("MD5", hashlib.md5, origin="hashlib")
 _generate_hash_function("SHA1", hashlib.sha1, origin="hashlib")
 _generate_hash_function("SHA256", hashlib.sha256, origin="hashlib")

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     5804bfcc13b6b35e2bab5d6d93f899f8648bbd89
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 20 03:42:21 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Mar 21 02:30:24 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5804bfcc

checksum: drop mhash usage for WHIRLPOOL; consolidate RIPEMD logic

- Drop mhash fallback logic for WHIRLPOOL as we already have:
  hashlib > pycrypto > bundled C > bundled pure Python

- Consolidate the RIPEMD logic accordingly as we don't need to share the
  mhash fallback logic between WHIRLPOOL and RIPEMD anymore.

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py | 49 +++++++++++++++++++++----------------------------
 1 file changed, 21 insertions(+), 28 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index 3b9796b32..b10643476 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -26,7 +26,7 @@ from portage.localization import _
 # SHA256: hashlib
 # SHA512: hashlib
 # RMD160: hashlib, pycrypto, mhash
-# WHIRLPOOL: hashlib, mhash, bundled (C), bundled (Python)
+# WHIRLPOOL: hashlib, bundled (C), bundled (Python)
 # BLAKE2B (512): hashlib
 # BLAKE2S (512): hashlib
 # SHA3_256: hashlib
@@ -141,33 +141,26 @@ if "RMD160" not in hashfunc_map:
         if rmd160hash_ is not None:
             _generate_hash_function("RMD160", rmd160hash_, origin="pycrypto")
     except ImportError:
-        pass
-
-        pass
-
-
-# Try to use mhash if available
-# mhash causes GIL presently, so it gets less priority than hashlib and
-# pycrypto. However, it might be the only accelerated implementation of
-# WHIRLPOOL available.
-if "RMD160" not in hashfunc_map or "WHIRLPOOL" not in hashfunc_map:
-    try:
-        import mhash
-
-        for local_name, hash_name in (
-            ("RMD160", "RIPEMD160"),
-            ("WHIRLPOOL", "WHIRLPOOL"),
-        ):
-            if local_name not in hashfunc_map and hasattr(mhash, f"MHASH_{hash_name}"):
-                _generate_hash_function(
-                    local_name,
-                    functools.partial(
-                        mhash.MHASH, getattr(mhash, f"MHASH_{hash_name}")
-                    ),
-                    origin="mhash",
-                )
-    except ImportError:
-        pass
+        # Try to use mhash if available
+        # mhash causes GIL presently, so it gets less priority than hashlib and
+        # pycrypto. However, it might be the only accelerated implementation of
+        # WHIRLPOOL available.
+        try:
+            import mhash
+
+            for local_name, hash_name in (("RMD160", "RIPEMD160"),):
+                if local_name not in hashfunc_map and hasattr(
+                    mhash, f"MHASH_{hash_name}"
+                ):
+                    _generate_hash_function(
+                        local_name,
+                        functools.partial(
+                            mhash.MHASH, getattr(mhash, f"MHASH_{hash_name}")
+                        ),
+                        origin="mhash",
+                    )
+        except ImportError:
+            pass
 
 
 _whirlpool_unaccelerated = False

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     d8c8d84f07a299b66dbd38ac7a294989d1315040
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 20 03:26:28 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Mar 21 02:30:24 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8c8d84f

checksum: drop < Python 3.6 checksum compatibilty via pyblake2, sha3

We've only supported >= Python 3.6 for quite some time, so these codepaths are
obsolete given hashlib will always provide support for BLAKE2 and SHA3.

Neither pyblake2 nor sha3 are even packaged in Gentoo anymore either (the
Python bindings).

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/checksum.py | 33 +++++----------------------------
 1 file changed, 5 insertions(+), 28 deletions(-)

diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py
index fdb8387a7..1797f268a 100644
--- a/lib/portage/checksum.py
+++ b/lib/portage/checksum.py
@@ -27,10 +27,10 @@ from portage.localization import _
 # SHA512: hashlib
 # RMD160: hashlib, pycrypto, mhash
 # WHIRLPOOL: hashlib, mhash, bundled
-# BLAKE2B (512): hashlib (3.6+), pyblake2, pycrypto
-# BLAKE2S (512): hashlib (3.6+), pyblake2, pycrypto
-# SHA3_256: hashlib (3.6+), pysha3, pycrypto
-# SHA3_512: hashlib (3.6+), pysha3, pycrypto
+# BLAKE2B (512): hashlib, pycrypto
+# BLAKE2S (512): hashlib,pycrypto
+# SHA3_256: hashlib, pycrypto
+# SHA3_512: hashlib, pycrypto
 
 
 # Dict of all available hash functions
@@ -101,7 +101,7 @@ class _generate_hash_function:
 # already defined.
 
 
-# Use hashlib from python-2.5 if available and prefer it over pycrypto and internal fallbacks.
+# Use hashlib if available and prefer it over pycrypto and internal fallbacks.
 # Need special handling for RMD160/WHIRLPOOL as they may not always be provided by hashlib.
 _generate_hash_function("MD5", hashlib.md5, origin="hashlib")
 _generate_hash_function("SHA1", hashlib.sha1, origin="hashlib")
@@ -110,7 +110,6 @@ _generate_hash_function("SHA512", hashlib.sha512, origin="hashlib")
 for local_name, hash_name in (
     ("RMD160", "ripemd160"),
     ("WHIRLPOOL", "whirlpool"),
-    # available since Python 3.6
     ("BLAKE2B", "blake2b"),
     ("BLAKE2S", "blake2s"),
     ("SHA3_256", "sha3_256"),
@@ -126,28 +125,6 @@ for local_name, hash_name in (
         )
 
 
-# Support using pyblake2 as fallback for python<3.6
-if "BLAKE2B" not in hashfunc_map or "BLAKE2S" not in hashfunc_map:
-    try:
-        import pyblake2
-
-        _generate_hash_function("BLAKE2B", pyblake2.blake2b, origin="pyblake2")
-        _generate_hash_function("BLAKE2S", pyblake2.blake2s, origin="pyblake2")
-    except ImportError:
-        pass
-
-
-# Support using pysha3 as fallback for python<3.6
-if "SHA3_256" not in hashfunc_map or "SHA3_512" not in hashfunc_map:
-    try:
-        import sha3
-
-        _generate_hash_function("SHA3_256", sha3.sha3_256, origin="pysha3")
-        _generate_hash_function("SHA3_512", sha3.sha3_512, origin="pysha3")
-    except ImportError:
-        pass
-
-
 # Use pycrypto when available, prefer it over the internal fallbacks
 # Check for 'new' attributes, since they can be missing if the module
 # is broken somehow.

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     59973d0f5829ebbae615a001a97abb652431a61b
Author:     Sheng Yu <syu.os <AT> protonmail <DOT> com>
AuthorDate: Sat Jun 24 20:28:20 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Thu Jun 29 08:22:53 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=59973d0f

gpkg: fix incorrect gpkg timestamp

[sam: Quoting dwfreed from the bug:
"utcnow() produces a naive datetime object, and most methods of datetime objects
treat naive datetime objects as local time."]

Bug: https://bugs.gentoo.org/909067
Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1060
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 7451c68c2..6ed2a6d00 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1012,7 +1012,7 @@ class gpkg:
         gpkg_version_file = tarfile.TarInfo(
             os.path.join(self.basename, self.gpkg_version)
         )
-        gpkg_version_file.mtime = datetime.utcnow().timestamp()
+        gpkg_version_file.mtime = datetime.now().timestamp()
         container.addfile(gpkg_version_file)
         checksum_info = checksum_helper(self.settings)
         checksum_info.finish()
@@ -1032,7 +1032,7 @@ class gpkg:
             checksum_info = checksum_helper(self.settings)
 
         image_tarinfo = self._create_tarinfo("image")
-        image_tarinfo.mtime = datetime.utcnow().timestamp()
+        image_tarinfo.mtime = datetime.now().timestamp()
         with tar_stream_writer(
             image_tarinfo, container, image_tar_format, compression_cmd, checksum_info
         ) as image_writer:
@@ -1114,7 +1114,7 @@ class gpkg:
             gpkg_version_file = tarfile.TarInfo(
                 os.path.join(new_basename, self.gpkg_version)
             )
-            gpkg_version_file.mtime = datetime.utcnow().timestamp()
+            gpkg_version_file.mtime = datetime.now().timestamp()
             container.addfile(gpkg_version_file)
             checksum_info = checksum_helper(self.settings)
             checksum_info.finish()
@@ -1182,7 +1182,7 @@ class gpkg:
             gpkg_version_file = tarfile.TarInfo(
                 os.path.join(self.prefix, self.gpkg_version)
             )
-            gpkg_version_file.mtime = datetime.utcnow().timestamp()
+            gpkg_version_file.mtime = datetime.now().timestamp()
             container.addfile(gpkg_version_file)
             checksum_info = checksum_helper(self.settings)
             checksum_info.finish()
@@ -1253,7 +1253,7 @@ class gpkg:
         if metadata is None:
             metadata = {}
         metadata_tarinfo = self._create_tarinfo("metadata")
-        metadata_tarinfo.mtime = datetime.utcnow().timestamp()
+        metadata_tarinfo.mtime = datetime.now().timestamp()
 
         if self.create_signature:
             checksum_info = checksum_helper(
@@ -1274,7 +1274,7 @@ class gpkg:
             ) as metadata_tar:
                 for m in metadata:
                     m_info = tarfile.TarInfo(os.path.join("metadata", m))
-                    m_info.mtime = datetime.utcnow().timestamp()
+                    m_info.mtime = datetime.now().timestamp()
 
                     if isinstance(metadata[m], bytes):
                         m_data = io.BytesIO(metadata[m])
@@ -1329,7 +1329,7 @@ class gpkg:
         gpkg_version_file = tarfile.TarInfo(
             os.path.join(self.basename, self.gpkg_version)
         )
-        gpkg_version_file.mtime = datetime.utcnow().timestamp()
+        gpkg_version_file.mtime = datetime.now().timestamp()
         container.addfile(gpkg_version_file)
         checksum_info = checksum_helper(self.settings)
         checksum_info.finish()
@@ -1350,7 +1350,7 @@ class gpkg:
         paths = list(contents)
         paths.sort()
         image_tarinfo = self._create_tarinfo("image")
-        image_tarinfo.mtime = datetime.utcnow().timestamp()
+        image_tarinfo.mtime = datetime.now().timestamp()
         with tar_stream_writer(
             image_tarinfo, container, image_tar_format, compression_cmd, checksum_info
         ) as image_writer:
@@ -1518,7 +1518,7 @@ class gpkg:
 
         manifest_tarinfo = tarfile.TarInfo(os.path.join(basename, "Manifest"))
         manifest_tarinfo.size = manifest.tell()
-        manifest_tarinfo.mtime = datetime.utcnow().timestamp()
+        manifest_tarinfo.mtime = datetime.now().timestamp()
         manifest.seek(0)
         container.addfile(manifest_tarinfo, manifest)
         manifest.close()
@@ -1562,7 +1562,7 @@ class gpkg:
         signature = io.BytesIO(checksum_info.gpg_output)
         signature_tarinfo = tarfile.TarInfo(f"{tarinfo.name}.sig")
         signature_tarinfo.size = len(signature.getvalue())
-        signature_tarinfo.mtime = datetime.utcnow().timestamp()
+        signature_tarinfo.mtime = datetime.now().timestamp()
         container.addfile(signature_tarinfo, signature)
 
         if manifest:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     003a8d8ab9aaf4a6d4d0548b1ee46f79af3caf78
Author:     Andrew Udvare <audvare <AT> gmail <DOT> com>
AuthorDate: Sun Jul 16 21:42:37 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Jul 29 03:57:45 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=003a8d8a

versions: add basic typing

Signed-off-by: Andrew Udvare <audvare <AT> gmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/versions.py | 88 +++++++++++++++++++++++++++----------------------
 1 file changed, 48 insertions(+), 40 deletions(-)

diff --git a/lib/portage/versions.py b/lib/portage/versions.py
index f3a224eec..4b97c6124 100644
--- a/lib/portage/versions.py
+++ b/lib/portage/versions.py
@@ -16,8 +16,10 @@ __all__ = [
 ]
 
 import re
+import typing
 import warnings
 from functools import lru_cache
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
 
 import portage
@@ -28,7 +30,7 @@ portage.proxy.lazyimport.lazyimport(
     "portage.util:cmp_sort_key",
 )
 from portage import _unicode_decode
-from portage.eapi import _get_eapi_attrs
+from portage.eapi import _eapi_attrs, _get_eapi_attrs
 from portage.exception import InvalidData
 from portage.localization import _
 
@@ -76,7 +78,7 @@ endversion_keys = ["pre", "p", "alpha", "beta", "rc"]
 _slot_re_cache = {}
 
 
-def _get_slot_re(eapi_attrs):
+def _get_slot_re(eapi_attrs: _eapi_attrs) -> typing.re.Pattern:
     cache_key = eapi_attrs.slot_operator
     slot_re = _slot_re_cache.get(cache_key)
     if slot_re is not None:
@@ -96,7 +98,7 @@ def _get_slot_re(eapi_attrs):
 _pv_re = None
 
 
-def _get_pv_re(eapi_attrs):
+def _get_pv_re(eapi_attrs: _eapi_attrs) -> typing.re.Pattern:
     global _pv_re
     if _pv_re is not None:
         return _pv_re
@@ -106,7 +108,7 @@ def _get_pv_re(eapi_attrs):
     return _pv_re
 
 
-def ververify(myver, silent=1):
+def ververify(myver: str, silent: int = 1) -> bool:
     if ver_regexp.match(myver):
         return True
     if not silent:
@@ -115,7 +117,7 @@ def ververify(myver, silent=1):
 
 
 @lru_cache(1024)
-def vercmp(ver1, ver2, silent=1):
+def vercmp(ver1: str, ver2: str, silent: int = 1) -> Optional[int]:
     """
     Compare two versions
     Example usage:
@@ -262,7 +264,7 @@ def vercmp(ver1, ver2, silent=1):
     return rval
 
 
-def pkgcmp(pkg1, pkg2):
+def pkgcmp(pkg1: Tuple[str, str, str], pkg2: Tuple[str, str, str]) -> Optional[int]:
     """
     Compare 2 package versions created in pkgsplit format.
 
@@ -289,7 +291,7 @@ def pkgcmp(pkg1, pkg2):
     return vercmp("-".join(pkg1[1:]), "-".join(pkg2[1:]))
 
 
-def _pkgsplit(mypkg, eapi=None):
+def _pkgsplit(mypkg: str, eapi: Any = None) -> Optional[Tuple[str, str, str]]:
     """
     @param mypkg: pv
     @return:
@@ -317,7 +319,11 @@ _missing_cat = "null"
 
 
 @lru_cache(10240)
-def catpkgsplit(mydata, silent=1, eapi=None):
+def catpkgsplit(
+    mydata: Union[str, "_pkg_str"],
+    silent: int = 1,
+    eapi: Any = None,
+) -> Optional[Tuple[str, ...]]:
     """
     Takes a Category/Package-Version-Rev and returns a list of each.
 
@@ -368,33 +374,33 @@ class _pkg_str(str):
 
     def __new__(
         cls,
-        cpv,
-        metadata=None,
-        settings=None,
-        eapi=None,
-        repo=None,
-        slot=None,
-        build_time=None,
-        build_id=None,
-        file_size=None,
-        mtime=None,
-        db=None,
+        cpv: str,
+        metadata: Optional[Dict[str, Any]] = None,
+        settings: Any = None,
+        eapi: Any = None,
+        repo: Optional[str] = None,
+        slot: Optional[str] = None,
+        build_time: Optional[int] = None,
+        build_id: Optional[str] = None,
+        file_size: Optional[int] = None,
+        mtime: Optional[int] = None,
+        db: Any = None,
     ):
         return str.__new__(cls, cpv)
 
     def __init__(
         self,
-        cpv,
-        metadata=None,
-        settings=None,
-        eapi=None,
-        repo=None,
-        slot=None,
-        build_time=None,
-        build_id=None,
-        file_size=None,
-        mtime=None,
-        db=None,
+        cpv: str,
+        metadata: Optional[Dict[str, Any]] = None,
+        settings: Any = None,
+        eapi: Any = None,
+        repo: Optional[str] = None,
+        slot: Optional[str] = None,
+        build_time: Optional[int] = None,
+        build_id: Optional[str] = None,
+        file_size: Optional[int] = None,
+        mtime: Optional[int] = None,
+        db: Any = None,
     ):
         if not isinstance(cpv, str):
             # Avoid TypeError from str.__init__ with PyPy.
@@ -456,13 +462,13 @@ class _pkg_str(str):
                 repo = _unknown_repo
             self.__dict__["repo"] = repo
 
-    def __setattr__(self, name, value):
+    def __setattr__(self, name: str, value: Any) -> None:
         raise AttributeError(
             "_pkg_str instances are immutable", self.__class__, name, value
         )
 
     @staticmethod
-    def _long(var, default):
+    def _long(var: Any, default: int) -> int:
         if var is not None:
             try:
                 var = int(var)
@@ -474,7 +480,7 @@ class _pkg_str(str):
         return var
 
     @property
-    def stable(self):
+    def stable(self) -> bool:
         try:
             return self._stable
         except AttributeError:
@@ -492,7 +498,9 @@ class _pkg_str(str):
             return stable
 
 
-def pkgsplit(mypkg, silent=1, eapi=None):
+def pkgsplit(
+    mypkg: str, silent: int = 1, eapi: Any = None
+) -> Optional[Tuple[str, str, str]]:
     """
     @param mypkg: either a pv or cpv
     @return:
@@ -509,7 +517,7 @@ def pkgsplit(mypkg, silent=1, eapi=None):
     return (cat + "/" + pn, ver, rev)
 
 
-def cpv_getkey(mycpv, eapi=None):
+def cpv_getkey(mycpv: Union[_pkg_str, str], eapi: Any = None) -> Optional[str]:
     """Calls catpkgsplit on a cpv and returns only the cp."""
     try:
         return mycpv.cp
@@ -535,7 +543,7 @@ def cpv_getkey(mycpv, eapi=None):
     return mysplit[0]
 
 
-def cpv_getversion(mycpv, eapi=None):
+def cpv_getversion(mycpv: Union[str, _pkg_str], eapi: Any = None) -> Optional[str]:
     """Returns the v (including revision) from an cpv."""
     try:
         return mycpv.version
@@ -547,7 +555,7 @@ def cpv_getversion(mycpv, eapi=None):
     return mycpv[len(cp + "-") :]
 
 
-def cpv_sort_key(eapi=None):
+def cpv_sort_key(eapi: Any = None) -> Any:
     """
     Create an object for sorting cpvs, to be used as the 'key' parameter
     in places like list.sort() or sorted(). This calls catpkgsplit() once for
@@ -562,7 +570,7 @@ def cpv_sort_key(eapi=None):
 
     split_cache = {}
 
-    def cmp_cpv(cpv1, cpv2):
+    def cmp_cpv(cpv1: Any, cpv2: Any) -> int:
         split1 = split_cache.get(cpv1, False)
         if split1 is False:
             split1 = None
@@ -595,11 +603,11 @@ def cpv_sort_key(eapi=None):
     return cmp_sort_key(cmp_cpv)
 
 
-def catsplit(mydep):
+def catsplit(mydep: str) -> List[str]:
     return mydep.split("/", 1)
 
 
-def best(mymatches, eapi=None):
+def best(mymatches: Sequence[Any], eapi: Any = None) -> Any:
     """Accepts None arguments; assumes matches are valid."""
     if not mymatches:
         return ""

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     fb3debbd9455bc6288b2f420312beac836876fbd
Author:     James Le Cuirot <chewi <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 23 21:27:15 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Aug  2 06:31:20 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=fb3debbd

Fix the test_doebuild_fd_pipes.py test under CI

The tests call _disable_legacy_globals, but they never reset
portage.data._initialized_globals, meaning that the globals never get
set again.

This led to the test trying to chown with GID 0 as an unprivileged user.
This only failed under CI after switching to pytest, despite it working
with runTests.py, as well as locally. Running it on its own also worked
fine. I don't know how it worked before, maybe it was just the ordering
and some luck.

Signed-off-by: James Le Cuirot <chewi <AT> gentoo.org>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 847a879cd..635eea31e 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -829,3 +829,4 @@ def _disable_legacy_globals():
     global _legacy_global_var_names
     for k in _legacy_global_var_names:
         globals().pop(k, None)
+    portage.data._initialized_globals.clear()

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Mike Gilbert]

commit:     9d278330839049a818ba9f1e3985c7e502c63451
Author:     Andrew Ammerlaan <andrewammerlaan <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 23 11:47:32 2023 +0000
Commit:     Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Thu Aug 24 18:23:41 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9d278330

lib/portage/gpg.py: send gpg output to stdout, then send stdout to /dev/null

GnuPG removes the --output file on failure. Removing /dev/null breaks all kinds
of things. Instead of telling GnuPG to output to /dev/null we output to
stdout and then send the stdout to /dev/null, this avoids the problem where
/dev/null is removed when the key fails to unlock.

Bug: https://bugs.gentoo.org/912808
Signed-off-by: Andrew Ammerlaan <andrewammerlaan <AT> gentoo.org>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>

 lib/portage/gpg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/gpg.py b/lib/portage/gpg.py
index 6238ea6f7b..3067872244 100644
--- a/lib/portage/gpg.py
+++ b/lib/portage/gpg.py
@@ -35,7 +35,7 @@ class GPG:
             f"--homedir {self.signing_gpg_home} "
             f"--digest-algo {self.digest_algo} "
             f"--local-user {self.signing_gpg_key} "
-            "--output /dev/null /dev/null",
+            "--output - /dev/null",
         )
 
         if "gpg-keepalive" in self.settings.features:
@@ -61,7 +61,7 @@ class GPG:
                 writemsg(f"{colorize('WARN', str(e))}\n")
 
             cmd = shlex_split(varexpand(self.GPG_unlock_command, mydict=self.settings))
-            return_code = subprocess.Popen(cmd).wait()
+            return_code = subprocess.Popen(cmd, stdout=subprocess.DEVNULL).wait()
 
             if return_code == os.EX_OK:
                 writemsg_stdout(f"{colorize('GOOD', 'unlocked')}\n")

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     31cec19a1b976c59f8f958f86631bca0f99a5ec5
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Fri Sep  8 19:49:21 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Sep  8 19:49:21 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=31cec19a

gpkg: drop dead code (< Python 3.9 conditional) for subprocess

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 73 +++++++++++++++++------------------------------------
 1 file changed, 23 insertions(+), 50 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 6ed2a6d00b..4b94853e4c 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -120,21 +120,13 @@ class tar_stream_writer:
         if cmd is None:
             self.proc = None
         else:
-            if sys.hexversion >= 0x03090000:
-                self.proc = subprocess.Popen(
-                    cmd,
-                    stdin=subprocess.PIPE,
-                    stdout=subprocess.PIPE,
-                    user=self.uid,
-                    group=self.gid,
-                )
-            else:
-                self.proc = subprocess.Popen(
-                    cmd,
-                    stdin=subprocess.PIPE,
-                    stdout=subprocess.PIPE,
-                    preexec_fn=self._drop_privileges,
-                )
+            self.proc = subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                user=self.uid,
+                group=self.gid,
+            )
 
             self.read_thread = threading.Thread(
                 target=self._cmd_read_thread, name="tar_stream_cmd_read", daemon=True
@@ -318,21 +310,13 @@ class tar_stream_reader:
             self.proc = None
         else:
             # Start external decompressor
-            if sys.hexversion >= 0x03090000:
-                self.proc = subprocess.Popen(
-                    cmd,
-                    stdin=subprocess.PIPE,
-                    stdout=subprocess.PIPE,
-                    user=self.uid,
-                    group=self.gid,
-                )
-            else:
-                self.proc = subprocess.Popen(
-                    cmd,
-                    stdin=subprocess.PIPE,
-                    stdout=subprocess.PIPE,
-                    preexec_fn=self._drop_privileges,
-                )
+            self.proc = subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                user=self.uid,
+                group=self.gid,
+            )
             self.read_io = self.proc.stdout
             # Start stdin block writing thread
             self.thread = threading.Thread(
@@ -575,26 +559,15 @@ class checksum_helper:
             )
             gpg_verify_command = [x for x in gpg_verify_command if x != ""]
 
-            if sys.hexversion >= 0x03090000:
-                self.gpg_proc = subprocess.Popen(
-                    gpg_verify_command,
-                    stdin=subprocess.PIPE,
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
-                    env=env,
-                    user=self.uid,
-                    group=self.gid,
-                )
-
-            else:
-                self.gpg_proc = subprocess.Popen(
-                    gpg_verify_command,
-                    stdin=subprocess.PIPE,
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
-                    env=env,
-                    preexec_fn=self._drop_privileges,
-                )
+            self.gpg_proc = subprocess.Popen(
+                gpg_verify_command,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                env=env,
+                user=self.uid,
+                group=self.gid,
+            )
 
     def __del__(self):
         self.finish()

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     5d60ca35e66e241c092c8f42b2edf65789e51f28
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Fri Sep  8 20:35:36 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Sep  8 20:35:36 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5d60ca35

gpkg: drop more dead code

_drop_privileges was only called by the <py3.9 compat bits in subprocess.Popen
w/ preexec_fn.

Fixes: 31cec19a1b976c59f8f958f86631bca0f99a5ec5
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 69 -----------------------------------------------------
 1 file changed, 69 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 4b94853e4c..d16dffbc27 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -142,29 +142,6 @@ class tar_stream_writer:
     def __exit__(self, exc_type, exc_value, traceback):
         self.close()
 
-    def _drop_privileges(self):
-        if self.uid:
-            try:
-                os.setuid(self.uid)
-            except PermissionError:
-                writemsg(
-                    colorize(
-                        "BAD", f"!!! Drop root privileges to user {self.uid} failed."
-                    )
-                )
-                raise
-
-        if self.gid:
-            try:
-                os.setgid(self.gid)
-            except PermissionError:
-                writemsg(
-                    colorize(
-                        "BAD", f"!!! Drop root privileges to group {self.gid} failed."
-                    )
-                )
-                raise
-
     def kill(self):
         """
         kill external program if any error happened in python
@@ -363,29 +340,6 @@ class tar_stream_reader:
                 writemsg(colorize("BAD", f"GPKG subprocess failed: {self.cmd} \n"))
                 raise CompressorOperationFailed("PIPE broken")
 
-    def _drop_privileges(self):
-        if self.uid:
-            try:
-                os.setuid(self.uid)
-            except PermissionError:
-                writemsg(
-                    colorize(
-                        "BAD", f"!!! Drop root privileges to user {self.uid} failed."
-                    )
-                )
-                raise
-
-        if self.gid:
-            try:
-                os.setgid(self.gid)
-            except PermissionError:
-                writemsg(
-                    colorize(
-                        "BAD", f"!!! Drop root privileges to group {self.gid} failed."
-                    )
-                )
-                raise
-
     def kill(self):
         """
         kill external program if any error happened in python
@@ -593,29 +547,6 @@ class checksum_helper:
             writemsg(colorize("BAD", f"!!!\n{self.gpg_result.decode()}"))
             raise InvalidSignature("GPG verify failed")
 
-    def _drop_privileges(self):
-        if self.uid:
-            try:
-                os.setuid(self.uid)
-            except PermissionError:
-                writemsg(
-                    colorize(
-                        "BAD", f"!!! Drop root privileges to user {self.uid} failed."
-                    )
-                )
-                raise
-
-        if self.gid:
-            try:
-                os.setgid(self.gid)
-            except PermissionError:
-                writemsg(
-                    colorize(
-                        "BAD", f"!!! Drop root privileges to group {self.gid} failed."
-                    )
-                )
-                raise
-
     def update(self, data):
         """
         Write data to hash libs and GPG stdin.

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     d1941c2a512c380261bb64f5c1ae72bb0e2e1f92
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 26 05:26:19 2023 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 26 05:51:29 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d1941c2a

Remove deprecated typing.re.Pattern usage

Fixes this DeprecationWarning triggered by
lib/portage/tests/versions/test_vercmp.py:

lib/portage/versions.py:82: DeprecationWarning: typing.re is deprecated, import directly from typing instead. typing.re will be removed in Python 3.12.
  def _get_slot_re(eapi_attrs: _eapi_attrs) -> typing.re.Pattern:

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/versions.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/portage/versions.py b/lib/portage/versions.py
index 74855e63be..0e515ba5c8 100644
--- a/lib/portage/versions.py
+++ b/lib/portage/versions.py
@@ -1,5 +1,5 @@
 # versions.py -- core Portage functionality
-# Copyright 1998-2016 Gentoo Foundation
+# Copyright 1998-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = [
@@ -79,7 +79,7 @@ endversion_keys = ["pre", "p", "alpha", "beta", "rc"]
 _slot_re_cache = {}
 
 
-def _get_slot_re(eapi_attrs: _eapi_attrs) -> typing.re.Pattern:
+def _get_slot_re(eapi_attrs: _eapi_attrs) -> typing.Pattern:
     cache_key = eapi_attrs.slot_operator
     slot_re = _slot_re_cache.get(cache_key)
     if slot_re is not None:
@@ -99,7 +99,7 @@ def _get_slot_re(eapi_attrs: _eapi_attrs) -> typing.re.Pattern:
 _pv_re = None
 
 
-def _get_pv_re(eapi_attrs: _eapi_attrs) -> typing.re.Pattern:
+def _get_pv_re(eapi_attrs: _eapi_attrs) -> typing.Pattern:
     global _pv_re
     if _pv_re is not None:
         return _pv_re

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     a34c877eb465561774c2bcd8536e7b9c2d987c54
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 30 16:48:49 2023 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 30 16:48:53 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a34c877e

Remove obsolete _eintr_func_wrapper

This wrapper is obsolete since python 3.5 implemented EINTR
retry in the standard library.

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/__init__.py | 25 +------------------------
 lib/portage/locks.py    |  2 +-
 2 files changed, 2 insertions(+), 25 deletions(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 635eea31e2..05c81be538 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -345,29 +345,6 @@ class _unicode_module_wrapper:
         return result
 
 
-class _eintr_func_wrapper:
-    """
-    Wraps a function and handles EINTR by calling the function as
-    many times as necessary (until it returns without raising EINTR).
-    """
-
-    __slots__ = ("_func",)
-
-    def __init__(self, func):
-        self._func = func
-
-    def __call__(self, *args, **kwargs):
-        while True:
-            try:
-                rval = self._func(*args, **kwargs)
-                break
-            except OSError as e:
-                if e.errno != errno.EINTR:
-                    raise
-
-        return rval
-
-
 import os as _os
 
 _os_overrides = {
@@ -375,7 +352,7 @@ _os_overrides = {
     id(_os.popen): _os.popen,
     id(_os.read): _os.read,
     id(_os.system): _os.system,
-    id(_os.waitpid): _eintr_func_wrapper(_os.waitpid),
+    id(_os.waitpid): _os.waitpid,
 }
 
 

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 9c77398bb1..1c3e13ce4f 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -297,7 +297,7 @@ def _lockfile_iteration(
 
     # try for a non-blocking lock, if it's held, throw a message
     # we're waiting on lockfile and use a blocking attempt.
-    locking_method = portage._eintr_func_wrapper(_get_lock_fn())
+    locking_method = _get_lock_fn()
     try:
         if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
             raise OSError(errno.ENOSYS, "Function not implemented")

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     7622da95fd97c07d7a51f14dfcfb07048db687c1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  3 14:59:17 2023 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct  3 14:59:17 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7622da95

_get_lock_fn: support multiprocessing spawn start method (bug 915119)

Ensure that _get_lock_fn arguments to multiprocessing.Process will
successfully pickle, as required by the spawn start method, which
is the default for macOS since Python 3.8.

Since file descriptors are not inherited unless the fork start
method is used, the subprocess should only try to close an
inherited file descriptor for the fork start method.

Bug: https://bugs.gentoo.org/915119
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/locks.py | 50 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 18 deletions(-)

diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 1c3e13ce4f..ee40451b12 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -79,6 +79,11 @@ class _lock_manager:
         del _open_inodes[self.inode_key]
 
 
+def _lockf_test_lock_fn(path, fd, flags):
+    fcntl.lockf(fd, flags)
+    return functools.partial(unlockfile, (path, fd, flags, fcntl.lockf))
+
+
 def _get_lock_fn():
     """
     Returns fcntl.lockf if proven to work, and otherwise returns fcntl.flock.
@@ -88,10 +93,7 @@ def _get_lock_fn():
     if _lock_fn is not None:
         return _lock_fn
 
-    if _test_lock_fn(
-        lambda path, fd, flags: fcntl.lockf(fd, flags)
-        and functools.partial(unlockfile, (path, fd, flags, fcntl.lockf))
-    ):
+    if _test_lock_fn(_lockf_test_lock_fn):
         _lock_fn = fcntl.lockf
         return _lock_fn
 
@@ -103,19 +105,6 @@ def _get_lock_fn():
 def _test_lock_fn(
     lock_fn: typing.Callable[[str, int, int], typing.Callable[[], None]]
 ) -> bool:
-    def _test_lock(fd, lock_path):
-        os.close(fd)
-        try:
-            with open(lock_path, "a") as f:
-                lock_fn(lock_path, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
-        except (TryAgain, OSError) as e:
-            if isinstance(e, TryAgain) or e.errno == errno.EAGAIN:
-                # Parent process holds lock, as expected.
-                sys.exit(0)
-
-        # Something went wrong.
-        sys.exit(1)
-
     fd, lock_path = tempfile.mkstemp()
     unlock_fn = None
     try:
@@ -125,7 +114,17 @@ def _test_lock_fn(
             pass
         else:
             _lock_manager(fd, os.fstat(fd), lock_path)
-            proc = multiprocessing.Process(target=_test_lock, args=(fd, lock_path))
+            proc = multiprocessing.Process(
+                target=_subprocess_test_lock,
+                args=(
+                    # Since file descriptors are not inherited unless the fork start
+                    # method is used, the subprocess should only try to close an
+                    # inherited file descriptor for the fork start method.
+                    fd if multiprocessing.get_start_method() == "fork" else None,
+                    lock_fn,
+                    lock_path,
+                ),
+            )
             proc.start()
             proc.join()
             if proc.exitcode == os.EX_OK:
@@ -141,6 +140,21 @@ def _test_lock_fn(
     return False
 
 
+def _subprocess_test_lock(fd, lock_fn, lock_path):
+    if fd is not None:
+        os.close(fd)
+    try:
+        with open(lock_path, "a") as f:
+            lock_fn(lock_path, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except (TryAgain, OSError) as e:
+        if isinstance(e, TryAgain) or e.errno == errno.EAGAIN:
+            # Parent process holds lock, as expected.
+            sys.exit(0)
+
+    # Something went wrong.
+    sys.exit(1)
+
+
 def _close_fds():
     """
     This is intended to be called after a fork, in order to close file

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     d7ff40702646bbbe9ae9e2c691f3290f2cc565ba
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 23 14:51:44 2023 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Oct 23 14:52:05 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d7ff4070

create_trees: Handle bytes arguments for utf8_mode

Bug: https://bugs.gentoo.org/916149
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/__init__.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index defefcb286..b26d0cfd53 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -618,6 +618,16 @@ class _trees_dict(dict):
 def create_trees(
     config_root=None, target_root=None, trees=None, env=None, sysroot=None, eprefix=None
 ):
+    if utf8_mode:
+        config_root = (
+            os.fsdecode(config_root) if isinstance(config_root, bytes) else config_root
+        )
+        target_root = (
+            os.fsdecode(target_root) if isinstance(target_root, bytes) else target_root
+        )
+        sysroot = os.fsdecode(sysroot) if isinstance(sysroot, bytes) else sysroot
+        eprefix = os.fsdecode(eprefix) if isinstance(eprefix, bytes) else eprefix
+
     if trees is None:
         trees = _trees_dict()
     elif not isinstance(trees, _trees_dict):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     f2e8b37f37f01c4152693e01575293c8c611709d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 24 19:17:21 2023 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct 24 20:52:30 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f2e8b37f

_unshare_validator: multiprocessing spawn compat

Since ctypes objects are not picklable for the multiprocessing
spawn start method, acquire them in the subprocess.

Bug: https://bugs.gentoo.org/916242
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 63656cfbe6..28f977a046 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -1,5 +1,5 @@
 # portage.py -- core Portage functionality
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 
@@ -943,6 +943,8 @@ class _unshare_validator:
         @rtype: int
         @returns: errno value, or 0 if no error occurred.
         """
+        # This ctypes library lookup caches the result for use in the
+        # subprocess when the multiprocessing start method is fork.
         filename = find_library("c")
         if filename is None:
             return errno.ENOTSUP
@@ -955,7 +957,7 @@ class _unshare_validator:
 
         proc = multiprocessing.Process(
             target=cls._run_subproc,
-            args=(subproc_pipe, cls._validate_subproc, (libc.unshare, flags)),
+            args=(subproc_pipe, cls._validate_subproc, (filename, flags)),
         )
         proc.start()
         subproc_pipe.close()
@@ -984,7 +986,7 @@ class _unshare_validator:
         subproc_pipe.close()
 
     @staticmethod
-    def _validate_subproc(unshare, flags):
+    def _validate_subproc(filename, flags):
         """
         Perform validation. Calls to this method must be isolated in a
         subprocess, since the unshare function is called for purposes of
@@ -997,7 +999,10 @@ class _unshare_validator:
         @rtype: int
         @returns: errno value, or 0 if no error occurred.
         """
-        return 0 if unshare(flags) == 0 else ctypes.get_errno()
+        # Since ctypes objects are not picklable for the multiprocessing
+        # spawn start method, acquire them here.
+        libc = LoadLibrary(filename)
+        return 0 if libc.unshare(flags) == 0 else ctypes.get_errno()
 
 
 _unshare_validate = _unshare_validator()

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     cc7c40199850acb3d36f0a6452987231d592c360
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Nov  1 05:00:13 2023 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Nov  2 14:58:09 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=cc7c4019

Use documented os.register_at_fork function

The multiprocessing.util.register_after_fork function is
undocumented, so use the documented os.register_at_fork
function instead.

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index b26d0cfd53..aa81bdb4c2 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -16,7 +16,7 @@ try:
     if not hasattr(errno, "ESTALE"):
         # ESTALE may not be defined on some systems, such as interix.
         errno.ESTALE = -1
-    import multiprocessing.util
+    import functools
     import re
     import types
     import platform
@@ -421,7 +421,7 @@ class _ForkWatcher:
 
 _ForkWatcher.hook(_ForkWatcher)
 
-multiprocessing.util.register_after_fork(_ForkWatcher, _ForkWatcher.hook)
+os.register_at_fork(after_in_child=functools.partial(_ForkWatcher.hook, _ForkWatcher))
 
 
 def getpid():

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     7100dfb769b9d98eb40a92c37d154cb4bc83292f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 26 22:24:59 2023 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Dec 26 22:28:07 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7100dfb7

_global_updates: Acquire global vardbapi lock

Also use bindbapi.writable attribute in case the
PKGDIR is missing for some reason.

Bug: https://bugs.gentoo.org/587088
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_global_updates.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py
index ba183e87b5..f7997fc37c 100644
--- a/lib/portage/_global_updates.py
+++ b/lib/portage/_global_updates.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import stat
@@ -38,9 +38,14 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
     if secpass < 2 or "SANDBOX_ACTIVE" in os.environ or len(trees) != 1:
         return False
 
-    return _do_global_updates(
-        trees, prev_mtimes, quiet=quiet, if_mtime_changed=if_mtime_changed
-    )
+    vardb = trees[trees._running_eroot]["vartree"].dbapi
+    vardb.lock()
+    try:
+        return _do_global_updates(
+            trees, prev_mtimes, quiet=quiet, if_mtime_changed=if_mtime_changed
+        )
+    finally:
+        vardb.unlock()
 
 
 def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
@@ -127,7 +132,7 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
                 retupd = True
 
     if retupd:
-        if os.access(bindb.bintree.pkgdir, os.W_OK):
+        if bindb.writable:
             # Call binarytree.populate(), since we want to make sure it's
             # only populated with local packages here (getbinpkgs=0).
             bindb.bintree.populate()

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     7ec7a647ef6e2d94e6f2b387d0a012e78cafbaff
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 29 08:20:55 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jan 29 08:43:52 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7ec7a647

process.spawn: add abstraction layer for os.fork()

Since os.fork() is unsafe in threaded processes, add a basic
abstraction layer that will ultimately allow support of other process
cloning implementations.

Usage of os.fork() is now wrapped in a _start_fork function which can
easily be replaced with an alternative implementation.  This function
takes target, args, and kwargs arguments which are equivalent to
the corresponding multiprocessing.Process parameters. It also has
fd_pipes and close_fds parameters, since implementation of these is
dependent on the process cloning implementation.

The process cloning implementation does not need to be bothered with
the details of the _exec function, since spawn uses an _exec_wrapper
function as a target function which calls _exec and handles any
exceptions appropriately (special exception handling is required for
success of test_spawnE2big related to bug 830187).

Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 203 +++++++++++++++++++++++++++++++++----------------
 1 file changed, 139 insertions(+), 64 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 0d58adecad..be1c5d350a 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -18,6 +18,7 @@ import os as _os
 
 from dataclasses import dataclass
 from functools import lru_cache
+from typing import Any, Optional, Callable
 
 from portage import os
 from portage import _encodings
@@ -450,68 +451,31 @@ def spawn(
     # fork, so that the result is cached in the main process.
     bool(groups)
 
-    parent_pid = portage.getpid()
-    pid = None
-    try:
-        pid = os.fork()
-
-        if pid == 0:
-            try:
-                _exec(
-                    binary,
-                    mycommand,
-                    opt_name,
-                    fd_pipes,
-                    env,
-                    gid,
-                    groups,
-                    uid,
-                    umask,
-                    cwd,
-                    pre_exec,
-                    close_fds,
-                    unshare_net,
-                    unshare_ipc,
-                    unshare_mount,
-                    unshare_pid,
-                    unshare_flags,
-                )
-            except SystemExit:
-                raise
-            except Exception as e:
-                if isinstance(e, OSError) and e.errno == errno.E2BIG:
-                    # If exec() failed with E2BIG, then this is
-                    # potentially because the environment variables
-                    # grew to large. The following will gather some
-                    # stats about the environment and print a
-                    # diagnostic message to help identifying the
-                    # culprit. See also
-                    # - https://bugs.gentoo.org/721088
-                    # - https://bugs.gentoo.org/830187
-                    if not env_stats:
-                        env_stats = calc_env_stats(env)
-
-                    writemsg(
-                        f"ERROR: Executing {mycommand} failed with E2BIG. Child process environment size: {env_stats.env_size} bytes. Largest environment variable: {env_stats.env_largest_name} ({env_stats.env_largest_size} bytes)\n"
-                    )
-
-                # We need to catch _any_ exception so that it doesn't
-                # propagate out of this function and cause exiting
-                # with anything other than os._exit()
-                writemsg(f"{e}:\n   {' '.join(mycommand)}\n", noiselevel=-1)
-                traceback.print_exc()
-                sys.stderr.flush()
-
-    finally:
-        # Don't used portage.getpid() here, in case there is a race
-        # with getpid cache invalidation via _ForkWatcher hook.
-        if pid == 0 or (pid is None and _os.getpid() != parent_pid):
-            # Call os._exit() from a finally block in order
-            # to suppress any finally blocks from earlier
-            # in the call stack (see bug #345289). This
-            # finally block has to be setup before the fork
-            # in order to avoid a race condition.
-            os._exit(1)
+    pid = _start_fork(
+        _exec_wrapper,
+        args=(
+            binary,
+            mycommand,
+            opt_name,
+            fd_pipes,
+            env,
+            gid,
+            groups,
+            uid,
+            umask,
+            cwd,
+            pre_exec,
+            close_fds,
+            unshare_net,
+            unshare_ipc,
+            unshare_mount,
+            unshare_pid,
+            unshare_flags,
+            env_stats,
+        ),
+        fd_pipes=fd_pipes,
+        close_fds=close_fds,
+    )
 
     if not isinstance(pid, int):
         raise AssertionError(f"fork returned non-integer: {repr(pid)}")
@@ -633,6 +597,76 @@ def _configure_loopback_interface():
         )
 
 
+def _exec_wrapper(
+    binary,
+    mycommand,
+    opt_name,
+    fd_pipes,
+    env,
+    gid,
+    groups,
+    uid,
+    umask,
+    cwd,
+    pre_exec,
+    close_fds,
+    unshare_net,
+    unshare_ipc,
+    unshare_mount,
+    unshare_pid,
+    unshare_flags,
+    env_stats,
+):
+    """
+    Calls _exec with the given args and handles any raised Exception.
+    The intention is for _exec_wrapper and _exec to be reusable with
+    other process cloning implementations besides _start_fork.
+    """
+    try:
+        _exec(
+            binary,
+            mycommand,
+            opt_name,
+            fd_pipes,
+            env,
+            gid,
+            groups,
+            uid,
+            umask,
+            cwd,
+            pre_exec,
+            close_fds,
+            unshare_net,
+            unshare_ipc,
+            unshare_mount,
+            unshare_pid,
+            unshare_flags,
+        )
+    except Exception as e:
+        if isinstance(e, OSError) and e.errno == errno.E2BIG:
+            # If exec() failed with E2BIG, then this is
+            # potentially because the environment variables
+            # grew to large. The following will gather some
+            # stats about the environment and print a
+            # diagnostic message to help identifying the
+            # culprit. See also
+            # - https://bugs.gentoo.org/721088
+            # - https://bugs.gentoo.org/830187
+            if not env_stats:
+                env_stats = calc_env_stats(env)
+
+            writemsg(
+                f"ERROR: Executing {mycommand} failed with E2BIG. Child process environment size: {env_stats.env_size} bytes. Largest environment variable: {env_stats.env_largest_name} ({env_stats.env_largest_size} bytes)\n"
+            )
+
+        # We need to catch _any_ exception so that it doesn't
+        # propagate out of this function and cause exiting
+        # with anything other than os._exit()
+        writemsg(f"{e}:\n   {' '.join(mycommand)}\n", noiselevel=-1)
+        traceback.print_exc()
+        sys.stderr.flush()
+
+
 def _exec(
     binary,
     mycommand,
@@ -733,8 +767,6 @@ def _exec(
     # the parent process (see bug #289486).
     signal.signal(signal.SIGQUIT, signal.SIG_DFL)
 
-    _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
-
     # Unshare (while still uid==0)
     if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
         filename = find_library("c")
@@ -1114,6 +1146,49 @@ def _setup_pipes(fd_pipes, close_fds=True, inheritable=None):
                     pass
 
 
+def _start_fork(
+    target: Callable[..., None],
+    args: Optional[tuple[Any, ...]] = (),
+    kwargs: Optional[dict[str, Any]] = {},
+    fd_pipes: Optional[dict[int, int]] = None,
+    close_fds: Optional[bool] = True,
+) -> int:
+    """
+    Execute the target function in a fork. The fd_pipes and
+    close_fds parameters are handled in the fork, before the target
+    function is called. The args and kwargs parameters are passed
+    as positional and keyword arguments for the target function.
+
+    The target, args, and kwargs parameters are intended to
+    be equivalent to the corresponding multiprocessing.Process
+    constructor parameters.
+
+    Ultimately, the intention is for spawn to support other
+    process cloning implementations besides _start_fork, since
+    fork is unsafe for threaded processes as discussed in
+    https://github.com/python/cpython/issues/84559.
+    """
+    parent_pid = portage.getpid()
+    pid = None
+    try:
+        pid = os.fork()
+
+        if pid == 0:
+            _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
+            target(*args, **kwargs)
+    finally:
+        # Don't used portage.getpid() here, in case there is a race
+        # with getpid cache invalidation via _ForkWatcher hook.
+        if pid == 0 or (pid is None and _os.getpid() != parent_pid):
+            # Call os._exit() from a finally block in order
+            # to suppress any finally blocks from earlier
+            # in the call stack (see bug #345289). This
+            # finally block has to be setup before the fork
+            # in order to avoid a race condition.
+            os._exit(1)
+    return pid
+
+
 def find_binary(binary):
     """
     Given a binary name, find the binary in PATH

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     c82dc8c8fe92979b82df4c71bb9961973367e8f9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 29 17:38:20 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jan 29 17:43:27 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c82dc8c8

_start_fork: Ensure any _setup_pipes exception is displayed

Fixes: 7ec7a647ef6e ("process.spawn: add abstraction layer for os.fork()")
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index be1c5d350a..71750a715f 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -658,13 +658,8 @@ def _exec_wrapper(
             writemsg(
                 f"ERROR: Executing {mycommand} failed with E2BIG. Child process environment size: {env_stats.env_size} bytes. Largest environment variable: {env_stats.env_largest_name} ({env_stats.env_largest_size} bytes)\n"
             )
-
-        # We need to catch _any_ exception so that it doesn't
-        # propagate out of this function and cause exiting
-        # with anything other than os._exit()
         writemsg(f"{e}:\n   {' '.join(mycommand)}\n", noiselevel=-1)
-        traceback.print_exc()
-        sys.stderr.flush()
+        raise
 
 
 def _exec(
@@ -1174,8 +1169,14 @@ def _start_fork(
         pid = os.fork()
 
         if pid == 0:
-            _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
-            target(*args, **kwargs)
+            try:
+                _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
+                target(*args, **kwargs)
+            except Exception:
+                # We need to catch _any_ exception and display it since the child
+                # process must unconditionally exit via os._exit() if exec fails.
+                traceback.print_exc()
+                sys.stderr.flush()
     finally:
         # Don't used portage.getpid() here, in case there is a race
         # with getpid cache invalidation via _ForkWatcher hook.

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     f97e414ce980299acc962e357db24106d62e4c7c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb  4 06:12:00 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Feb  4 08:27:37 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f97e414c

set_term_size: Wait asynchronously if event loop is running

When set_term_size is called from an asynchronous context, we
can wait for the process to exit asynchronously. This will
prevent a "RuntimeError: This event loop is already running"
error in the future when synchronous spawn calls will need
to run the event loop in order to wait for the spawned
process(es).

Bug: https://bugs.gentoo.org/923750
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/output.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/lib/portage/output.py b/lib/portage/output.py
index cdeeb18e99..7d3a6278f3 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __docformat__ = "epytext"
@@ -13,7 +13,9 @@ import portage
 
 portage.proxy.lazyimport.lazyimport(
     globals(),
+    "portage.process:spawn",
     "portage.util:writemsg",
+    "portage.util.futures:asyncio",
 )
 import portage.util.formatter as formatter
 
@@ -557,13 +559,20 @@ def set_term_size(lines, columns, fd):
     Set the number of lines and columns for the tty that is connected to fd.
     For portability, this simply calls `stty rows $lines columns $columns`.
     """
-    from portage.process import spawn
 
     cmd = ["stty", "rows", str(lines), "columns", str(columns)]
     try:
-        spawn(cmd, env=os.environ, fd_pipes={0: fd})
+        proc = spawn(cmd, env=os.environ, fd_pipes={0: fd}, returnproc=True)
     except CommandNotFound:
         writemsg(_("portage: stty: command not found\n"), noiselevel=-1)
+    else:
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            asyncio.ensure_future(proc.wait(), loop).add_done_callback(
+                lambda future: future.result()
+            )
+        else:
+            loop.run_until_complete(proc.wait())
 
 
 class EOutput:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     c65d2d8b6c17d849e85f8c13c1a287ff1a07ccbd
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb  4 07:22:37 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Feb  4 08:27:37 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c65d2d8b

process.spawn: Avoid os.environ pickling error

This error was observed when pickling os.environ for
muliprocessing start method "spawn" after set_term_size
was updated to use returnproc:

  File "/usr/lib/python3.12/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object '_createenviron.<locals>.encode'. Did you mean: '_download_dir'?

Bug: https://bugs.gentoo.org/923750
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 01426179d7..b223ecb887 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -527,6 +527,9 @@ def spawn(
         mycommand = mycommand.split()
 
     env = os.environ if env is None else env
+    # Sometimes os.environ can fail to pickle as shown in bug 923750
+    # comment 4, so copy it to a dict.
+    env = env if isinstance(env, dict) else dict(env)
 
     env_stats = None
     if warn_on_large_env:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     1ae4f574f69eca3146137ffb40c5afd8a4872777
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb  4 00:22:08 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb  7 00:55:46 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1ae4f574

process.spawn: Enable returnpid warning for internals

All internal returnpid consumers have been migrated to
use the new returnproc parameter instead.

Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 20327b38bc..1ca59ee594 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -680,12 +680,11 @@ def spawn(
     # If the caller wants to handle cleaning up the processes, we tell
     # it about all processes that were created.
     if returnpid:
-        if not portage._internal_caller:
-            warnings.warn(
-                "The portage.process.spawn returnpid paramenter is deprecated and replaced by returnproc",
-                UserWarning,
-                stacklevel=1,
-            )
+        warnings.warn(
+            "The portage.process.spawn returnpid parameter is deprecated and replaced by returnproc",
+            UserWarning,
+            stacklevel=1,
+        )
         return mypids
 
     # Otherwise we clean them up.

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     3e10368e52ecead86b75478ca448ef0f59333e3e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb  4 04:34:45 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb  7 00:55:46 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e10368e

process.spawn: Use _start_proc for returnpid=False

This essentially completes the implementation of bug 916566,
eliminating os.fork() usage when "spawn" becomes the default
multiprocessing start method.

Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 1ca59ee594..a33e7b4747 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -586,10 +586,10 @@ def spawn(
 
         # Create a tee process, giving it our stdout and stderr
         # as well as the read end of the pipe.
-        mypids.extend(
+        mypids.append(
             spawn(
                 ("tee", "-i", "-a", logfile),
-                returnpid=True,
+                returnproc=True,
                 fd_pipes={0: pr, 1: fd_pipes[1], 2: fd_pipes[2]},
             )
         )
@@ -634,7 +634,7 @@ def spawn(
     # fork, so that the result is cached in the main process.
     bool(groups)
 
-    start_func = _start_proc if returnproc else _start_fork
+    start_func = _start_proc if returnproc or not returnpid else _start_fork
 
     pid = start_func(
         _exec_wrapper,
@@ -666,7 +666,7 @@ def spawn(
         # _start_proc returns a MultiprocessingProcess instance.
         return pid
 
-    if not isinstance(pid, int):
+    if returnpid and not isinstance(pid, int):
         raise AssertionError(f"fork returned non-integer: {repr(pid)}")
 
     # Add the pid to our local and the global pid lists.
@@ -687,6 +687,8 @@ def spawn(
         )
         return mypids
 
+    loop = global_event_loop()
+
     # Otherwise we clean them up.
     while mypids:
         # Pull the last reader in the pipe chain. If all processes
@@ -695,25 +697,22 @@ def spawn(
         pid = mypids.pop(0)
 
         # and wait for it.
-        retval = os.waitpid(pid, 0)[1]
+        retval = loop.run_until_complete(pid.wait())
 
         if retval:
             # If it failed, kill off anything else that
             # isn't dead yet.
             for pid in mypids:
-                # With waitpid and WNOHANG, only check the
-                # first element of the tuple since the second
-                # element may vary (bug #337465).
-                if os.waitpid(pid, os.WNOHANG)[0] == 0:
-                    os.kill(pid, signal.SIGTERM)
-                    os.waitpid(pid, 0)
-
-            # If it got a signal, return the signal that was sent.
-            if retval & 0xFF:
-                return (retval & 0xFF) << 8
-
-            # Otherwise, return its exit code.
-            return retval >> 8
+                waiter = asyncio.ensure_future(pid.wait(), loop)
+                try:
+                    loop.run_until_complete(
+                        asyncio.wait_for(asyncio.shield(waiter), 0.001)
+                    )
+                except (TimeoutError, asyncio.TimeoutError):
+                    pid.terminate()
+                    loop.run_until_complete(waiter)
+
+            return retval
 
     # Everything succeeded
     return 0

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     fba76a545f29fb8b529197c25c64700ef77413ae
Author:     Matoro Mahri <matoro_gentoo <AT> matoro <DOT> tk>
AuthorDate: Wed Jan 31 20:25:40 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Feb  9 07:08:22 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=fba76a54

gpkg: on error, close stdin before killing external programs

Otherwise, this may trigger a BrokenPipeError.  Sensitive to buffer
fullness, this was observed on 64k page size system immediately after
triggering E2BIG error from kernel.

Signed-off-by: Matoro Mahri <matoro_gentoo <AT> matoro.tk>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 031b3f87cb..05f1b5f2a6 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -149,9 +149,9 @@ class tar_stream_writer:
         kill external program if any error happened in python
         """
         if self.proc is not None:
-            self.killed = True
-            self.proc.kill()
             self.proc.stdin.close()
+            self.proc.kill()
+            self.killed = True
             self.close()
 
     def _cmd_read_thread(self):
@@ -347,9 +347,9 @@ class tar_stream_reader:
         kill external program if any error happened in python
         """
         if self.proc is not None:
-            self.killed = True
-            self.proc.kill()
             self.proc.stdin.close()
+            self.proc.kill()
+            self.killed = True
             self.close()
 
     def read(self, bufsize=-1):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     c4049323b3c4c04ccc56023fa31169da58b57831
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Fri Feb  9 08:47:31 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Feb  9 08:51:17 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c4049323

Revert "gpkg: on error, close stdin before killing external programs"

This reverts commit fba76a545f29fb8b529197c25c64700ef77413ae. This seems
to cause hangs in CI with pypy3.

Zac mentioned this at https://github.com/gentoo/portage/pull/1246#issuecomment-1935511379
and I can reproduce it with `tox -e pypy3-test` with PyPy 7.3.15.

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 05f1b5f2a6..031b3f87cb 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -149,9 +149,9 @@ class tar_stream_writer:
         kill external program if any error happened in python
         """
         if self.proc is not None:
-            self.proc.stdin.close()
-            self.proc.kill()
             self.killed = True
+            self.proc.kill()
+            self.proc.stdin.close()
             self.close()
 
     def _cmd_read_thread(self):
@@ -347,9 +347,9 @@ class tar_stream_reader:
         kill external program if any error happened in python
         """
         if self.proc is not None:
-            self.proc.stdin.close()
-            self.proc.kill()
             self.killed = True
+            self.proc.kill()
+            self.proc.stdin.close()
             self.close()
 
     def read(self, bufsize=-1):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     ad61940b03be2f24c0b54c1070a4923abe18e633
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb  9 16:22:45 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb  9 23:52:11 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ad61940b

gpkg: Less aggressive subprocess.Popen kill in order to avoid BrokenPipeError

Do not kill tar_stream_reader instances if we can successfully
close them, since that can trigger BrokenPipeError during
self.proc.stdin.close() calls, and this state is best avoided
because it's unclear how the caller should handle the error.

If a BrokenPipeError does occur then simply print a traceback
and hope that the corresponding file descriptor is closed
during garbage collection. Do not try to reverse the order
of self.proc.kill() and self.proc.stdin.close() as in commit
fba76a545f2 since that triggered pypy ci job hangs for which
no reliable solution has been found so far.

Bug: https://bugs.gentoo.org/923368
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/gpkg.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 031b3f87cb..f1d8f97f8e 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1,7 +1,8 @@
-# Copyright 2001-2020 Gentoo Authors
+# Copyright 2001-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import tarfile
+import traceback
 import io
 import threading
 import subprocess
@@ -151,7 +152,10 @@ class tar_stream_writer:
         if self.proc is not None:
             self.killed = True
             self.proc.kill()
-            self.proc.stdin.close()
+            try:
+                self.proc.stdin.close()
+            except BrokenPipeError:
+                traceback.print_exc()
             self.close()
 
     def _cmd_read_thread(self):
@@ -213,7 +217,7 @@ class tar_stream_writer:
         if self.proc is not None:
             self.proc.stdin.close()
             if self.proc.wait() != os.EX_OK:
-                if not self.error:
+                if not (self.killed or self.error):
                     raise CompressorOperationFailed("compression failed")
             if self.read_thread.is_alive():
                 self.read_thread.join()
@@ -349,7 +353,10 @@ class tar_stream_reader:
         if self.proc is not None:
             self.killed = True
             self.proc.kill()
-            self.proc.stdin.close()
+            try:
+                self.proc.stdin.close()
+            except BrokenPipeError:
+                traceback.print_exc()
             self.close()
 
     def read(self, bufsize=-1):
@@ -986,11 +993,13 @@ class gpkg:
                     try:
                         image_safe = tar_safe_extract(image, "image")
                         image_safe.extractall(decompress_dir)
+                        image_tar.close()
                     except Exception as ex:
                         writemsg(colorize("BAD", "!!!Extract failed."))
                         raise
                     finally:
-                        image_tar.kill()
+                        if not image_tar.closed:
+                            image_tar.kill()
 
     def update_metadata(self, metadata, new_basename=None, force=False):
         """

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     03be12ec5f46629fa928e5fcd45d3fe6745d5d0a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb  9 22:12:02 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 10 06:08:59 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=03be12ec

GPG: Use threading.Event for thread safety

Use threading.Event for thread safety during GPG stop, and use the
wait method to improve responsiveness for stop requests.

Bug: https://bugs.gentoo.org/924192
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/gpg.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/portage/gpg.py b/lib/portage/gpg.py
index 3067872244..d8a4cfcfc4 100644
--- a/lib/portage/gpg.py
+++ b/lib/portage/gpg.py
@@ -1,10 +1,9 @@
-# Copyright 2001-2020 Gentoo Authors
+# Copyright 2001-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import subprocess
 import sys
 import threading
-import time
 
 from portage import os
 from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
@@ -24,6 +23,7 @@ class GPG:
         """
         self.settings = settings
         self.thread = None
+        self._terminated = None
         self.GPG_signing_base_command = self.settings.get(
             "BINPKG_GPG_SIGNING_BASE_COMMAND"
         )
@@ -73,6 +73,7 @@ class GPG:
                 self.GPG_unlock_command = shlex_split(
                     varexpand(self.GPG_unlock_command, mydict=self.settings)
                 )
+                self._terminated = threading.Event()
                 self.thread = threading.Thread(target=self.gpg_keepalive, daemon=True)
                 self.thread.start()
 
@@ -81,16 +82,17 @@ class GPG:
         Stop keepalive thread.
         """
         if self.thread is not None:
-            self.keepalive = False
+            self._terminated.set()
 
     def gpg_keepalive(self):
         """
         Call GPG unlock command every 5 mins to avoid the passphrase expired.
         """
         count = 0
-        while self.keepalive:
+        while not self._terminated.is_set():
             if count < 5:
-                time.sleep(60)
+                if self._terminated.wait(60):
+                    break
                 count += 1
                 continue
             else:
@@ -102,5 +104,5 @@ class GPG:
                 stdout=subprocess.DEVNULL,
                 stderr=subprocess.STDOUT,
             )
-            if proc.wait() != os.EX_OK:
+            if proc.wait() != os.EX_OK and not self._terminated.is_set():
                 raise GPGException("GPG keepalive failed")

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     a1024a6d02ca3a55f86525e0d8d5089e754d3713
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 11 05:38:58 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Feb 11 19:46:55 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a1024a6d

spawn_wrapper: Make pre_exec function picklable

Local functions are unpicklable, which triggered this error
with the multiprocessing "spawn" start method:

AttributeError: Can't pickle local object 'spawn_wrapper.__call__.<locals>._pre_exec'

Bug: https://bugs.gentoo.org/924273
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_selinux.py | 17 ++++++++---------
 lib/portage/process.py  | 26 ++++++++++++++++++--------
 2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/lib/portage/_selinux.py b/lib/portage/_selinux.py
index bf6ad24895..5ae1b4e715 100644
--- a/lib/portage/_selinux.py
+++ b/lib/portage/_selinux.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 # Don't use the unicode-wrapped os and shutil modules here since
@@ -6,6 +6,7 @@
 import os
 import shutil
 import warnings
+from functools import partial
 
 try:
     import selinux
@@ -134,14 +135,12 @@ class spawn_wrapper:
 
     def __call__(self, *args, **kwargs):
         if self._con is not None:
-            pre_exec = kwargs.get("pre_exec")
-
-            def _pre_exec():
-                if pre_exec is not None:
-                    pre_exec()
-                setexec(self._con)
-
-            kwargs["pre_exec"] = _pre_exec
+            pre_exec = partial(setexec, self._con)
+            kwargs["pre_exec"] = (
+                portage.process._chain_pre_exec_fns(pre_exec, kwargs["pre_exec"])
+                if kwargs.get("pre_exec")
+                else pre_exec
+            )
 
         return self._spawn_func(*args, **kwargs)
 

diff --git a/lib/portage/process.py b/lib/portage/process.py
index a33e7b4747..6cad250e34 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -18,7 +18,7 @@ import os as _os
 import warnings
 
 from dataclasses import dataclass
-from functools import lru_cache
+from functools import lru_cache, partial
 from typing import Any, Optional, Callable, Union
 
 from portage import os
@@ -1383,18 +1383,28 @@ def _start_fork(
     return pid
 
 
-class _setup_pipes_after_fork:
-    def __init__(self, target, fd_pipes):
+class _chain_pre_exec_fns:
+    """
+    Wraps a target function to call pre_exec functions just before
+    the original target function.
+    """
+
+    def __init__(self, target, *args):
         self._target = target
-        self._fd_pipes = fd_pipes
+        self._pre_exec_fns = args
 
     def __call__(self, *args, **kwargs):
-        for fd in set(self._fd_pipes.values()):
-            os.set_inheritable(fd, True)
-        _setup_pipes(self._fd_pipes, close_fds=False, inheritable=True)
+        for pre_exec in self._pre_exec_fns:
+            pre_exec()
         return self._target(*args, **kwargs)
 
 
+def _setup_pipes_after_fork(fd_pipes):
+    for fd in set(fd_pipes.values()):
+        os.set_inheritable(fd, True)
+    _setup_pipes(fd_pipes, close_fds=False, inheritable=True)
+
+
 def _start_proc(
     target: Callable[..., None],
     args: Optional[tuple[Any, ...]] = (),
@@ -1419,7 +1429,7 @@ def _start_proc(
     # which ForkProcess does not handle because its target
     # function does not necessarily exec.
     if fd_pipes and multiprocessing.get_start_method() == "fork":
-        target = _setup_pipes_after_fork(target, fd_pipes)
+        target = _chain_pre_exec_fns(target, partial(_setup_pipes_after_fork, fd_pipes))
         fd_pipes = None
 
     proc = ForkProcess(

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     419cce79f9082308c848df0a98f367de4d1c50a3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 11 21:58:10 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb 12 07:56:10 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=419cce79

process._exec: Use _start_fork for os.fork() error handling

Use _start_fork for os.fork() error handling, ensuring
that if exec fails then the child process will display
a traceback before it exits via os._exit to suppress any
finally blocks from parent's call stack (bug 345289).

Bug: https://bugs.gentoo.org/345289
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/924313
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 259 ++++++++++++++++++++++++++++---------------------
 1 file changed, 151 insertions(+), 108 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 6cad250e34..f4758c824c 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -961,7 +961,13 @@ def _exec(
         if filename is not None:
             libc = LoadLibrary(filename)
             if libc is not None:
-                try:
+                # unshare() may not be supported by libc
+                if not hasattr(libc, "unshare"):
+                    unshare_net = False
+                    unshare_ipc = False
+                    unshare_mount = False
+                    unshare_pid = False
+                else:
                     # Since a failed unshare call could corrupt process
                     # state, first validate that the call can succeed.
                     # The parent process should call _unshare_validate
@@ -992,117 +998,154 @@ def _exec(
                         )
                     else:
                         if unshare_pid:
-                            main_child_pid = os.fork()
-                            if main_child_pid == 0:
-                                # pid namespace requires us to become init
-                                binary, myargs = (
-                                    portage._python_interpreter,
-                                    [
-                                        portage._python_interpreter,
-                                        os.path.join(portage._bin_path, "pid-ns-init"),
-                                        _unicode_encode(
-                                            "" if uid is None else str(uid)
-                                        ),
-                                        _unicode_encode(
-                                            "" if gid is None else str(gid)
-                                        ),
-                                        _unicode_encode(
-                                            ""
-                                            if groups is None
-                                            else ",".join(
-                                                str(group) for group in groups
-                                            )
-                                        ),
-                                        _unicode_encode(
-                                            "" if umask is None else str(umask)
-                                        ),
-                                        _unicode_encode(
-                                            ",".join(str(fd) for fd in fd_pipes)
-                                        ),
-                                        binary,
-                                    ]
-                                    + myargs,
-                                )
-                                uid = None
-                                gid = None
-                                groups = None
-                                umask = None
-                            else:
-                                # Execute a supervisor process which will forward
-                                # signals to init and forward exit status to the
-                                # parent process. The supervisor process runs in
-                                # the global pid namespace, so skip /proc remount
-                                # and other setup that's intended only for the
-                                # init process.
-                                binary, myargs = portage._python_interpreter, [
+                            # pid namespace requires us to become init
+                            binary, myargs = (
+                                portage._python_interpreter,
+                                [
                                     portage._python_interpreter,
                                     os.path.join(portage._bin_path, "pid-ns-init"),
-                                    str(main_child_pid),
+                                    _unicode_encode("" if uid is None else str(uid)),
+                                    _unicode_encode("" if gid is None else str(gid)),
+                                    _unicode_encode(
+                                        ""
+                                        if groups is None
+                                        else ",".join(str(group) for group in groups)
+                                    ),
+                                    _unicode_encode(
+                                        "" if umask is None else str(umask)
+                                    ),
+                                    _unicode_encode(
+                                        ",".join(str(fd) for fd in fd_pipes)
+                                    ),
+                                    binary,
                                 ]
+                                + myargs,
+                            )
+                            uid = None
+                            gid = None
+                            groups = None
+                            umask = None
+
+                            # Use _start_fork for os.fork() error handling, ensuring
+                            # that if exec fails then the child process will display
+                            # a traceback before it exits via os._exit to suppress any
+                            # finally blocks from parent's call stack (bug 345289).
+                            main_child_pid = _start_fork(
+                                _exec2,
+                                args=(
+                                    binary,
+                                    myargs,
+                                    env,
+                                    gid,
+                                    groups,
+                                    uid,
+                                    umask,
+                                    cwd,
+                                    pre_exec,
+                                    unshare_net,
+                                    unshare_ipc,
+                                    unshare_mount,
+                                    unshare_pid,
+                                ),
+                                fd_pipes=None,
+                                close_fds=False,
+                            )
 
-                                os.execve(binary, myargs, env)
+                            # Execute a supervisor process which will forward
+                            # signals to init and forward exit status to the
+                            # parent process. The supervisor process runs in
+                            # the global pid namespace, so skip /proc remount
+                            # and other setup that's intended only for the
+                            # init process.
+                            binary, myargs = portage._python_interpreter, [
+                                portage._python_interpreter,
+                                os.path.join(portage._bin_path, "pid-ns-init"),
+                                str(main_child_pid),
+                            ]
+
+                            os.execve(binary, myargs, env)
+
+    # Reachable only if unshare_pid is False.
+    _exec2(
+        binary,
+        myargs,
+        env,
+        gid,
+        groups,
+        uid,
+        umask,
+        cwd,
+        pre_exec,
+        unshare_net,
+        unshare_ipc,
+        unshare_mount,
+        unshare_pid,
+    )
 
-                        if unshare_mount:
-                            # mark the whole filesystem as slave to avoid
-                            # mounts escaping the namespace
-                            s = subprocess.Popen(["mount", "--make-rslave", "/"])
-                            mount_ret = s.wait()
-                            if mount_ret != 0:
-                                # TODO: should it be fatal maybe?
-                                writemsg(
-                                    "Unable to mark mounts slave: %d\n" % (mount_ret,),
-                                    noiselevel=-1,
-                                )
-                        if unshare_pid:
-                            # we need at least /proc being slave
-                            s = subprocess.Popen(["mount", "--make-slave", "/proc"])
-                            mount_ret = s.wait()
-                            if mount_ret != 0:
-                                # can't proceed with shared /proc
-                                writemsg(
-                                    "Unable to mark /proc slave: %d\n" % (mount_ret,),
-                                    noiselevel=-1,
-                                )
-                                os._exit(1)
-                            # mount new /proc for our namespace
-                            s = subprocess.Popen(
-                                ["mount", "-n", "-t", "proc", "proc", "/proc"]
-                            )
-                            mount_ret = s.wait()
-                            if mount_ret != 0:
-                                writemsg(
-                                    "Unable to mount new /proc: %d\n" % (mount_ret,),
-                                    noiselevel=-1,
-                                )
-                                os._exit(1)
-                        if unshare_net:
-                            # use 'localhost' to avoid hostname resolution problems
-                            try:
-                                # pypy3 does not implement socket.sethostname()
-                                new_hostname = b"localhost"
-                                if hasattr(socket, "sethostname"):
-                                    socket.sethostname(new_hostname)
-                                else:
-                                    if (
-                                        libc.sethostname(
-                                            new_hostname, len(new_hostname)
-                                        )
-                                        != 0
-                                    ):
-                                        errno_value = ctypes.get_errno()
-                                        raise OSError(
-                                            errno_value, os.strerror(errno_value)
-                                        )
-                            except Exception as e:
-                                writemsg(
-                                    'Unable to set hostname: %s (for FEATURES="network-sandbox")\n'
-                                    % (e,),
-                                    noiselevel=-1,
-                                )
-                            _configure_loopback_interface()
-                except AttributeError:
-                    # unshare() not supported by libc
-                    pass
+
+def _exec2(
+    binary,
+    myargs,
+    env,
+    gid,
+    groups,
+    uid,
+    umask,
+    cwd,
+    pre_exec,
+    unshare_net,
+    unshare_ipc,
+    unshare_mount,
+    unshare_pid,
+):
+    if unshare_mount:
+        # mark the whole filesystem as slave to avoid
+        # mounts escaping the namespace
+        s = subprocess.Popen(["mount", "--make-rslave", "/"])
+        mount_ret = s.wait()
+        if mount_ret != 0:
+            # TODO: should it be fatal maybe?
+            writemsg(
+                "Unable to mark mounts slave: %d\n" % (mount_ret,),
+                noiselevel=-1,
+            )
+    if unshare_pid:
+        # we need at least /proc being slave
+        s = subprocess.Popen(["mount", "--make-slave", "/proc"])
+        mount_ret = s.wait()
+        if mount_ret != 0:
+            # can't proceed with shared /proc
+            writemsg(
+                "Unable to mark /proc slave: %d\n" % (mount_ret,),
+                noiselevel=-1,
+            )
+            os._exit(1)
+        # mount new /proc for our namespace
+        s = subprocess.Popen(["mount", "-n", "-t", "proc", "proc", "/proc"])
+        mount_ret = s.wait()
+        if mount_ret != 0:
+            writemsg(
+                "Unable to mount new /proc: %d\n" % (mount_ret,),
+                noiselevel=-1,
+            )
+            os._exit(1)
+    if unshare_net:
+        # use 'localhost' to avoid hostname resolution problems
+        try:
+            # pypy3 does not implement socket.sethostname()
+            new_hostname = b"localhost"
+            if hasattr(socket, "sethostname"):
+                socket.sethostname(new_hostname)
+            else:
+                if libc.sethostname(new_hostname, len(new_hostname)) != 0:
+                    errno_value = ctypes.get_errno()
+                    raise OSError(errno_value, os.strerror(errno_value))
+        except Exception as e:
+            writemsg(
+                f'Unable to set hostname: {e} (for FEATURES="network-sandbox")\n',
+                noiselevel=-1,
+            )
+        _configure_loopback_interface()
 
     # Set requested process permissions.
     if gid:
@@ -1260,7 +1303,7 @@ def _setup_pipes(fd_pipes, close_fds=True, inheritable=None):
     actually does nothing in this case), which avoids possible
     interference.
     """
-
+    fd_pipes = {} if fd_pipes is None else fd_pipes
     reverse_map = {}
     # To protect from cases where direct assignment could
     # clobber needed fds ({1:2, 2:1}) we create a reverse map

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     58b4c156543705f631444040c2b962c4ac760f6a
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 21 02:07:59 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Feb 21 02:08:07 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=58b4c156

gpkg: add missing newline to gpkg format error

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index f1d8f97f8e..edb0e43fbf 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -2111,7 +2111,7 @@ class gpkg:
 
         if self.basename and self.prefix and not self.prefix.startswith(self.basename):
             writemsg(
-                colorize("WARN", f"Package basename mismatched, using {self.prefix}")
+                colorize("WARN", f"Package basename mismatched, using {self.prefix}\n")
             )
 
         all_files = tar.getmembers()

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     14fe81d73c3330ec31bbc712ada40cc5bdda2c61
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 21 02:07:19 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Feb 21 02:07:19 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=14fe81d7

binpkg: add missing newline to gpkg format error

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/binpkg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index 2078e3ca58..9ecd52cf3c 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -67,7 +67,7 @@ def get_binpkg_format(binpkg_path, check_file=False, remote=False):
         writemsg(
             colorize(
                 "WARN",
-                "File {} binpkg format mismatch, actual format: {}".format(
+                "File {} binpkg format mismatch, actual format: {}\n".format(
                     binpkg_path, file_format
                 ),
             )

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     3f4250dc7d32e9915224b1c9c4bc04c2740abcda
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 23 20:35:04 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 24 20:07:39 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3f4250dc

process.spawn: Fix logic for missing libc.unshare on musl

Fix unshare_* variables to be False when the libc is missing,
libc.unshare is missing, or libc.unshare fails.

Also, if socket.sethostname is missing then _exec2 needs
libc for the network-sandbox sethostname call which is wrapped
by a blanket Exception handler.

Fixes: 419cce79f908 ("process._exec: Use _start_fork for os.fork() error handling")
Bug: https://bugs.gentoo.org/925311
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 213 +++++++++++++++++++++++++------------------------
 1 file changed, 110 insertions(+), 103 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index f4758c824c..d16262e75a 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -956,114 +956,119 @@ def _exec(
     signal.signal(signal.SIGQUIT, signal.SIG_DFL)
 
     # Unshare (while still uid==0)
+    have_unshare = False
+    libc = None
     if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
         filename = find_library("c")
         if filename is not None:
             libc = LoadLibrary(filename)
             if libc is not None:
-                # unshare() may not be supported by libc
-                if not hasattr(libc, "unshare"):
-                    unshare_net = False
-                    unshare_ipc = False
-                    unshare_mount = False
-                    unshare_pid = False
-                else:
-                    # Since a failed unshare call could corrupt process
-                    # state, first validate that the call can succeed.
-                    # The parent process should call _unshare_validate
-                    # before it forks, so that all child processes can
-                    # reuse _unshare_validate results that have been
-                    # cached by the parent process.
-                    errno_value = _unshare_validate(unshare_flags)
-                    if errno_value == 0 and libc.unshare(unshare_flags) != 0:
-                        errno_value = ctypes.get_errno()
-                    if errno_value != 0:
-                        involved_features = []
-                        if unshare_ipc:
-                            involved_features.append("ipc-sandbox")
-                        if unshare_mount:
-                            involved_features.append("mount-sandbox")
-                        if unshare_net:
-                            involved_features.append("network-sandbox")
-                        if unshare_pid:
-                            involved_features.append("pid-sandbox")
-
-                        writemsg(
-                            'Unable to unshare: %s (for FEATURES="%s")\n'
-                            % (
-                                errno.errorcode.get(errno_value, "?"),
-                                " ".join(involved_features),
-                            ),
-                            noiselevel=-1,
-                        )
-                    else:
-                        if unshare_pid:
-                            # pid namespace requires us to become init
-                            binary, myargs = (
-                                portage._python_interpreter,
-                                [
-                                    portage._python_interpreter,
-                                    os.path.join(portage._bin_path, "pid-ns-init"),
-                                    _unicode_encode("" if uid is None else str(uid)),
-                                    _unicode_encode("" if gid is None else str(gid)),
-                                    _unicode_encode(
-                                        ""
-                                        if groups is None
-                                        else ",".join(str(group) for group in groups)
-                                    ),
-                                    _unicode_encode(
-                                        "" if umask is None else str(umask)
-                                    ),
-                                    _unicode_encode(
-                                        ",".join(str(fd) for fd in fd_pipes)
-                                    ),
-                                    binary,
-                                ]
-                                + myargs,
-                            )
-                            uid = None
-                            gid = None
-                            groups = None
-                            umask = None
-
-                            # Use _start_fork for os.fork() error handling, ensuring
-                            # that if exec fails then the child process will display
-                            # a traceback before it exits via os._exit to suppress any
-                            # finally blocks from parent's call stack (bug 345289).
-                            main_child_pid = _start_fork(
-                                _exec2,
-                                args=(
-                                    binary,
-                                    myargs,
-                                    env,
-                                    gid,
-                                    groups,
-                                    uid,
-                                    umask,
-                                    cwd,
-                                    pre_exec,
-                                    unshare_net,
-                                    unshare_ipc,
-                                    unshare_mount,
-                                    unshare_pid,
-                                ),
-                                fd_pipes=None,
-                                close_fds=False,
-                            )
-
-                            # Execute a supervisor process which will forward
-                            # signals to init and forward exit status to the
-                            # parent process. The supervisor process runs in
-                            # the global pid namespace, so skip /proc remount
-                            # and other setup that's intended only for the
-                            # init process.
-                            binary, myargs = portage._python_interpreter, [
-                                portage._python_interpreter,
-                                os.path.join(portage._bin_path, "pid-ns-init"),
-                                str(main_child_pid),
-                            ]
-
-                            os.execve(binary, myargs, env)
+                have_unshare = hasattr(libc, "unshare")
+
+    if not have_unshare:
+        # unshare() may not be supported by libc
+        unshare_net = False
+        unshare_ipc = False
+        unshare_mount = False
+        unshare_pid = False
+
+    if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
+        # Since a failed unshare call could corrupt process
+        # state, first validate that the call can succeed.
+        # The parent process should call _unshare_validate
+        # before it forks, so that all child processes can
+        # reuse _unshare_validate results that have been
+        # cached by the parent process.
+        errno_value = _unshare_validate(unshare_flags)
+        if errno_value == 0 and libc.unshare(unshare_flags) != 0:
+            errno_value = ctypes.get_errno()
+        if errno_value != 0:
+            involved_features = []
+            if unshare_ipc:
+                involved_features.append("ipc-sandbox")
+            if unshare_mount:
+                involved_features.append("mount-sandbox")
+            if unshare_net:
+                involved_features.append("network-sandbox")
+            if unshare_pid:
+                involved_features.append("pid-sandbox")
+
+            writemsg(
+                'Unable to unshare: %s (for FEATURES="%s")\n'
+                % (
+                    errno.errorcode.get(errno_value, "?"),
+                    " ".join(involved_features),
+                ),
+                noiselevel=-1,
+            )
+
+            unshare_net = False
+            unshare_ipc = False
+            unshare_mount = False
+            unshare_pid = False
+
+    if unshare_pid:
+        # pid namespace requires us to become init
+        binary, myargs = (
+            portage._python_interpreter,
+            [
+                portage._python_interpreter,
+                os.path.join(portage._bin_path, "pid-ns-init"),
+                _unicode_encode("" if uid is None else str(uid)),
+                _unicode_encode("" if gid is None else str(gid)),
+                _unicode_encode(
+                    "" if groups is None else ",".join(str(group) for group in groups)
+                ),
+                _unicode_encode("" if umask is None else str(umask)),
+                _unicode_encode(",".join(str(fd) for fd in fd_pipes)),
+                binary,
+            ]
+            + myargs,
+        )
+        uid = None
+        gid = None
+        groups = None
+        umask = None
+
+        # Use _start_fork for os.fork() error handling, ensuring
+        # that if exec fails then the child process will display
+        # a traceback before it exits via os._exit to suppress any
+        # finally blocks from parent's call stack (bug 345289).
+        main_child_pid = _start_fork(
+            _exec2,
+            args=(
+                binary,
+                myargs,
+                env,
+                gid,
+                groups,
+                uid,
+                umask,
+                cwd,
+                pre_exec,
+                unshare_net,
+                unshare_ipc,
+                unshare_mount,
+                unshare_pid,
+                libc,
+            ),
+            fd_pipes=None,
+            close_fds=False,
+        )
+
+        # Execute a supervisor process which will forward
+        # signals to init and forward exit status to the
+        # parent process. The supervisor process runs in
+        # the global pid namespace, so skip /proc remount
+        # and other setup that's intended only for the
+        # init process.
+        binary, myargs = portage._python_interpreter, [
+            portage._python_interpreter,
+            os.path.join(portage._bin_path, "pid-ns-init"),
+            str(main_child_pid),
+        ]
+
+        os.execve(binary, myargs, env)
 
     # Reachable only if unshare_pid is False.
     _exec2(
@@ -1080,6 +1085,7 @@ def _exec(
         unshare_ipc,
         unshare_mount,
         unshare_pid,
+        libc,
     )
 
 
@@ -1097,6 +1103,7 @@ def _exec2(
     unshare_ipc,
     unshare_mount,
     unshare_pid,
+    libc,
 ):
     if unshare_mount:
         # mark the whole filesystem as slave to avoid

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     92615cd37d7a1efce923c474e455f59fe61a589c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 25 05:09:48 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Feb 25 08:24:59 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=92615cd3

_start_proc: Prevent premature ForkProcess garbage collection

In order to prevent the constructed ForkProcess instance from
being prematurely garbage collected, return a reference to the
caller inside of a _GCProtector object, preventing messages
reported in bug 925456 like this:

    [ERROR] Task was destroyed but it is pending!

Fixes: a69c1b853a47 ("process.spawn: Use multiprocessing.Process for returnproc")
Bug: https://bugs.gentoo.org/925456
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1284
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/process.py | 41 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index d16262e75a..cc9ed7bf78 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -36,6 +36,7 @@ portage.proxy.lazyimport.lazyimport(
 
 from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
 from portage.exception import CommandNotFound
+from portage.proxy.objectproxy import ObjectProxy
 from portage.util._ctypes import find_library, LoadLibrary, ctypes
 
 try:
@@ -1493,8 +1494,44 @@ def _start_proc(
     proc.start()
 
     # ForkProcess conveniently holds a MultiprocessingProcess
-    # instance that is suitable to return here.
-    return proc._proc
+    # instance that is suitable to return here, but use _GCProtector
+    # to protect the ForkProcess instance from being garbage collected
+    # and triggering messages like this (bug 925456):
+    # [ERROR] Task was destroyed but it is pending!
+    return _GCProtector(proc._proc, proc.async_wait)
+
+
+class _GCProtector(ObjectProxy):
+    """
+    Proxy a target object, and also hold a reference to something
+    extra in order to protect it from garbage collection. Override
+    the wait method to first call target's wait method and then
+    wait for extra (a coroutine function) before returning the result.
+    """
+
+    __slots__ = ("_extra", "_target")
+
+    def __init__(self, target, extra):
+        super().__init__()
+        object.__setattr__(self, "_target", target)
+        object.__setattr__(self, "_extra", extra)
+
+    def _get_target(self):
+        return object.__getattribute__(self, "_target")
+
+    def __getattribute__(self, attr):
+        if attr == "wait":
+            return object.__getattribute__(self, attr)
+        return getattr(object.__getattribute__(self, "_target"), attr)
+
+    async def wait(self):
+        """
+        Wrap the target's wait method to also wait for an extra
+        coroutine function.
+        """
+        result = await object.__getattribute__(self, "_target").wait()
+        await object.__getattribute__(self, "_extra")()
+        return result
 
 
 def find_binary(binary):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     f070275fe05d5053c3756ebb5d0a602db8ba515d
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 15:48:48 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 15:49:16 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f070275f

binpkg: add another missing newline to error message

```
Error reading binpkg '/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar': [Errno 22] Invalid argument!!! Invalid binary package: '/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar', Error reading binpkg '/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar': [Errno 22] Invalid argument
```

Bug: https://bugs.gentoo.org/925714
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/binpkg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index 9ecd52cf3c..a48e09bdb8 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -54,7 +54,7 @@ def get_binpkg_format(binpkg_path, check_file=False, remote=False):
         # We got many different exceptions here, so have to catch all of them.
         file_format = None
         writemsg(
-            colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}"),
+            colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}\n"),
         )
         raise InvalidBinaryPackageFormat(f"Error reading binpkg '{binpkg_path}': {err}")
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     957902f84edece635210689f46e20741e76d9dba
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 15:51:15 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 15:51:15 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=957902f8

gpkg: add missing new lines to error messages

Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index edb0e43fbf..5cd1f2394e 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -382,7 +382,7 @@ class tar_stream_reader:
             try:
                 if self.proc.wait() != os.EX_OK:
                     if not self.killed:
-                        writemsg(colorize("BAD", f"GPKG external program failed."))
+                        writemsg(colorize("BAD", f"GPKG external program failed.\n"))
                         raise CompressorOperationFailed("decompression failed")
             finally:
                 self.proc.stdout.close()
@@ -418,7 +418,7 @@ class checksum_helper:
                 else:
                     self.uid = pwd.getpwnam(drop_user).pw_uid
             except KeyError:
-                writemsg(colorize("BAD", f"!!! Failed to find user {drop_user}."))
+                writemsg(colorize("BAD", f"!!! Failed to find user {drop_user}.\n"))
                 raise
 
             try:
@@ -428,7 +428,7 @@ class checksum_helper:
                 else:
                     self.gid = grp.getgrnam(drop_group).gr_gid
             except KeyError:
-                writemsg(colorize("BAD", f"!!! Failed to find group {drop_group}."))
+                writemsg(colorize("BAD", f"!!! Failed to find group {drop_group}.\n"))
                 raise
         else:
             self.uid = None
@@ -636,33 +636,33 @@ class tar_safe_extract:
                 ):
                     writemsg(
                         colorize(
-                            "BAD", f"Danger: duplicate files detected: {member.name}"
+                            "BAD", f"Danger: duplicate files detected: {member.name}\n"
                         )
                     )
                     raise ValueError("Duplicate files detected.")
                 if member.name.startswith("/"):
                     writemsg(
                         colorize(
-                            "BAD", f"Danger: absolute path detected: {member.name}"
+                            "BAD", f"Danger: absolute path detected: {member.name}\n"
                         )
                     )
                     raise ValueError("Absolute path detected.")
                 if member.name.startswith("../") or ("/../" in member.name):
                     writemsg(
                         colorize(
-                            "BAD", f"Danger: path traversal detected: {member.name}"
+                            "BAD", f"Danger: path traversal detected: {member.name}\n"
                         )
                     )
                     raise ValueError("Path traversal detected.")
                 if member.isdev():
                     writemsg(
-                        colorize("BAD", f"Danger: device file detected: {member.name}")
+                        colorize("BAD", f"Danger: device file detected: {member.name}\n")
                     )
                     raise ValueError("Device file detected.")
                 if member.islnk() and (member.linkname not in self.file_list):
                     writemsg(
                         colorize(
-                            "BAD", f"Danger: hardlink escape detected: {member.name}"
+                            "BAD", f"Danger: hardlink escape detected: {member.name}\n"
                         )
                     )
                     raise ValueError("Hardlink escape detected.")
@@ -995,7 +995,7 @@ class gpkg:
                         image_safe.extractall(decompress_dir)
                         image_tar.close()
                     except Exception as ex:
-                        writemsg(colorize("BAD", "!!!Extract failed."))
+                        writemsg(colorize("BAD", "!!!Extract failed.\n"))
                         raise
                     finally:
                         if not image_tar.closed:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     fa7fb4c5119aa077a0731e1f8892ea0791a4968b
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 28 16:00:57 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 16:01:04 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa7fb4c5

gpkg: placate black

Fixes: 957902f84edece635210689f46e20741e76d9dba
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/gpkg.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 5cd1f2394e..2b957d58c4 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -656,7 +656,9 @@ class tar_safe_extract:
                     raise ValueError("Path traversal detected.")
                 if member.isdev():
                     writemsg(
-                        colorize("BAD", f"Danger: device file detected: {member.name}\n")
+                        colorize(
+                            "BAD", f"Danger: device file detected: {member.name}\n"
+                        )
                     )
                     raise ValueError("Device file detected.")
                 if member.islnk() and (member.linkname not in self.file_list):

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     381fad5e3554ec94ec5626e8c17874f32b30b752
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Tue Aug 29 07:26:36 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Apr 26 22:05:48 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=381fad5e

lib: use more pure git-describe output for --version

Use `git describe --dirty` output rather than mangling git-describe and reinventing
--dirty by manually checking for changes post-commit.

We no longer mangle the 7th commit post-tag into _p7, but instead do: ${tag}-7-${last_commit}.

This is similar to gnulib's git-version-gen (which we may still want to import,
not sure, this seems enough for now) and is familiar output for developers.

Example:
* Old: 3.0.51_p7
* New: 3.0.51-7-g098b30548

Bug: https://bugs.gentoo.org/912209
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/__init__.py | 35 ++++-------------------------------
 1 file changed, 4 insertions(+), 31 deletions(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index aa81bdb4c2..a468eeaff3 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -720,10 +720,7 @@ if installation.TYPE == installation.TYPES.SOURCE:
                     BASH_BINARY,
                     "-c",
                     (
-                        f"cd {_shell_quote(PORTAGE_BASE_PATH)} ; git describe --match 'portage-*' || exit $? ; "
-                        'if [ -n "`git diff-index --name-only --diff-filter=M HEAD`" ] ; '
-                        "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; "
-                        "exit 0"
+                        f"cd {_shell_quote(PORTAGE_BASE_PATH)} ; git describe --dirty --match 'portage-*' || exit $? ; "
                     ),
                 ]
                 cmd = [
@@ -735,33 +732,9 @@ if installation.TYPE == installation.TYPES.SOURCE:
                 output = _unicode_decode(proc.communicate()[0], encoding=encoding)
                 status = proc.wait()
                 if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-                    output_lines = output.splitlines()
-                    if output_lines:
-                        version_split = output_lines[0].split("-")
-                        if len(version_split) > 1:
-                            VERSION = version_split[1]
-                            patchlevel = False
-                            if len(version_split) > 2:
-                                patchlevel = True
-                                VERSION = f"{VERSION}_p{version_split[2]}"
-                            if len(output_lines) > 1 and output_lines[1] == "modified":
-                                head_timestamp = None
-                                if len(output_lines) > 3:
-                                    try:
-                                        head_timestamp = int(output_lines[3])
-                                    except ValueError:
-                                        pass
-                                timestamp = int(time.time())
-                                if (
-                                    head_timestamp is not None
-                                    and timestamp > head_timestamp
-                                ):
-                                    timestamp = timestamp - head_timestamp
-                                if not patchlevel:
-                                    VERSION = f"{VERSION}_p0"
-                                VERSION = f"{VERSION}_p{timestamp}"
-                            return VERSION
-            VERSION = "HEAD"
+                    VERSION = output.lstrip('portage-').strip()
+            else:
+                VERSION = "HEAD"
             return VERSION
 
     VERSION = _LazyVersion()

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Sam James]

commit:     36235596e061bf8cf4729b3915f9aaa6ec80baa3
Author:     Alfred Wingate <parona <AT> protonmail <DOT> com>
AuthorDate: Sat Apr  6 08:28:43 2024 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Apr 26 22:05:48 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=36235596

lib: adhere to python package version conventions

* Commit metadata isn't valid version that python tooling is expected to
  parse. Follow python ecosystem conventions and make it a local
  version.

https://packaging.python.org/en/latest/specifications/version-specifiers/#local-version-segments

Example:
* Old: 3.0.63-g08a2bc380
* New: 3.0.63+g08a2bc380

Bug: https://bugs.gentoo.org/926966
Signed-off-by: Alfred Wingate <parona <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1314
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index a468eeaff3..21bf993170 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -732,7 +732,7 @@ if installation.TYPE == installation.TYPES.SOURCE:
                 output = _unicode_decode(proc.communicate()[0], encoding=encoding)
                 status = proc.wait()
                 if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-                    VERSION = output.lstrip('portage-').strip()
+                    VERSION = output.lstrip("portage-").strip().replace("-g", "+g")
             else:
                 VERSION = "HEAD"
             return VERSION

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     eb855b8cd1248f49649003dcfb9bf009b70e88cb
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  2 17:56:44 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  2 18:05:35 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=eb855b8c

tar_stream_writer: Add missing error attribute

This attribute was previously initialized only
in an exception handler.

Fixes: b8c3f38ec5ee ("Add more error handling for binpkgs")
Bug: https://bugs.gentoo.org/933385
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/gpkg.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index fc4d7b1fb6..9606f6d3c8 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -100,6 +100,7 @@ class tar_stream_writer:
         self.closed = False
         self.container = container
         self.killed = False
+        self.error = False
         self.tar_format = tar_format
         self.tarinfo = tarinfo
         self.uid = uid

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     f620a0769a509966295954c2b0c76e46e8fb4289
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  2 21:53:04 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  9 17:53:31 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f620a076

tar_safe_extract: Use tarfile.fully_trusted_filter

This suppresses a DeprecationWarning triggered because the
tarfile.data_filter will become the new default in python3.14.
The fully_trusted filter should be suitable here because
tar_safe_extract already performs security validation on
tar members prior to extraction.

Bug: https://bugs.gentoo.org/933433
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/gpkg.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 9606f6d3c8..fdb54c69b8 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -628,6 +628,15 @@ class tar_safe_extract:
         if self.closed:
             raise OSError("Tar file is closed.")
         temp_dir = tempfile.TemporaryDirectory(dir=dest_dir)
+        # The below tar member security checks can be refactored as a filter function
+        # that raises an exception. Use tarfile.fully_trusted_filter for now, which
+        # is simply an identity function:
+        # def fully_trusted_filter(member, dest_path):
+        #     return member
+        try:
+            self.tar.extraction_filter = tarfile.fully_trusted_filter
+        except AttributeError:
+            pass
         try:
             while True:
                 member = self.tar.next()

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     cfd767cd35f5affd3b61b665b0f8814fe2de24c4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 14 05:30:42 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Aug 14 15:22:05 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfd767cd

run_exitfuncs: Drop hooks inherited via fork

Drop hooks inherited via fork because they can trigger redundant
actions as shown in bug 937891. Note that atexit hooks only work
after fork since issue 83856 was fixed in Python 3.13.

Bug: https://bugs.gentoo.org/937891
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/process.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/portage/process.py b/lib/portage/process.py
index 38adebda66..e6f6feb357 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -208,7 +208,7 @@ def atexit_register(func, *args, **kargs):
         # which is associated with the current thread.
         global_event_loop()._coroutine_exithandlers.append((func, args, kargs))
     else:
-        _exithandlers.append((func, args, kargs))
+        _exithandlers.append((func, args, kargs, portage.getpid()))
 
 
 def run_exitfuncs():
@@ -222,7 +222,12 @@ def run_exitfuncs():
     # original function is in the output to stderr.
     exc_info = None
     while _exithandlers:
-        func, targs, kargs = _exithandlers.pop()
+        func, targs, kargs, pid = _exithandlers.pop()
+        if pid != portage.getpid():
+            # Drop hooks inherited via fork because they can trigger redundant
+            # actions as shown in bug 937891. Note that atexit hooks only work
+            # after fork since issue 83856 was fixed in Python 3.13.
+            continue
         try:
             func(*targs, **kargs)
         except SystemExit:

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Ulrich Müller]

commit:     492506adede9d96c661699b90295b70e50f30160
Author:     Ulrich Müller <ulm <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 20 06:02:31 2024 +0000
Commit:     Ulrich Müller <ulm <AT> gentoo <DOT> org>
CommitDate: Mon Sep  9 18:04:44 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=492506ad

eapi.py: Use attrs instead of hardcoding EAPIs in functions

Adding new attrs as needed. Their name is the same as the
corresponding PMS feature label (if one exists).

Signed-off-by: Ulrich Müller <ulm <AT> gentoo.org>

 lib/portage/eapi.py | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py
index ee691d311d..86b27bdbc5 100644
--- a/lib/portage/eapi.py
+++ b/lib/portage/eapi.py
@@ -41,7 +41,7 @@ def eapi_has_strong_blocks(eapi: str) -> bool:
 
 
 def eapi_has_src_prepare_and_src_configure(eapi: str) -> bool:
-    return eapi not in ("0", "1")
+    return _get_eapi_attrs(eapi).src_prepare_src_configure
 
 
 def eapi_supports_prefix(eapi: str) -> bool:
@@ -77,15 +77,15 @@ def eapi_exports_ECLASSDIR(eapi: str) -> bool:
 
 
 def eapi_has_pkg_pretend(eapi: str) -> bool:
-    return eapi not in ("0", "1", "2", "3")
+    return _get_eapi_attrs(eapi).pkg_pretend
 
 
 def eapi_has_implicit_rdepend(eapi: str) -> bool:
-    return eapi in ("0", "1", "2", "3")
+    return _get_eapi_attrs(eapi).rdepend_depend
 
 
 def eapi_has_dosed_dohard(eapi: str) -> bool:
-    return eapi in ("0", "1", "2", "3")
+    return _get_eapi_attrs(eapi).dosed_dohard
 
 
 def eapi_has_required_use(eapi: str) -> bool:
@@ -109,11 +109,11 @@ def eapi_has_repo_deps(eapi: str) -> bool:
 
 
 def eapi_supports_stable_use_forcing_and_masking(eapi: str) -> bool:
-    return eapi not in ("0", "1", "2", "3", "4", "4-slot-abi")
+    return _get_eapi_attrs(eapi).stablemask
 
 
 def eapi_allows_directories_on_profile_level_and_repository_level(eapi: str) -> bool:
-    return eapi not in ("0", "1", "2", "3", "4", "4-slot-abi", "5", "6")
+    return _get_eapi_attrs(eapi).profile_file_dirs
 
 
 def eapi_allows_package_provided(eapi: str) -> bool:
@@ -150,6 +150,7 @@ _eapi_attrs = collections.namedtuple(
         "allows_package_provided",
         "bdepend",
         "broot",
+        "dosed_dohard",
         "empty_groups_always_true",
         "exports_AA",
         "exports_EBUILD_PHASE_FUNC",
@@ -164,14 +165,19 @@ _eapi_attrs = collections.namedtuple(
         "iuse_effective",
         "posixish_locale",
         "path_variables_end_with_trailing_slash",
+        "pkg_pretend",
         "prefix",
+        "profile_file_dirs",
+        "rdepend_depend",
         "repo_deps",
         "required_use",
         "required_use_at_most_one_of",
         "selective_src_uri_restriction",
         "slot_operator",
         "slot_deps",
+        "src_prepare_src_configure",
         "src_uri_arrows",
+        "stablemask",
         "strong_blocks",
         "sysroot",
         "use_deps",
@@ -223,6 +229,7 @@ def _get_eapi_attrs(eapi_str: Optional[str]) -> _eapi_attrs:
             allows_package_provided=True,
             bdepend=False,
             broot=True,
+            dosed_dohard=False,
             empty_groups_always_true=False,
             exports_AA=False,
             exports_EBUILD_PHASE_FUNC=True,
@@ -236,15 +243,20 @@ def _get_eapi_attrs(eapi_str: Optional[str]) -> _eapi_attrs:
             iuse_defaults=True,
             iuse_effective=False,
             path_variables_end_with_trailing_slash=False,
+            pkg_pretend=True,
             posixish_locale=False,
             prefix=True,
+            profile_file_dirs=False,
+            rdepend_depend=False,
             repo_deps=True,
             required_use=True,
             required_use_at_most_one_of=True,
             selective_src_uri_restriction=True,
             slot_deps=True,
             slot_operator=True,
+            src_prepare_src_configure=True,
             src_uri_arrows=True,
+            stablemask=True,
             strong_blocks=True,
             sysroot=True,
             use_deps=True,
@@ -256,6 +268,7 @@ def _get_eapi_attrs(eapi_str: Optional[str]) -> _eapi_attrs:
             allows_package_provided=eapi <= Eapi("6"),
             bdepend=eapi >= Eapi("7"),
             broot=eapi >= Eapi("7"),
+            dosed_dohard=eapi <= Eapi("3"),
             empty_groups_always_true=eapi <= Eapi("6"),
             exports_AA=eapi <= Eapi("3"),
             exports_EBUILD_PHASE_FUNC=eapi >= Eapi("5"),
@@ -269,15 +282,20 @@ def _get_eapi_attrs(eapi_str: Optional[str]) -> _eapi_attrs:
             iuse_defaults=eapi >= Eapi("1"),
             iuse_effective=eapi >= Eapi("5"),
             path_variables_end_with_trailing_slash=eapi <= Eapi("6"),
+            pkg_pretend=eapi >= Eapi("4"),
             posixish_locale=eapi >= Eapi("6"),
             prefix=eapi >= Eapi("3"),
+            profile_file_dirs=eapi >= Eapi("7"),
+            rdepend_depend=eapi <= Eapi("3"),
             repo_deps=False,
             required_use=eapi >= Eapi("4"),
             required_use_at_most_one_of=eapi >= Eapi("5"),
             selective_src_uri_restriction=eapi >= Eapi("8"),
             slot_deps=eapi >= Eapi("1"),
             slot_operator=eapi >= Eapi("5"),
+            src_prepare_src_configure=eapi >= Eapi("2"),
             src_uri_arrows=eapi >= Eapi("2"),
+            stablemask=eapi >= Eapi("5"),
             strong_blocks=eapi >= Eapi("2"),
             sysroot=eapi >= Eapi("7"),
             use_deps=eapi >= Eapi("2"),

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Ulrich Müller]

commit:     e5569b2dbd97ee804ee37ec17c74ae64ec8826b6
Author:     Ulrich Müller <ulm <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 20 06:01:36 2024 +0000
Commit:     Ulrich Müller <ulm <AT> gentoo <DOT> org>
CommitDate: Mon Sep  9 18:04:38 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e5569b2d

eapi.py: Sort _eapi_attrs alphabetically

Signed-off-by: Ulrich Müller <ulm <AT> gentoo.org>

 lib/portage/eapi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/eapi.py b/lib/portage/eapi.py
index 2c17018709..ee691d311d 100644
--- a/lib/portage/eapi.py
+++ b/lib/portage/eapi.py
@@ -150,6 +150,7 @@ _eapi_attrs = collections.namedtuple(
         "allows_package_provided",
         "bdepend",
         "broot",
+        "empty_groups_always_true",
         "exports_AA",
         "exports_EBUILD_PHASE_FUNC",
         "exports_ECLASSDIR",
@@ -172,10 +173,9 @@ _eapi_attrs = collections.namedtuple(
         "slot_deps",
         "src_uri_arrows",
         "strong_blocks",
+        "sysroot",
         "use_deps",
         "use_dep_defaults",
-        "empty_groups_always_true",
-        "sysroot",
     ),
 )
 

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     320df528974c09338c7fef12e41ce66c5593c101
Author:     Etienne Buira <etienne.buira <AT> free <DOT> fr>
AuthorDate: Sat Nov  2 14:33:27 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Nov  2 15:45:49 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=320df528

gpkg: avoid os.path.islink where lstat is available

Signed-off-by: Etienne Buira <etienne.buira <AT> free.fr>
Bug: https://bugs.gentoo.org/942512
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/gpkg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 5f392e95e8..c4a2666bc5 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1943,7 +1943,7 @@ class gpkg:
 
                 file_stat = os.lstat(f)
 
-                if os.path.islink(f):
+                if stat.S_ISLNK(file_stat):
                     path_link = os.readlink(f)
                     path_link_length = len(
                         os.fsencode(path_link)
@@ -2035,7 +2035,7 @@ class gpkg:
 
             file_stat = os.lstat(path)
 
-            if os.path.islink(path):
+            if stat.S_ISLNK(file_stat):
                 path_link = os.readlink(path)
                 path_link_length = len(
                     _unicode_encode(

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     33e5f70ec3ec7334d646daf1cf18a22fec8faf39
Author:     Etienne Buira <etienne.buira <AT> free <DOT> fr>
AuthorDate: Sat Nov  2 15:18:34 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Nov  2 15:47:01 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=33e5f70e

gpkg: use already queried size instead of syscall

Signed-off-by: Etienne Buira <etienne.buira <AT> free.fr>
Bug: https://bugs.gentoo.org/942512
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/gpkg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index c4a2666bc5..e9347919cc 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1963,7 +1963,7 @@ class gpkg:
                 if stat.S_ISLNK(file_stat.st_mode):
                     continue
 
-                file_size = os.path.getsize(f)
+                file_size = file_stat.st_size
                 image_total_size += file_size
                 image_max_file_size = max(image_max_file_size, file_size)
 
@@ -2054,7 +2054,7 @@ class gpkg:
                 if stat.S_ISLNK(file_stat.st_mode):
                     continue
 
-                file_size = os.path.getsize(path)
+                file_size = file_stat.st_size
                 image_total_size += file_size
                 if file_size > image_max_file_size:
                     image_max_file_size = file_size

[gentoo-commits] proj/portage:master commit in: lib/portage/

[Zac Medico]

commit:     ac88048bb1d6a28dbd415223e2e0fc3ec8b16a6c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Nov  2 22:10:44 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Nov  2 22:11:47 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ac88048b

gpkg: Fix S_ISLNK usage

Fixes: 320df528974c ("gpkg: avoid os.path.islink where lstat is available")
Bug: https://bugs.gentoo.org/942512
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/gpkg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index e9347919cc..53785c2711 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -1943,7 +1943,7 @@ class gpkg:
 
                 file_stat = os.lstat(f)
 
-                if stat.S_ISLNK(file_stat):
+                if stat.S_ISLNK(file_stat.st_mode):
                     path_link = os.readlink(f)
                     path_link_length = len(
                         os.fsencode(path_link)
@@ -2035,7 +2035,7 @@ class gpkg:
 
             file_stat = os.lstat(path)
 
-            if stat.S_ISLNK(file_stat):
+            if stat.S_ISLNK(file_stat.st_mode):
                 path_link = os.readlink(path)
                 path_link_length = len(
                     _unicode_encode(