[gentoo-portage-dev] [PATCH 0/4] Modernize metadata passing & add INHERIT to md5-cache

[gentoo-portage-dev] [PATCH 0/4] Modernize metadata passing & add INHERIT to md5-cache

[Michał Górny]

Hi,

Here's a patchset that:

1. Removes some leftover cruft from the old cache format.

2. Switches internal ebuild.sh/Python metadata logic from using a format
   resembling the old cache with hardcoded line number to data mapping
   (sic!) in favor of plain key=value format.

3. Adds INHERIT key to md5-cache for better compatibility with PkgCore.
   This key lists eclasses directly inherited by ebuild (vs _eclasses_
   that lists indirect inherits as well).

Michał Górny (4):
  Remove deprecated dbkey support from doebuild/ebuild.sh
  Switch internal metadata to key=value format
  Remove UNUSED* auxdbkeys
  Include INHERIT value in generated cache

 bin/ebuild.sh                                 | 36 ++++++++-----------
 bin/phase-functions.sh                        |  2 +-
 bin/portageq                                  |  3 +-
 lib/_emerge/EbuildMetadataPhase.py            | 15 ++++----
 lib/_emerge/Package.py                        |  3 +-
 lib/portage/__init__.py                       |  4 +--
 lib/portage/dbapi/__init__.py                 |  3 +-
 .../ebuild/_config/special_env_vars.py        |  2 +-
 lib/portage/package/ebuild/doebuild.py        | 22 ++----------
 repoman/lib/repoman/qa_data.py                |  2 +-
 10 files changed, 33 insertions(+), 59 deletions(-)

-- 
2.33.0

[gentoo-portage-dev] [PATCH 1/4] Remove deprecated dbkey support from doebuild/ebuild.sh

[Michał Górny]

Remove the support for the dbkey logic that used to write metadata into
a file.  This logic has stopped being used and became deprecated
in 2013.  If any external tool is still using it, it's probably been
broken by changes in metadata itself since, and would definitely
be broken by the incoming change in metadata format.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 bin/ebuild.sh                          | 23 ++++-------------------
 lib/portage/package/ebuild/doebuild.py | 22 +++-------------------
 2 files changed, 7 insertions(+), 38 deletions(-)

diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 3042e6c8c..32995d95b 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -754,14 +754,6 @@ if [[ $EBUILD_PHASE = depend ]] ; then
 	export SANDBOX_ON="0"
 	set -f
 
-	if [ -n "${dbkey}" ] ; then
-		if [ ! -d "${dbkey%/*}" ]; then
-			install -d -g ${PORTAGE_GID} -m2775 "${dbkey%/*}"
-		fi
-		# Make it group writable. 666&~002==664
-		umask 002
-	fi
-
 	auxdbkeys="DEPEND RDEPEND SLOT SRC_URI RESTRICT HOMEPAGE LICENSE
 		DESCRIPTION KEYWORDS INHERITED IUSE REQUIRED_USE PDEPEND BDEPEND
 		EAPI PROPERTIES DEFINED_PHASES IDEPEND UNUSED_04
@@ -775,17 +767,10 @@ if [[ $EBUILD_PHASE = depend ]] ; then
 	fi
 
 	# The extra $(echo) commands remove newlines.
-	if [ -n "${dbkey}" ] ; then
-		> "${dbkey}"
-		for f in ${auxdbkeys} ; do
-			echo $(echo ${!f}) >> "${dbkey}" || exit $?
-		done
-	else
-		for f in ${auxdbkeys} ; do
-			eval "echo \$(echo \${!f}) 1>&${PORTAGE_PIPE_FD}" || exit $?
-		done
-		eval "exec ${PORTAGE_PIPE_FD}>&-"
-	fi
+	for f in ${auxdbkeys} ; do
+		eval "echo \$(echo \${!f}) 1>&${PORTAGE_PIPE_FD}" || exit $?
+	done
+	eval "exec ${PORTAGE_PIPE_FD}>&-"
 	set +f
 else
 	# Note: readonly variables interfere with __preprocess_ebuild_env(), so
diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
index 366cbb9ca..5115ff6a3 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -568,7 +568,7 @@ _doebuild_commands_without_builddir = (
 )
 
 def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, listonly=0,
-	fetchonly=0, cleanup=0, dbkey=DeprecationWarning, use_cache=1, fetchall=0, tree=None,
+	fetchonly=0, cleanup=0, use_cache=1, fetchall=0, tree=None,
 	mydbapi=None, vartree=None, prev_mtimes=None,
 	fd_pipes=None, returnpid=False):
 	"""
@@ -591,9 +591,6 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0,
 	@type fetchonly: Boolean
 	@param cleanup: Passed to prepare_build_dirs (TODO: what does it do?)
 	@type cleanup: Boolean
-	@param dbkey: A file path where metadata generated by the 'depend' phase
-		will be written.
-	@type dbkey: String
 	@param use_cache: Enables the cache
 	@type use_cache: Boolean
 	@param fetchall: Used to wrap fetch(), fetches all URIs (even ones invalid due to USE conditionals)
@@ -637,11 +634,6 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0,
 			"settings['EROOT'] is used.",
 			DeprecationWarning, stacklevel=2)
 
-	if dbkey is not DeprecationWarning:
-		warnings.warn("portage.doebuild() called "
-			"with deprecated dbkey argument.",
-			DeprecationWarning, stacklevel=2)
-
 	if not tree:
 		writemsg("Warning: tree not specified to doebuild\n")
 		tree = "porttree"
@@ -836,16 +828,8 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0,
 
 		# get possible slot information from the deps file
 		if mydo == "depend":
-			writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2)
-			if returnpid:
-				return _spawn_phase(mydo, mysettings,
-					fd_pipes=fd_pipes, returnpid=returnpid)
-			if dbkey and dbkey is not DeprecationWarning:
-				mysettings["dbkey"] = dbkey
-			else:
-				mysettings["dbkey"] = \
-					os.path.join(mysettings.depcachedir, "aux_db_key_temp")
-
+			if not returnpid:
+				raise TypeError("returnpid must be True for depend phase")
 			return _spawn_phase(mydo, mysettings,
 				fd_pipes=fd_pipes, returnpid=returnpid)
 
-- 
2.33.0

[gentoo-portage-dev] [PATCH 2/4] Switch internal metadata to key=value format

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 bin/ebuild.sh                      | 13 +++++++------
 lib/_emerge/EbuildMetadataPhase.py | 15 +++++++++------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 32995d95b..381bcb5c8 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -754,10 +754,11 @@ if [[ $EBUILD_PHASE = depend ]] ; then
 	export SANDBOX_ON="0"
 	set -f
 
-	auxdbkeys="DEPEND RDEPEND SLOT SRC_URI RESTRICT HOMEPAGE LICENSE
+	metadata_keys=(
+		DEPEND RDEPEND SLOT SRC_URI RESTRICT HOMEPAGE LICENSE
 		DESCRIPTION KEYWORDS INHERITED IUSE REQUIRED_USE PDEPEND BDEPEND
-		EAPI PROPERTIES DEFINED_PHASES IDEPEND UNUSED_04
-		UNUSED_03 UNUSED_02 UNUSED_01"
+		EAPI PROPERTIES DEFINED_PHASES IDEPEND
+	)
 
 	if ! ___eapi_has_BDEPEND; then
 		unset BDEPEND
@@ -767,10 +768,10 @@ if [[ $EBUILD_PHASE = depend ]] ; then
 	fi
 
 	# The extra $(echo) commands remove newlines.
-	for f in ${auxdbkeys} ; do
-		eval "echo \$(echo \${!f}) 1>&${PORTAGE_PIPE_FD}" || exit $?
+	for f in "${metadata_keys[@]}" ; do
+		echo "${f}=$(echo ${!f})" >&${PORTAGE_PIPE_FD} || exit $?
 	done
-	eval "exec ${PORTAGE_PIPE_FD}>&-"
+	exec {PORTAGE_PIPE_FD}>&-
 	set +f
 else
 	# Note: readonly variables interfere with __preprocess_ebuild_env(), so
diff --git a/lib/_emerge/EbuildMetadataPhase.py b/lib/_emerge/EbuildMetadataPhase.py
index d00f194c2..5fd0e8a4d 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -151,13 +151,16 @@ class EbuildMetadataPhase(SubProcess):
 			metadata_lines = _unicode_decode(b''.join(self._raw_metadata),
 				encoding=_encodings['repo.content'],
 				errors='replace').splitlines()
+			metadata = {}
 			metadata_valid = True
-			if len(portage.auxdbkeys) != len(metadata_lines):
-				# Don't trust bash's returncode if the
-				# number of lines is incorrect.
-				metadata_valid = False
-			else:
-				metadata = dict(zip(portage.auxdbkeys, metadata_lines))
+			for l in metadata_lines:
+				if '=' not in l:
+					metadata_valid = False
+					break
+				key, value = l.split('=', 1)
+				metadata[key] = value
+
+			if metadata_valid:
 				parsed_eapi = self._eapi
 				if parsed_eapi is None:
 					parsed_eapi = "0"
-- 
2.33.0

[gentoo-portage-dev] [PATCH 3/4] Remove UNUSED* auxdbkeys

[Michał Górny]

The UNUSED* auxdbkeys are a relict of old metadata cache format that
required a fixed number of lines.  This format is no longer supported
by Portage, and all uses of auxdbkeys strip UNUSED values, so just
remove them entirely.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 bin/portageq                   | 3 +--
 lib/_emerge/Package.py         | 3 +--
 lib/portage/__init__.py        | 4 +---
 lib/portage/dbapi/__init__.py  | 3 +--
 repoman/lib/repoman/qa_data.py | 2 +-
 5 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/bin/portageq b/bin/portageq
index 385259f9d..d3cb9f140 100755
--- a/bin/portageq
+++ b/bin/portageq
@@ -238,8 +238,7 @@ docstrings['metadata'] = """
 <eroot> <pkgtype> <category/package> [<key>]+
 Returns metadata values for the specified package.
 Available keys: %s
-"""  % ','.join(sorted(x for x in portage.auxdbkeys \
-if not x.startswith('UNUSED_')))
+"""  % ','.join(sorted(x for x in portage.auxdbkeys))
 metadata.__doc__ = docstrings['metadata']
 
 
diff --git a/lib/_emerge/Package.py b/lib/_emerge/Package.py
index e8809a89d..4e25619ae 100644
--- a/lib/_emerge/Package.py
+++ b/lib/_emerge/Package.py
@@ -791,8 +791,7 @@ class Package(Task):
 			pkg = self
 		return pkg
 
-_all_metadata_keys = set(x for x in portage.auxdbkeys \
-	if not x.startswith("UNUSED_"))
+_all_metadata_keys = set(x for x in portage.auxdbkeys)
 _all_metadata_keys.update(Package.metadata_keys)
 _all_metadata_keys = frozenset(_all_metadata_keys)
 
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 6e22a174b..232d77f0e 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -519,10 +519,8 @@ auxdbkeys = (
 	'RESTRICT',  'HOMEPAGE',  'LICENSE',   'DESCRIPTION',
 	'KEYWORDS',  'INHERITED', 'IUSE', 'REQUIRED_USE',
 	'PDEPEND',   'BDEPEND', 'EAPI',
-	'PROPERTIES', 'DEFINED_PHASES', 'IDEPEND', 'UNUSED_04',
-	'UNUSED_03', 'UNUSED_02', 'UNUSED_01',
+	'PROPERTIES', 'DEFINED_PHASES', 'IDEPEND',
 )
-auxdbkeylen = len(auxdbkeys)
 
 def portageexit():
 	pass
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index d7facc9b6..3caefb816 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -28,8 +28,7 @@ class dbapi:
 	_category_re = re.compile(r'^\w[-.+\w]*$', re.UNICODE)
 	_categories = None
 	_use_mutable = False
-	_known_keys = frozenset(x for x in auxdbkeys
-		if not x.startswith("UNUSED_0"))
+	_known_keys = frozenset(auxdbkeys)
 	_pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository")
 
 	def __init__(self):
diff --git a/repoman/lib/repoman/qa_data.py b/repoman/lib/repoman/qa_data.py
index afb403d8d..4785581e2 100644
--- a/repoman/lib/repoman/qa_data.py
+++ b/repoman/lib/repoman/qa_data.py
@@ -79,7 +79,7 @@ class QAData:
 
 		self.missingvars = qadata.get("missingvars", [])
 		logging.debug("QAData: missingvars: %s", self.missingvars)
-		self.allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
+		self.allvars = set(portage.auxdbkeys)
 		self.allvars.update(Package.metadata_keys)
 		self.allvars = sorted(self.allvars)
 
-- 
2.33.0

[gentoo-portage-dev] [PATCH 4/4] Include INHERIT value in generated cache

[Michał Górny]

PkgCore uses an additional md5-cache INHERIT key to indicate eclasses
explicitly inherited in an ebuild.  Update Portage to emit the same key
to restore cache compatibility.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 bin/ebuild.sh                                          | 8 +++++++-
 bin/phase-functions.sh                                 | 2 +-
 lib/portage/__init__.py                                | 2 +-
 lib/portage/package/ebuild/_config/special_env_vars.py | 2 +-
 4 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 381bcb5c8..07ca58d22 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -402,6 +402,9 @@ inherit() {
 			unset $__export_funcs_var
 
 			has $1 $INHERITED || export INHERITED="$INHERITED $1"
+			if [[ ${ECLASS_DEPTH} -eq 1 ]]; then
+				export PORTAGE_EXPLICIT_INHERIT="${PORTAGE_EXPLICIT_INHERIT} $1"
+			fi
 		fi
 
 		shift
@@ -648,6 +651,7 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then
 		unset INHERITED IUSE REQUIRED_USE ECLASS E_IUSE E_REQUIRED_USE
 		unset E_DEPEND E_RDEPEND E_PDEPEND E_BDEPEND E_IDEPEND E_PROPERTIES
 		unset E_RESTRICT PROVIDES_EXCLUDE REQUIRES_EXCLUDE
+		unset PORTAGE_EXPLICIT_INHERIT
 
 		if [[ $PORTAGE_DEBUG != 1 || ${-/x/} != $- ]] ; then
 			source "$EBUILD" || die "error sourcing ebuild"
@@ -757,7 +761,7 @@ if [[ $EBUILD_PHASE = depend ]] ; then
 	metadata_keys=(
 		DEPEND RDEPEND SLOT SRC_URI RESTRICT HOMEPAGE LICENSE
 		DESCRIPTION KEYWORDS INHERITED IUSE REQUIRED_USE PDEPEND BDEPEND
-		EAPI PROPERTIES DEFINED_PHASES IDEPEND
+		EAPI PROPERTIES DEFINED_PHASES IDEPEND INHERIT
 	)
 
 	if ! ___eapi_has_BDEPEND; then
@@ -767,6 +771,8 @@ if [[ $EBUILD_PHASE = depend ]] ; then
 		unset IDEPEND
 	fi
 
+	INHERIT=${PORTAGE_EXPLICIT_INHERIT}
+
 	# The extra $(echo) commands remove newlines.
 	for f in "${metadata_keys[@]}" ; do
 		echo "${f}=$(echo ${!f})" >&${PORTAGE_PIPE_FD} || exit $?
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index 0bb5d86e1..d3221993d 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -20,7 +20,7 @@ PORTAGE_READONLY_VARS="D EBUILD EBUILD_PHASE EBUILD_PHASE_FUNC \
 	PORTAGE_BUILD_USER PORTAGE_BUNZIP2_COMMAND \
 	PORTAGE_BZIP2_COMMAND PORTAGE_COLORMAP PORTAGE_CONFIGROOT \
 	PORTAGE_DEBUG PORTAGE_DEPCACHEDIR PORTAGE_EBUILD_EXIT_FILE \
-	PORTAGE_ECLASS_LOCATIONS \
+	PORTAGE_ECLASS_LOCATIONS PORTAGE_EXPLICIT_INHERIT \
 	PORTAGE_GID PORTAGE_GRPNAME PORTAGE_INST_GID PORTAGE_INST_UID \
 	PORTAGE_INTERNAL_CALLER PORTAGE_IPC_DAEMON PORTAGE_IUSE PORTAGE_LOG_FILE \
 	PORTAGE_MUTABLE_FILTERED_VARS PORTAGE_OVERRIDE_EPREFIX PORTAGE_PROPERTIES \
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 232d77f0e..a41ca4323 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -519,7 +519,7 @@ auxdbkeys = (
 	'RESTRICT',  'HOMEPAGE',  'LICENSE',   'DESCRIPTION',
 	'KEYWORDS',  'INHERITED', 'IUSE', 'REQUIRED_USE',
 	'PDEPEND',   'BDEPEND', 'EAPI',
-	'PROPERTIES', 'DEFINED_PHASES', 'IDEPEND',
+	'PROPERTIES', 'DEFINED_PHASES', 'IDEPEND', 'INHERIT',
 )
 
 def portageexit():
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py
index 456af1838..8e314a6d6 100644
--- a/lib/portage/package/ebuild/_config/special_env_vars.py
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py
@@ -22,7 +22,7 @@ env_blacklist = frozenset((
 	"KEYWORDS", "LICENSE", "MERGE_TYPE",
 	"PDEPEND", "PF", "PKGUSE", "PORTAGE_BACKGROUND",
 	"PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDDIR_LOCKED",
-	"PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT",
+	"PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", "PORTAGE_EXPLICIT_INHERIT",
 	"PORTAGE_INTERNAL_CALLER", "PORTAGE_IUSE",
 	"PORTAGE_NONFATAL", "PORTAGE_PIPE_FD", "PORTAGE_REPO_NAME",
 	"PORTAGE_USE", "PROPERTIES", "RDEPEND", "REPOSITORY",
-- 
2.33.0