[gentoo-dev] [PATCH 1/9] install-qa-check.d: Generalize 60python-{pyc → site}

[gentoo-dev] [PATCH 1/9] install-qa-check.d: Generalize 60python-{pyc → site}

[Michał Górny]

Rename `60python-pyc` check to `60python-site`, as it will be used
to perform other checks on the site-packages directory.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 .../{60python-pyc => 60python-site}           | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)
 rename metadata/install-qa-check.d/{60python-pyc => 60python-site} (86%)

diff --git a/metadata/install-qa-check.d/60python-pyc b/metadata/install-qa-check.d/60python-site
similarity index 86%
rename from metadata/install-qa-check.d/60python-pyc
rename to metadata/install-qa-check.d/60python-site
index fe4f3f62c4ef..5f812ecd01e1 100644
--- a/metadata/install-qa-check.d/60python-pyc
+++ b/metadata/install-qa-check.d/60python-site
@@ -1,10 +1,11 @@
-# Copyright 2019-2022 Gentoo Authors
+# Copyright 2019-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-# QA check: ensure that Python modules are compiled after installing
+# QA checks related to site-packages directory:
+# - missing, mismatched or stray .pyc files
 # Maintainer: Python project <python@gentoo.org>
 
-python_pyc_check() {
+python_site_check() {
 	local save=$(shopt -p nullglob)
 	shopt -s nullglob
 	local progs=( "${EPREFIX}"/usr/lib/python-exec/*/gpep517 )
@@ -69,7 +70,7 @@ python_pyc_check() {
 		eqawarn "not byte-compiled."
 		eqawarn "The following files are missing:"
 		eqawarn
-		eqatag -v python-pyc.missing "${missing[@]}"
+		eqatag -v python-site.pyc.missing "${missing[@]}"
 		found=1
 	fi
 
@@ -79,7 +80,7 @@ python_pyc_check() {
 		eqawarn "that seem to be invalid (do not have the correct header)."
 		eqawarn "The following files are invalid:"
 		eqawarn
-		eqatag -v python-pyc.invalid "${invalid[@]}"
+		eqatag -v python-site.pyc.invalid "${invalid[@]}"
 		found=1
 	fi
 
@@ -88,7 +89,7 @@ python_pyc_check() {
 		eqawarn "QA Notice: This package installs one or more compiled Python modules whose"
 		eqawarn ".py files have different content (size or hash) than recorded:"
 		eqawarn
-		eqatag -v python-pyc.mismatched.data "${mismatched_data[@]}"
+		eqatag -v python-site.pyc.mismatched.data "${mismatched_data[@]}"
 		found=1
 	fi
 
@@ -97,7 +98,7 @@ python_pyc_check() {
 		eqawarn "QA Notice: This package installs one or more compiled Python modules whose"
 		eqawarn ".py files have different timestamps than recorded:"
 		eqawarn
-		eqatag -v python-pyc.mismatched.timestamp "${mismatched_timestamp[@]}"
+		eqatag -v python-site.pyc.mismatched.timestamp "${mismatched_timestamp[@]}"
 		found=1
 	fi
 
@@ -107,7 +108,7 @@ python_pyc_check() {
 		eqawarn "that do not match installed modules (or their implementation)."
 		eqawarn "The following files are stray:"
 		eqawarn
-		eqatag -v python-pyc.stray "${stray[@]}"
+		eqatag -v python-site.pyc.stray "${stray[@]}"
 		found=1
 	fi
 
@@ -118,7 +119,7 @@ python_pyc_check() {
 	fi
 }
 
-python_pyc_check
+python_site_check
 
 : # guarantee successful exit
 
-- 
2.43.0

[gentoo-dev] [PATCH 2/9] install-qa-check.d/60python-site: Add invalid site-packages check

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 metadata/install-qa-check.d/60python-site | 45 ++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/metadata/install-qa-check.d/60python-site b/metadata/install-qa-check.d/60python-site
index 5f812ecd01e1..459509f8a136 100644
--- a/metadata/install-qa-check.d/60python-site
+++ b/metadata/install-qa-check.d/60python-site
@@ -11,12 +11,21 @@ python_site_check() {
 	local progs=( "${EPREFIX}"/usr/lib/python-exec/*/gpep517 )
 	${save}
 
+	local forbidden_package_names=(
+		# NB: setuptools/discovery.py is a good source of ideas
+		benchmark benchmarks dist doc docs examples scripts tasks
+		test tests tools util utils
+		.pytest_cache .hypothesis _trial_temp
+	)
+
 	local invalid=()
 	local mismatched_timestamp=()
 	local mismatched_data=()
 	local missing=()
 	local stray=()
 
+	local stray_packages=()
+
 	# Avoid running the check if sufficiently new gpep517 is not installed
 	# yet. It's valid to schedule (for merge order) >=gpep517-8 after
 	# packages which have this check run if they don't use distutils-r1.
@@ -24,12 +33,33 @@ python_site_check() {
 		return
 	fi
 
+	local f prog
 	for prog in "${progs[@]}"; do
 		local impl=${prog%/*}
 		impl=${impl##*/}
 
 		# NB: using ${impl}* to catch pypy3.* for pypy3
-		[[ -d "${ED}"/usr/lib/${impl}*/site-packages ]] || continue
+		local sitedir=( "${ED}"/usr/lib/${impl}*/site-packages )
+		[[ -d ${sitedir} ]] || continue
+
+		# check for stray files in site-packages
+		while IFS= read -d $'\0' -r f; do
+			stray_packages+=( "${f#${ED}}" )
+		done < <(
+			find "${sitedir}" -maxdepth 1 -type f '!' '(' \
+					-name '*.egg-info' -o \
+					-name '*.pth' -o \
+					-name '*.py' -o \
+					-name '*.pyi' -o \
+					-name "*$(get_modname)" \
+				')' -print0
+		)
+		# check for forbidden packages
+		for f in "${forbidden_package_names[@]}"; do
+			[[ -d ${sitedir}/${f} ]] && stray_packages+=(
+				"${sitedir#${ED}}/${f}"
+			)
+		done
 
 		einfo "Verifying compiled files for ${impl}"
 		local kind pyc py
@@ -117,6 +147,19 @@ python_site_check() {
 		eqawarn "For more information on bytecode files and related issues, please see:"
 		eqawarn "  https://projects.gentoo.org/python/guide/qawarn.html#compiled-bytecode-related-warnings"
 	fi
+
+	if [[ ${stray_packages[@]} ]]; then
+		eqawarn
+		eqawarn "QA Notice: The following unexpected files/directories were found"
+		eqawarn "top-level in the site-packages directory:"
+		eqawarn
+		eqatag -v python-site.stray "${stray_packages[@]}"
+		eqawarn
+		eqawarn "This is most likely a bug in the build system.  More information"
+		eqawarn "can be found in the Python Guide:"
+		eqawarn "https://projects.gentoo.org/python/guide/qawarn.html#stray-top-level-files-in-site-packages"
+		die "Failing install because of stray top-level files in site-packages"
+	fi
 }
 
 python_site_check
-- 
2.43.0

[gentoo-dev] [PATCH 3/9] distutils-r1.eclass: Stray file check moved to install-qa-check.d

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/distutils-r1.eclass | 33 ---------------------------------
 1 file changed, 33 deletions(-)

diff --git a/eclass/distutils-r1.eclass b/eclass/distutils-r1.eclass
index c0d1992ccce0..8dfee7cb9232 100644
--- a/eclass/distutils-r1.eclass
+++ b/eclass/distutils-r1.eclass
@@ -2091,39 +2091,6 @@ _distutils-r1_post_python_install() {
 	if [[ -d ${sitedir} ]]; then
 		_distutils-r1_strip_namespace_packages "${sitedir}"
 
-		local forbidden_package_names=(
-			examples test tests
-			.pytest_cache .hypothesis _trial_temp
-		)
-		local strays=()
-		local p
-		mapfile -d $'\0' -t strays < <(
-			find "${sitedir}" -maxdepth 1 -type f '!' '(' \
-					-name '*.egg-info' -o \
-					-name '*.pth' -o \
-					-name '*.py' -o \
-					-name '*.pyi' -o \
-					-name "*$(get_modname)" \
-				')' -print0
-		)
-		for p in "${forbidden_package_names[@]}"; do
-			[[ -d ${sitedir}/${p} ]] && strays+=( "${sitedir}/${p}" )
-		done
-
-		if [[ -n ${strays[@]} ]]; then
-			eerror "The following unexpected files/directories were found top-level"
-			eerror "in the site-packages directory:"
-			eerror
-			for p in "${strays[@]}"; do
-				eerror "  ${p#${ED}}"
-			done
-			eerror
-			eerror "This is most likely a bug in the build system.  More information"
-			eerror "can be found in the Python Guide:"
-			eerror "https://projects.gentoo.org/python/guide/qawarn.html#stray-top-level-files-in-site-packages"
-			die "Failing install because of stray top-level files in site-packages"
-		fi
-
 		if [[ ! ${DISTUTILS_EXT} && ! ${_DISTUTILS_EXT_WARNED} ]]; then
 			if [[ $(find "${sitedir}" -name "*$(get_modname)" | head -n 1) ]]
 			then
-- 
2.43.0

[gentoo-dev] [PATCH 4/9] install-qa-check.d/60python-site: Add bad version check

[Michał Górny]

This was a case e.g. with =dev-python/pyrqlite-2.2.0.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 metadata/install-qa-check.d/60python-site | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/metadata/install-qa-check.d/60python-site b/metadata/install-qa-check.d/60python-site
index 459509f8a136..a54dda53a454 100644
--- a/metadata/install-qa-check.d/60python-site
+++ b/metadata/install-qa-check.d/60python-site
@@ -24,6 +24,7 @@ python_site_check() {
 	local missing=()
 	local stray=()
 
+	local bad_versions=()
 	local stray_packages=()
 
 	# Avoid running the check if sufficiently new gpep517 is not installed
@@ -42,6 +43,18 @@ python_site_check() {
 		local sitedir=( "${ED}"/usr/lib/${impl}*/site-packages )
 		[[ -d ${sitedir} ]] || continue
 
+		# check for bad package versions
+		while IFS= read -d $'\0' -r f; do
+			bad_versions+=( "${f#${ED}}" )
+		done < <(
+			find "${sitedir}" -maxdepth 1 '(' \
+				-name '*-0.0.0.dist-info' -o \
+				-name '*-UNKNOWN.dist-info' -o \
+				-name '*-0.0.0.egg-info' -o \
+				-name '*-UNKNOWN.egg-info' \
+				')' -print0
+		)
+
 		# check for stray files in site-packages
 		while IFS= read -d $'\0' -r f; do
 			stray_packages+=( "${f#${ED}}" )
@@ -148,6 +161,14 @@ python_site_check() {
 		eqawarn "  https://projects.gentoo.org/python/guide/qawarn.html#compiled-bytecode-related-warnings"
 	fi
 
+	if [[ ${bad_versions[@]} ]]; then
+		eqawarn
+		eqawarn "QA Notice: The following Python packages were installed with"
+		eqawarn "invalid/suspicious versions in the site-packages directory:"
+		eqawarn
+		eqatag -v python-site.bad_version "${bad_versions[@]}"
+	fi
+
 	if [[ ${stray_packages[@]} ]]; then
 		eqawarn
 		eqawarn "QA Notice: The following unexpected files/directories were found"
-- 
2.43.0

[gentoo-dev] [PATCH 5/9] install-qa-check.d/60python-site: Forbid lib & usr package names

[Michał Górny]

Forbid `lib`, `$(get_libdir)` and `usr` as package names, to catch
accidentally duplicating sitedir as prefix.

Bug: https://bugs.gentoo.org/618134
Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 metadata/install-qa-check.d/60python-site | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/metadata/install-qa-check.d/60python-site b/metadata/install-qa-check.d/60python-site
index a54dda53a454..3fd697605a70 100644
--- a/metadata/install-qa-check.d/60python-site
+++ b/metadata/install-qa-check.d/60python-site
@@ -15,6 +15,8 @@ python_site_check() {
 		# NB: setuptools/discovery.py is a good source of ideas
 		benchmark benchmarks dist doc docs examples scripts tasks
 		test tests tools util utils
+		# catch double-prefix installs, e.g. https://bugs.gentoo.org/618134
+		lib $(get_libdir) usr
 		.pytest_cache .hypothesis _trial_temp
 	)
 
-- 
2.43.0

[gentoo-dev] [PATCH 6/9] install-qa-check.d/60python-site: Add check for wrong libdir

[Michał Górny]

Bug: https://bugs.gentoo.org/702016
Bug: https://bugs.gentoo.org/770961
Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 metadata/install-qa-check.d/60python-site | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/metadata/install-qa-check.d/60python-site b/metadata/install-qa-check.d/60python-site
index 3fd697605a70..facc41fbb060 100644
--- a/metadata/install-qa-check.d/60python-site
+++ b/metadata/install-qa-check.d/60python-site
@@ -9,6 +9,10 @@ python_site_check() {
 	local save=$(shopt -p nullglob)
 	shopt -s nullglob
 	local progs=( "${EPREFIX}"/usr/lib/python-exec/*/gpep517 )
+	local bad_libdirs=()
+	[[ $(get_libdir) != lib ]] && bad_libdirs=(
+		"${ED}/usr/$(get_libdir)"/{python3,pypy}*
+	)
 	${save}
 
 	local forbidden_package_names=(
@@ -183,6 +187,14 @@ python_site_check() {
 		eqawarn "https://projects.gentoo.org/python/guide/qawarn.html#stray-top-level-files-in-site-packages"
 		die "Failing install because of stray top-level files in site-packages"
 	fi
+
+	if [[ ${bad_libdirs[@]} ]]; then
+		eqawarn
+		eqawarn "QA Notice: Package installs Python files to /usr/$(get_libdir)"
+		eqawarn "instead of /usr/lib (use \$(python_get_sitedir)):"
+		eqawarn
+		eqatag -v python-site.libdir "${bad_libdirs[@]#${ED}}"
+	fi
 }
 
 python_site_check
-- 
2.43.0

[gentoo-dev] [PATCH 7/9] install-qa-check.d/60python-site: Check for UNKNOWN package name

[Michał Górny]

Bug: https://bugs.gentoo.org/836765
Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 metadata/install-qa-check.d/60python-site | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/metadata/install-qa-check.d/60python-site b/metadata/install-qa-check.d/60python-site
index facc41fbb060..26f26d6cc527 100644
--- a/metadata/install-qa-check.d/60python-site
+++ b/metadata/install-qa-check.d/60python-site
@@ -55,9 +55,9 @@ python_site_check() {
 		done < <(
 			find "${sitedir}" -maxdepth 1 '(' \
 				-name '*-0.0.0.dist-info' -o \
-				-name '*-UNKNOWN.dist-info' -o \
+				-name '*UNKNOWN*.dist-info' -o \
 				-name '*-0.0.0.egg-info' -o \
-				-name '*-UNKNOWN.egg-info' \
+				-name '*UNKNOWN*.egg-info' \
 				')' -print0
 		)
 
@@ -170,7 +170,7 @@ python_site_check() {
 	if [[ ${bad_versions[@]} ]]; then
 		eqawarn
 		eqawarn "QA Notice: The following Python packages were installed with"
-		eqawarn "invalid/suspicious versions in the site-packages directory:"
+		eqawarn "invalid/suspicious names or versions in the site-packages directory:"
 		eqawarn
 		eqatag -v python-site.bad_version "${bad_versions[@]}"
 	fi
-- 
2.43.0

[gentoo-dev] [PATCH 8/9] install-qa-check.d/60python-site: Check for out-of-sitepkg install

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 metadata/install-qa-check.d/60python-site | 28 ++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/metadata/install-qa-check.d/60python-site b/metadata/install-qa-check.d/60python-site
index 26f26d6cc527..cb31c7f98e08 100644
--- a/metadata/install-qa-check.d/60python-site
+++ b/metadata/install-qa-check.d/60python-site
@@ -31,6 +31,7 @@ python_site_check() {
 	local stray=()
 
 	local bad_versions=()
+	local outside_site=()
 	local stray_packages=()
 
 	# Avoid running the check if sufficiently new gpep517 is not installed
@@ -46,7 +47,24 @@ python_site_check() {
 		impl=${impl##*/}
 
 		# NB: using ${impl}* to catch pypy3.* for pypy3
-		local sitedir=( "${ED}"/usr/lib/${impl}*/site-packages )
+		local pydir=( "${ED}"/usr/lib/${impl}* )
+		[[ -d ${pydir} ]] || continue
+
+		# check for packages installing outside site-packages
+		case ${CATEGORY}/${PN} in
+			dev-lang/python|dev-python/pypy*)
+				;;
+			*)
+				while IFS= read -d $'\0' -r f; do
+					outside_site+=( "${f}" )
+				done < <(
+					find "${pydir}" -mindepth 1 -maxdepth 1 \
+						'!' -name site-packages -print0
+				)
+				;;
+		esac
+
+		local sitedir=( "${pydir}"/site-packages )
 		[[ -d ${sitedir} ]] || continue
 
 		# check for bad package versions
@@ -195,6 +213,14 @@ python_site_check() {
 		eqawarn
 		eqatag -v python-site.libdir "${bad_libdirs[@]#${ED}}"
 	fi
+
+	if [[ ${outside_site[@]} ]]; then
+		eqawarn
+		eqawarn "QA Notice: Files found installed directly into Python stdlib,"
+		eqawarn "instead of site-packages (use \$(python_get_sitedir)):"
+		eqawarn
+		eqatag -v python-site.stdlib "${outside_site[@]}"
+	fi
 }
 
 python_site_check
-- 
2.43.0

[gentoo-dev] [PATCH 9/9] install-qa-check.d/60python-site: allow site-packages/README.txt

[Michał Górny]

This is installed by dev-lang/python and dev-python/pypy*.
Historically, we didn't need to exempt them since the check was
in distutils-r1 and these ebuilds did not use it.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 metadata/install-qa-check.d/60python-site | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/metadata/install-qa-check.d/60python-site b/metadata/install-qa-check.d/60python-site
index cb31c7f98e08..afef445ad715 100644
--- a/metadata/install-qa-check.d/60python-site
+++ b/metadata/install-qa-check.d/60python-site
@@ -88,7 +88,8 @@ python_site_check() {
 					-name '*.pth' -o \
 					-name '*.py' -o \
 					-name '*.pyi' -o \
-					-name "*$(get_modname)" \
+					-name "*$(get_modname)" -o \
+					-name 'README.txt' \
 				')' -print0
 		)
 		# check for forbidden packages
-- 
2.43.0