[gentoo-dev] [PATCH 1/4] dist-kernel-utils.eclass: fix module cleanup when using binpkgs

[gentoo-dev] [PATCH 1/4] dist-kernel-utils.eclass: fix module cleanup when using binpkgs

[Andrew Ammerlaan]

When installing a binpkg -nt is not a good check because the modules in the
binpkg we are installing may actually be older then what we have in root.

Instead introduce a new function dist-kernel_get_module_suffix to find the
desired compression based on USE=modules-sign and the kernel config. Then,
remove only those files that do not match our desired compression.

Signed-off-by: Andrew Ammerlaan <andrewammerlaan@gentoo.org>
---
 eclass/dist-kernel-utils.eclass | 59 +++++++++++++++++++++++++++------
 1 file changed, 49 insertions(+), 10 deletions(-)

diff --git a/eclass/dist-kernel-utils.eclass b/eclass/dist-kernel-utils.eclass
index 4bc3fab44aae..0b0eb0ec8818 100644
--- a/eclass/dist-kernel-utils.eclass
+++ b/eclass/dist-kernel-utils.eclass
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Gentoo Authors
+# Copyright 2020-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 # @ECLASS: dist-kernel-utils.eclass
@@ -159,6 +159,35 @@ dist-kernel_PV_to_KV() {
 	echo "${kv}"
 }
 
+# @FUNCTION: dist-kernel_get_module_suffix
+# @USAGE: <kernel_dir>
+# @DESCRIPTION:
+# Returns the suffix for kernel modules based on the CONFIG_MODULES_COMPESS_*
+# setting in the kernel config and USE=modules-compress.
+dist-kernel_get_module_suffix() {
+	debug-print-function ${FUNCNAME} "${@}"
+
+	[[ ${#} -eq 1 ]] || die "${FUNCNAME}: invalid arguments"
+
+	local config=${1}/.config
+
+	if ! in_iuse modules-compress || ! use modules-compress; then
+		echo .ko
+	elif [[ ! -r ${config} ]]; then
+		die "Cannot find kernel config ${config}"
+	elif grep -q "CONFIG_MODULE_COMPRESS_NONE=y" "${config}"; then
+		echo .ko
+	elif grep -q "CONFIG_MODULE_COMPRESS_GZIP=y" "${config}"; then
+		echo .ko.gz
+	elif grep -q "CONFIG_MODULE_COMPRESS_XZ=y" "${config}"; then
+		echo .ko.xz
+	elif grep -q "CONFIG_MODULE_COMPRESS_ZSTD=y" "${config}"; then
+		echo .ko.zst
+	else
+		die "Module compression is enabled, but compressor not known"
+	fi
+}
+
 # @FUNCTION: dist-kernel_compressed_module_cleanup
 # @USAGE: <path>
 # @DESCRIPTION:
@@ -169,20 +198,29 @@ dist-kernel_compressed_module_cleanup() {
 
 	[[ ${#} -ne 1 ]] && die "${FUNCNAME}: invalid arguments"
 	local path=${1}
-	local basename f
+	local preferred=$(dist-kernel_get_module_suffix "${path}/source")
+	local basename suffix
 
 	while read -r basename; do
 		local prev=
-		for f in "${path}/${basename}"{,.gz,.xz,.zst}; do
-			if [[ ! -e ${f} ]]; then
-				continue
+		for suffix in .ko .ko.gz .ko.xz .ko.zst; do
+			[[ ${suffix} == ${preferred} ]] && continue
+			local current=${path}/${basename}${suffix}
+			[[ -f ${current} ]] || continue
+
+			if [[ -f ${path}/${basename}${preferred} ]]; then
+				# If the module with the desired compression exists, remove
+				# all other variations.
+				rm -v "${current}" || die
 			elif [[ -z ${prev} ]]; then
-				prev=${f}
-			elif [[ ${f} -nt ${prev} ]]; then
+				# If not, then keep whichever of the duplicate modules is the
+				# newest. Normally you should not end up here.
+				prev=${current}
+			elif [[ ${current} -nt ${prev} ]]; then
 				rm -v "${prev}" || die
-				prev=${f}
+				prev=${current}
 			else
-				rm -v "${f}" || die
+				rm -v "${current}" || die
 			fi
 		done
 	done < <(
@@ -192,7 +230,8 @@ dist-kernel_compressed_module_cleanup() {
 			-o -name '*.ko.gz' \
 			-o -name '*.ko.xz' \
 			-o -name '*.ko.zst' \
-			\) | sed -e 's:[.]\(gz\|xz\|zst\)$::' | sort | uniq -d || die
+			\) | sed -e 's:[.]ko\(\|[.]gz\|[.]xz\|[.]zst\)$::' |
+				sort | uniq -d || die
 	)
 }
 
-- 
2.45.2

[gentoo-dev] [PATCH 2/4] kernel-install.eclass: use dist-kernel_get_module_suffix to find compression

[Andrew Ammerlaan]

Adjusts kernel-install_compress_modules to use the new function
dist-kernel_get_module_suffix. This makes no functional difference
at the moment since gentoo-kernel-bin is the only consumer and it has
XZ compression in the config. Still this makes it possible to compile
alternate prebuilt kernels with alternate module compression support, and may
in the future help to support gzip and zstd module compression in
gentoo-kernel-bin.

Signed-off-by: Andrew Ammerlaan <andrewammerlaan@gentoo.org>
---
 eclass/kernel-install.eclass | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/eclass/kernel-install.eclass b/eclass/kernel-install.eclass
index 0a85bfb8629d..e5dd6db5b7ed 100644
--- a/eclass/kernel-install.eclass
+++ b/eclass/kernel-install.eclass
@@ -766,13 +766,35 @@ kernel-install_compress_modules() {
 
 	if use modules-compress; then
 		einfo "Compressing kernel modules ..."
-		# xz options taken from scripts/Makefile.modinst
-		# we don't do 'xz -T' because it applies multithreading per file,
-		# so it works only for big files, and we have lots of small files
-		# instead
-		find "${ED}/lib" -name '*.ko' -print0 |
-			xargs -0 -P "$(makeopts_jobs)" -n 128 \
-				xz --check=crc32 --lzma2=dict=1MiB
+		if [[ -z ${KV_FULL} ]]; then
+			KV_FULL=${PV}${KV_LOCALVERSION}
+		fi
+		local suffix=$(dist-kernel_get_module_suffix "${ED}/usr/src/linux-${KV_FULL}")
+		local compress=()
+		# Options taken from linux-mod-r1.eclass.
+		# We don't instruct the compressor to parallelize because it applies
+		# multithreading per file, so it works only for big files, and we have
+		# lots of small files instead.
+		case ${suffix} in
+			.ko)
+				return
+				;;
+			.ko.gz)
+				compress+=( gzip )
+				;;
+			.ko.xz)
+				compress+=( xz --check=crc32 --lzma2=dict=1MiB )
+				;;
+			.ko.zst)
+				compress+=( zstd -q --rm )
+				;;
+			*)
+				die "Unknown compressor: ${suffix}"
+				;;
+		esac
+
+		find "${ED}/lib/modules/${KV_FULL}" -name '*.ko' -print0 |
+			xargs -0 -P "$(makeopts_jobs)" -n 128 "${compress[@]}"
 		assert "Compressing kernel modules failed"
 	fi
 }
-- 
2.45.2

[gentoo-dev] [PATCH 3/4] eclass/tests/tests-common.sh: add in_iuse function

[Andrew Ammerlaan]

Signed-off-by: Andrew Ammerlaan <andrewammerlaan@gentoo.org>
---
 eclass/tests/tests-common.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/eclass/tests/tests-common.sh b/eclass/tests/tests-common.sh
index 45b1e20b933a..f4e18f38fee9 100644
--- a/eclass/tests/tests-common.sh
+++ b/eclass/tests/tests-common.sh
@@ -55,6 +55,8 @@ has() {
 }
 use() { has "$1" ${IUSE} ; }
 
+in_iuse() { use "$@" ; }
+
 die() {
 	echo "die: $*" 1>&2
 	exit 1
-- 
2.45.2

[gentoo-dev] [PATCH 4/4] eclass/tests/dist-kernel-utils.sh: add compressed_module_cleanup tests

[Andrew Ammerlaan]

Signed-off-by: Andrew Ammerlaan <andrewammerlaan@gentoo.org>
---
 eclass/tests/dist-kernel-utils.sh | 59 +++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/eclass/tests/dist-kernel-utils.sh b/eclass/tests/dist-kernel-utils.sh
index 50ba001f8e9c..28c8f7213a53 100755
--- a/eclass/tests/dist-kernel-utils.sh
+++ b/eclass/tests/dist-kernel-utils.sh
@@ -20,9 +20,68 @@ test_PV_to_KV() {
 	tend $?
 }
 
+test_compressed_module_cleanup() {
+	mkdir -p "${tmpdir}/source" || die
+	pushd "${tmpdir}" >/dev/null || die
+
+	local module option fail=0
+	for option in NONE GZIP XZ ZSTD; do
+		tbegin "CONFIG_MODULE_COMPRESS_${option}"
+		echo "CONFIG_MODULE_COMPRESS_${option}=y" > source/.config
+
+		touch a.ko b.ko.gz c.ko.xz d.ko.gz e.ko f.ko.xz || die
+		# ensure some files are older
+		touch -d "2 hours ago" d.ko e.ko.xz f.ko.gz || die
+
+		IUSE=modules-compress dist-kernel_compressed_module_cleanup .
+
+		local to_keep=( a.ko b.ko.gz c.ko.xz )
+		local to_remove=()
+
+		case ${option} in
+			NONE)
+				to_keep+=( d.ko e.ko f.ko.xz )
+				to_remove+=( d.ko.gz e.ko.xz f.ko.gz )
+				;;
+			GZIP)
+				to_keep+=( d.ko.gz e.ko f.ko.gz )
+				to_remove+=( d.ko e.ko.xz f.ko.xz )
+				;;
+			XZ)
+				to_keep+=( d.ko.gz e.ko.xz f.ko.xz )
+				to_remove+=( d.ko e.ko f.ko.gz )
+				;;
+			ZSTD)
+				to_keep+=( d.ko.gz e.ko f.ko.xz )
+				to_remove+=( d.ko e.ko.xz f.ko.gz )
+				;;
+		esac
+
+		for module in "${to_keep[@]}"; do
+			if [[ ! -f ${module} ]]; then
+				eerror "Module ${module} was removed"
+				fail=1
+			fi
+		done
+
+		for module in "${to_remove[@]}"; do
+			if [[ -f ${module} ]]; then
+				eerror "Module ${module} was not removed"
+				fail=1
+			fi
+		done
+		tend ${fail}
+	done
+
+	popd >/dev/null || die
+}
+
+
 test_PV_to_KV 6.0_rc1 6.0.0-rc1
 test_PV_to_KV 6.0 6.0.0
 test_PV_to_KV 6.0.1_rc1 6.0.1-rc1
 test_PV_to_KV 6.0.1 6.0.1
 
+test_compressed_module_cleanup
+
 texit
-- 
2.45.2