[gentoo-dev] [PATCH 00/15] unpacker.eclass: Tests, bugfixes and GPKG support

[gentoo-dev] [PATCH 00/15] unpacker.eclass: Tests, bugfixes and GPKG support

[Michał Górny]

Hi,

Here's a patch series for unpacker.eclass that does the following:

- add tests for unpacking various file formats
- fix handling broken/invalid `.zst` and `.7z` files
- use lowercase suffixes everywhere consistently
- add support for `.lz4` and `.lzo`
- add support for on-the-fly unpacking of the image archive from `.gpkg.tar`
- use parallel xz decompression (by @thesamesam)
- support lbzip2 if available
- fix handling `.deb` that use GNU ar format
- unpack `.deb` on-the-fly (i.e. without temporary files)

Also available as PR: https://github.com/gentoo/gentoo/pull/27431

Please review.


Michał Górny (14):
  eclass/tests: Add tests for unpacker.eclass
  unpacker.eclass: Remove `-f` from zstd arguments
  unpacker.eclass: Fix unpack_7z to respect the exit status
  unpacker.eclass: Remove support for EAPI 5
  unpacker.eclass: Use bash substitution instead of tr for lowercase
  unpacker.eclass: Use lowercase in unpacker_src_uri_depends
  unpacker.eclass: Remove uppercase RAR/LHA variants
  unpacker.eclass: Move decompressor recognition into a function
  unpacker.eclass: Add support for .lz4 and .lzo compression
  unpacker.eclass: Add on-the-fly .gpkg.tar unpacking support
  sys-kernel/gentoo-kernel-bin: Use unpacker.eclass for .gpkg.tar
  unpacker.eclass: Support lbzip2 as parallel bz2 decompressor
  unpacker.eclass: Fix handling GNU ar archives in hand-weaved impl
  unpacker.eclass: Unpack .deb packages on-the-fly as well

Sam James (1):
  unpacker.eclass: decompress xz in parallel

 eclass/tests/tests-common.sh                  |   7 +
 eclass/tests/unpacker.sh                      | 291 ++++++++++++++++++
 eclass/unpacker.eclass                        | 179 +++++++----
 .../gentoo-kernel-bin-5.19.11.ebuild          |  17 +-
 4 files changed, 428 insertions(+), 66 deletions(-)
 create mode 100755 eclass/tests/unpacker.sh

-- 
2.37.3

[gentoo-dev] [PATCH 01/15] eclass/tests: Add tests for unpacker.eclass

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/tests/tests-common.sh |   7 ++
 eclass/tests/unpacker.sh     | 233 +++++++++++++++++++++++++++++++++++
 2 files changed, 240 insertions(+)
 create mode 100755 eclass/tests/unpacker.sh

diff --git a/eclass/tests/tests-common.sh b/eclass/tests/tests-common.sh
index a677842b6ac5..45b1e20b933a 100644
--- a/eclass/tests/tests-common.sh
+++ b/eclass/tests/tests-common.sh
@@ -60,6 +60,13 @@ die() {
 	exit 1
 }
 
+assert() {
+	local x pipestatus=${PIPESTATUS[*]}
+	for x in ${pipestatus} ; do
+		[[ ${x} -eq 0 ]] || die "$@"
+	done
+}
+
 has_version() {
 	while [[ $1 == -* ]]; do
 		shift
diff --git a/eclass/tests/unpacker.sh b/eclass/tests/unpacker.sh
new file mode 100755
index 000000000000..af979b0e2995
--- /dev/null
+++ b/eclass/tests/unpacker.sh
@@ -0,0 +1,233 @@
+#!/bin/bash
+# Copyright 2022 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+source tests-common.sh || exit
+
+inherit unpacker
+
+# silence the output
+unpack_banner() { :; }
+
+TESTFILE=test.in
+TESTDIR=$(mktemp -d || die)
+trap 'cd - >/dev/null && rm -r "${TESTDIR}"' EXIT
+
+# prepare some test data
+# NB: we need something "compressible", as compress(1) will return
+# an error if the file "is larger than before compression"
+cp ../unpacker.eclass "${TESTDIR}/${TESTFILE}" || die
+cd "${TESTDIR}" || die
+
+test_unpack() {
+	local archive=${1}
+	local unpacked=${2}
+	local deps=${3}
+	local packcmd=${4}
+
+	local x
+	for x in ${deps}; do
+		if ! type "${x}" &>/dev/null; then
+			ewarn "Skipping ${archive}, tool ${x} not found"
+			return
+		fi
+	done
+
+	rm -rf testdir || die
+	mkdir -p testdir || die
+
+	tbegin "unpacking ${archive}"
+	eval "${packcmd}"
+	assert "packing ${archive} failed"
+	cd testdir || die
+	local out
+	out=$(
+		_unpacker "../${archive}" 2>&1
+	)
+	ret=$?
+	if [[ ${ret} -eq 0 ]]; then
+		if [[ ! -f ${unpacked} ]]; then
+			eerror "${unpacked} not found after unpacking"
+			ret=1
+		elif ! diff -u "${unpacked}" "../${TESTFILE}"; then
+			eerror "${unpacked} different than input"
+			ret=1
+		fi
+	fi
+	[[ ${ret} -ne 0 ]] && echo "${out}" >&2
+	tend ${ret}
+
+	cd .. || die
+	rm -f "${archive}" || die
+}
+
+test_compressed_file() {
+	local suffix=${1}
+	local tool=${2}
+
+	test_unpack "test${suffix}" test "${tool}" \
+		"${tool} -c \${TESTFILE} > \${archive}"
+}
+
+test_compressed_file_multistream() {
+	local suffix=${1}
+	local tool=${2}
+
+	test_unpack "test+multistream${suffix}" "test+multistream" "${tool}" \
+		"head -n 300 \${TESTFILE} | ${tool} -c > \${archive} &&
+		tail -n +301 \${TESTFILE} | ${tool} -c >> \${archive}"
+}
+
+test_compressed_file_with_junk() {
+	local suffix=${1}
+	local tool=${2}
+	local flag=${3}
+
+	test_unpack "test+junk${suffix}" "test+junk" "${tool}" \
+		"${tool} -c \${TESTFILE} > \${archive} && cat test.in >> \${archive}"
+}
+
+test_compressed_tar() {
+	local suffix=${1}
+	local tool=${2}
+
+	test_unpack "test${suffix}" test.in "tar ${tool}" \
+		"tar -c \${TESTFILE} | ${tool} -c > \${archive}"
+}
+
+test_compressed_cpio() {
+	local suffix=${1}
+	local tool=${2}
+
+	test_unpack "test${suffix}" test.in "cpio ${tool}" \
+		"cpio -o --quiet <<<\${TESTFILE} | ${tool} -c > \${archive}"
+}
+
+create_deb() {
+	local suffix=${1}
+	local tool=${2}
+	local archive=${3}
+	local infile=${4}
+
+	echo 2.0 > debian-binary || die
+	: > control || die
+	tar -cf control.tar control || die
+	tar -c "${infile}" | ${tool} > "data.tar${suffix}"
+	assert "packing data.tar${suffix} failed"
+	ar r "${archive}" debian-binary control.tar "data.tar${suffix}" \
+		2>/dev/null || die
+	rm -f control control.tar "data.tar${suffix}" debian-binary || die
+}
+
+test_deb() {
+	local suffix=${1}
+	local tool=${2}
+	local tool_cmd
+
+	if [[ -n ${tool} ]]; then
+		tool_cmd="${tool} -c"
+	else
+		tool_cmd=cat
+	fi
+
+	test_unpack "test-${tool}_1.2.3_noarch.deb" test.in "ar tar ${tool}" \
+		"create_deb '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}"
+}
+
+test_reject_junk() {
+	local suffix=${1}
+	local archive=test${1}
+
+	rm -rf testdir || die
+	mkdir -p testdir || die
+
+	tbegin "rejecting junk named ${archive}"
+	cat test.in >> "${archive}" || die
+	cd testdir || die
+	(
+		# some decompressors (e.g. cpio) are very verbose about junk
+		_unpacker "../${archive}" &>/dev/null
+	)
+	[[ $? -ne 0 ]]
+	ret=$?
+	tend ${ret}
+
+	cd .. || die
+	rm -f "${archive}" || die
+}
+
+test_compressed_file .bz2 bzip2
+test_compressed_file .Z compress
+test_compressed_file .gz gzip
+test_compressed_file .lzma lzma
+test_compressed_file .xz xz
+test_compressed_file .lz lzip
+test_compressed_file .zst zstd
+
+test_compressed_file_multistream .bz2 bzip2
+test_compressed_file_multistream .gz gzip
+test_compressed_file_multistream .xz xz
+test_compressed_file_multistream .lz lzip
+test_compressed_file_multistream .zst zstd
+
+test_compressed_file_with_junk .bz2 bzip2
+test_compressed_file_with_junk .lz lzip
+
+test_unpack test.tar test.in tar 'tar -cf ${archive} ${TESTFILE}'
+test_compressed_tar .tar.bz2 bzip2
+test_compressed_tar .tbz bzip2
+test_compressed_tar .tbz2 bzip2
+test_compressed_tar .tar.Z compress
+test_compressed_tar .tar.gz gzip
+test_compressed_tar .tgz gzip
+test_compressed_tar .tar.lzma lzma
+test_compressed_tar .tar.xz xz
+test_compressed_tar .txz xz
+test_compressed_tar .tar.lz lzip
+test_compressed_tar .tar.zst zstd
+
+test_unpack test.cpio test.in cpio 'cpio -o --quiet <<<${TESTFILE} > ${archive}'
+test_compressed_cpio .cpio.bz2 bzip2
+test_compressed_cpio .cpio.Z compress
+test_compressed_cpio .cpio.gz gzip
+test_compressed_cpio .cpio.lzma lzma
+test_compressed_cpio .cpio.xz xz
+test_compressed_cpio .cpio.lz lzip
+test_compressed_cpio .cpio.zst zstd
+
+test_deb
+test_deb .gz gzip
+test_deb .xz xz
+test_deb .bz2 bzip2
+test_deb .lzma lzma
+
+test_unpack test.zip test.in zip 'zip -q ${archive} ${TESTFILE}'
+# test handling non-adjusted zip with junk prepended
+test_unpack test.zip test.in zip \
+	'zip -q testdir/tmp.zip ${TESTFILE} && cat test.in testdir/tmp.zip > ${archive}'
+test_unpack test.7z test.in 7z '7z -bso0 a ${archive} ${TESTFILE}'
+test_unpack test.lha test.in lha 'lha a -q ${archive} ${TESTFILE}'
+test_unpack test.lzh test.in lha 'lha a -q ${archive} ${TESTFILE}'
+test_unpack test.rar test.in rar 'rar -idq a ${archive} ${TESTFILE}'
+
+# TODO: .run/.sh/.bin
+
+test_reject_junk .bz2
+test_reject_junk .Z
+test_reject_junk .gz
+test_reject_junk .lzma
+test_reject_junk .xz
+test_reject_junk .lz
+test_reject_junk .zst
+test_reject_junk .tar
+test_reject_junk .cpio
+test_reject_junk .deb
+test_reject_junk .zip
+test_reject_junk .7z
+test_reject_junk .rar
+test_reject_junk .lha
+test_reject_junk .lzh
+
+texit
-- 
2.37.3

[gentoo-dev] [PATCH 02/15] unpacker.eclass: Remove `-f` from zstd arguments

[Michał Górny]

Remove `-f` from zstd arguments.  This option causes zstd to ignore
input errors, notably causing it to pass invalid files through rather
than returning an error.

Closes: https://bugs.gentoo.org/872662
Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index f6e83c53bf23..c26523a419df 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -406,7 +406,7 @@ _unpacker() {
 		: ${UNPACKER_LZIP:=$(type -P plzip || type -P pdlzip || type -P lzip)}
 		comp="${UNPACKER_LZIP} -dc" ;;
 	*.zst)
-		comp="zstd -dfc" ;;
+		comp="zstd -dc" ;;
 	esac
 
 	# then figure out if there are any archiving aspects
-- 
2.37.3

[gentoo-dev] [PATCH 03/15] unpacker.eclass: Fix unpack_7z to respect the exit status

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index c26523a419df..1f2f09e33ad6 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -344,8 +344,11 @@ unpack_7z() {
 
 	local p7z=$(find_unpackable_file "$1")
 	unpack_banner "${p7z}"
-	local output="$(7z x -y "${p7z}")"
 
+	# warning: putting local and command substitution in a single call
+	# discards the exit status!
+	local output
+	output="$(7z x -y "${p7z}")"
 	if [ $? -ne 0 ]; then
 		echo "${output}" >&2
 		die "unpacking ${p7z} failed (arch=unpack_7z)"
-- 
2.37.3

[gentoo-dev] [PATCH 04/15] unpacker.eclass: Remove support for EAPI 5

[Michał Górny]

There are no ebuilds using it in EAPI 5 anymore, and it is the last EAPI
requiring support for bash 3.2.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index 1f2f09e33ad6..915a31c86437 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -4,7 +4,7 @@
 # @ECLASS: unpacker.eclass
 # @MAINTAINER:
 # base-system@gentoo.org
-# @SUPPORTED_EAPIS: 5 6 7 8
+# @SUPPORTED_EAPIS: 6 7 8
 # @BLURB: helpers for extraneous file formats and consistent behavior across EAPIs
 # @DESCRIPTION:
 # Some extraneous file formats are not part of PMS, or are only in certain
@@ -16,7 +16,7 @@
 #  - support partial unpacks?
 
 case ${EAPI:-0} in
-	[5678]) ;;
+	[678]) ;;
 	*) die "${ECLASS}: EAPI ${EAPI:-0} not supported" ;;
 esac
 
@@ -440,7 +440,7 @@ _unpacker() {
 	esac
 
 	# 7z, rar and lha/lzh are handled by package manager in EAPI < 8
-	if [[ ${EAPI} != [567] ]]; then
+	if [[ ${EAPI} != [67] ]]; then
 		case ${m} in
 		*.7z)
 			arch="unpack_7z" ;;
-- 
2.37.3

[gentoo-dev] [PATCH 05/15] unpacker.eclass: Use bash substitution instead of tr for lowercase

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index 915a31c86437..482cf141ee1d 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -389,7 +389,7 @@ _unpacker() {
 	[[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>"
 
 	local a=$1
-	local m=$(echo "${a}" | tr '[:upper:]' '[:lower:]')
+	local m=${a,,}
 	a=$(find_unpackable_file "${a}")
 
 	# first figure out the decompression method
-- 
2.37.3

[gentoo-dev] [PATCH 06/15] unpacker.eclass: Use lowercase in unpacker_src_uri_depends

[Michał Górny]

Transform the URIs to lowercase in unpacker_src_uri_depends() for
consistency with the behavior of _unpacker().

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index 482cf141ee1d..e07c25d0ffa9 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -509,7 +509,8 @@ unpacker_src_uri_depends() {
 	fi
 
 	for uri in "$@" ; do
-		case ${uri} in
+		local m=${uri,,}
+		case ${m} in
 		*.cpio.*|*.cpio)
 			d="app-arch/cpio" ;;
 		*.rar|*.RAR)
-- 
2.37.3

Re: [gentoo-dev] [PATCH 06/15] unpacker.eclass: Use lowercase in unpacker_src_uri_depends

[John Helmert III]

[-- Attachment #1: Type: text/plain, Size: 833 bytes --]

On Sun, Sep 25, 2022 at 08:23:08PM +0200, Michał Górny wrote:
> Transform the URIs to lowercase in unpacker_src_uri_depends() for
> consistency with the behavior of _unpacker().
> 
> Signed-off-by: Michał Górny <mgorny@gentoo.org>
> ---
>  eclass/unpacker.eclass | 3 ++-
>  1 file changed, 2 insertions(+), 1 deletion(-)
> 
> diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
> index 482cf141ee1d..e07c25d0ffa9 100644
> --- a/eclass/unpacker.eclass
> +++ b/eclass/unpacker.eclass
> @@ -509,7 +509,8 @@ unpacker_src_uri_depends() {
>  	fi
>  
>  	for uri in "$@" ; do
> -		case ${uri} in
> +		local m=${uri,,}
> +		case ${m} in
>  		*.cpio.*|*.cpio)
>  			d="app-arch/cpio" ;;
>  		*.rar|*.RAR)

If m is always lowercased, no need to check for uppercased extensions?

> -- 
> 2.37.3
> 
> 

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [gentoo-dev] [PATCH 06/15] unpacker.eclass: Use lowercase in unpacker_src_uri_depends

[John Helmert III]

[-- Attachment #1: Type: text/plain, Size: 1008 bytes --]

On Sun, Sep 25, 2022 at 04:04:07PM -0500, John Helmert III wrote:
> On Sun, Sep 25, 2022 at 08:23:08PM +0200, Michał Górny wrote:
> > Transform the URIs to lowercase in unpacker_src_uri_depends() for
> > consistency with the behavior of _unpacker().
> > 
> > Signed-off-by: Michał Górny <mgorny@gentoo.org>
> > ---
> >  eclass/unpacker.eclass | 3 ++-
> >  1 file changed, 2 insertions(+), 1 deletion(-)
> > 
> > diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
> > index 482cf141ee1d..e07c25d0ffa9 100644
> > --- a/eclass/unpacker.eclass
> > +++ b/eclass/unpacker.eclass
> > @@ -509,7 +509,8 @@ unpacker_src_uri_depends() {
> >  	fi
> >  
> >  	for uri in "$@" ; do
> > -		case ${uri} in
> > +		local m=${uri,,}
> > +		case ${m} in
> >  		*.cpio.*|*.cpio)
> >  			d="app-arch/cpio" ;;
> >  		*.rar|*.RAR)
> 
> If m is always lowercased, no need to check for uppercased extensions?

Sorry, this was done in the next patch

> 
> > -- 
> > 2.37.3
> > 
> > 



[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

[gentoo-dev] [PATCH 07/15] unpacker.eclass: Remove uppercase RAR/LHA variants

[Michał Górny]

Remove the uppercase variants of RAR/LHA that were copied from Portage
implementation.  The functions always convert filenames to lowercase,
so accounting for them is redundant.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index e07c25d0ffa9..ca6761488100 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -444,9 +444,9 @@ _unpacker() {
 		case ${m} in
 		*.7z)
 			arch="unpack_7z" ;;
-		*.rar|*.RAR)
+		*.rar)
 			arch="unpack_rar" ;;
-		*.LHA|*.LHa|*.lha|*.lzh)
+		*.lha|*.lzh)
 			arch="unpack_lha" ;;
 		esac
 	fi
@@ -513,7 +513,7 @@ unpacker_src_uri_depends() {
 		case ${m} in
 		*.cpio.*|*.cpio)
 			d="app-arch/cpio" ;;
-		*.rar|*.RAR)
+		*.rar)
 			d="app-arch/unrar" ;;
 		*.7z)
 			d="app-arch/p7zip" ;;
@@ -525,7 +525,7 @@ unpacker_src_uri_depends() {
 			d="|| ( app-arch/plzip app-arch/pdlzip app-arch/lzip )" ;;
 		*.zst)
 			d="app-arch/zstd" ;;
-		*.LHA|*.LHa|*.lha|*.lzh)
+		*.lha|*.lzh)
 			d="app-arch/lha" ;;
 		esac
 		deps+=" ${d}"
-- 
2.37.3

[gentoo-dev] [PATCH 08/15] unpacker.eclass: Move decompressor recognition into a function

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 44 +++++++++++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 18 deletions(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index ca6761488100..8fb1c2abd1cf 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -379,6 +379,31 @@ unpack_lha() {
 	lha xfq "${lha}" || die "unpacking ${lha} failed (arch=unpack_lha)"
 }
 
+# @FUNCTION: _unpacker_get_decompressor
+# @INTERNAL
+# @USAGE: <filename>
+# @DESCRIPTION:
+# Get decompressor command for specified filename.
+_unpacker_get_decompressor() {
+	case ${1} in
+	*.bz2|*.tbz|*.tbz2)
+		local bzcmd=${PORTAGE_BZIP2_COMMAND:-$(type -P pbzip2 || type -P bzip2)}
+		local bzuncmd=${PORTAGE_BUNZIP2_COMMAND:-${bzcmd} -d}
+		: ${UNPACKER_BZ2:=${bzuncmd}}
+		echo "${UNPACKER_BZ2} -c"
+		;;
+	*.z|*.gz|*.tgz)
+		echo "gzip -dc" ;;
+	*.lzma|*.xz|*.txz)
+		echo "xz -dc" ;;
+	*.lz)
+		: ${UNPACKER_LZIP:=$(type -P plzip || type -P pdlzip || type -P lzip)}
+		echo "${UNPACKER_LZIP} -dc" ;;
+	*.zst)
+		echo "zstd -dc" ;;
+	esac
+}
+
 # @FUNCTION: _unpacker
 # @USAGE: <one archive to unpack>
 # @INTERNAL
@@ -393,24 +418,7 @@ _unpacker() {
 	a=$(find_unpackable_file "${a}")
 
 	# first figure out the decompression method
-	local comp=""
-	case ${m} in
-	*.bz2|*.tbz|*.tbz2)
-		local bzcmd=${PORTAGE_BZIP2_COMMAND:-$(type -P pbzip2 || type -P bzip2)}
-		local bzuncmd=${PORTAGE_BUNZIP2_COMMAND:-${bzcmd} -d}
-		: ${UNPACKER_BZ2:=${bzuncmd}}
-		comp="${UNPACKER_BZ2} -c"
-		;;
-	*.z|*.gz|*.tgz)
-		comp="gzip -dc" ;;
-	*.lzma|*.xz|*.txz)
-		comp="xz -dc" ;;
-	*.lz)
-		: ${UNPACKER_LZIP:=$(type -P plzip || type -P pdlzip || type -P lzip)}
-		comp="${UNPACKER_LZIP} -dc" ;;
-	*.zst)
-		comp="zstd -dc" ;;
-	esac
+	local comp=$(_unpacker_get_decompressor "${m}")
 
 	# then figure out if there are any archiving aspects
 	local arch=""
-- 
2.37.3

[gentoo-dev] [PATCH 09/15] unpacker.eclass: Add support for .lz4 and .lzo compression

[Michał Górny]

Add support for .lz4 and .lzo formats that can be used for .tar.gpkg
compression.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/tests/unpacker.sh | 7 +++++++
 eclass/unpacker.eclass   | 8 ++++++++
 2 files changed, 15 insertions(+)

diff --git a/eclass/tests/unpacker.sh b/eclass/tests/unpacker.sh
index af979b0e2995..60b651759a52 100755
--- a/eclass/tests/unpacker.sh
+++ b/eclass/tests/unpacker.sh
@@ -165,6 +165,8 @@ test_compressed_file .lzma lzma
 test_compressed_file .xz xz
 test_compressed_file .lz lzip
 test_compressed_file .zst zstd
+test_compressed_file .lz4 lz4
+test_compressed_file .lzo lzop
 
 test_compressed_file_multistream .bz2 bzip2
 test_compressed_file_multistream .gz gzip
@@ -187,6 +189,8 @@ test_compressed_tar .tar.xz xz
 test_compressed_tar .txz xz
 test_compressed_tar .tar.lz lzip
 test_compressed_tar .tar.zst zstd
+test_compressed_tar .tar.lz4 lz4
+test_compressed_tar .tar.lzo lzop
 
 test_unpack test.cpio test.in cpio 'cpio -o --quiet <<<${TESTFILE} > ${archive}'
 test_compressed_cpio .cpio.bz2 bzip2
@@ -196,6 +200,8 @@ test_compressed_cpio .cpio.lzma lzma
 test_compressed_cpio .cpio.xz xz
 test_compressed_cpio .cpio.lz lzip
 test_compressed_cpio .cpio.zst zstd
+test_compressed_cpio .cpio.lz4 lz4
+test_compressed_cpio .cpio.lzo lzop
 
 test_deb
 test_deb .gz gzip
@@ -223,6 +229,7 @@ test_reject_junk .lz
 test_reject_junk .zst
 test_reject_junk .tar
 test_reject_junk .cpio
+test_reject_junk .gpkg.tar
 test_reject_junk .deb
 test_reject_junk .zip
 test_reject_junk .7z
diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index 8fb1c2abd1cf..a64c5eae18aa 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -401,6 +401,10 @@ _unpacker_get_decompressor() {
 		echo "${UNPACKER_LZIP} -dc" ;;
 	*.zst)
 		echo "zstd -dc" ;;
+	*.lz4)
+		echo "lz4 -dc" ;;
+	*.lzo)
+		echo "lzop -dc" ;;
 	esac
 }
 
@@ -535,6 +539,10 @@ unpacker_src_uri_depends() {
 			d="app-arch/zstd" ;;
 		*.lha|*.lzh)
 			d="app-arch/lha" ;;
+		*.lz4)
+			d="app-arch/lz4" ;;
+		*.lzo)
+			d="app-arch/lzop" ;;
 		esac
 		deps+=" ${d}"
 	done
-- 
2.37.3

[gentoo-dev] [PATCH 10/15] unpacker.eclass: Add on-the-fly .gpkg.tar unpacking support

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/tests/unpacker.sh | 47 ++++++++++++++++++++++++++++++++++++++++
 eclass/unpacker.eclass   | 38 ++++++++++++++++++++++++++++++++
 2 files changed, 85 insertions(+)

diff --git a/eclass/tests/unpacker.sh b/eclass/tests/unpacker.sh
index 60b651759a52..bbbfa32623ab 100755
--- a/eclass/tests/unpacker.sh
+++ b/eclass/tests/unpacker.sh
@@ -136,6 +136,43 @@ test_deb() {
 		"create_deb '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}"
 }
 
+create_gpkg() {
+	local suffix=${1}
+	local tool=${2}
+	local archive=${3}
+	local infile=${4}
+	local gpkg_dir=${archive%.gpkg.tar}
+
+	mkdir image metadata "${gpkg_dir}" || die
+	cp "${infile}" image/ || die
+	tar -c metadata | ${tool} > "${gpkg_dir}/metadata.tar${suffix}"
+	assert "packing metadata.tar${suffix} failed"
+	: > "${gpkg_dir}/metadata.tar${suffix}.sig" || die
+	tar -c image | ${tool} > "${gpkg_dir}/image.tar${suffix}"
+	assert "packing image.tar${suffix} failed"
+	: > "${gpkg_dir}/image.tar${suffix}.sig" || die
+	: > "${gpkg_dir}"/gpkg-1 || die
+	tar -cf "${archive}" --format=ustar \
+		"${gpkg_dir}"/{gpkg-1,{metadata,image}.tar"${suffix}"} || die
+	rm -r image metadata "${gpkg_dir}" || die
+}
+
+test_gpkg() {
+	local suffix=${1}
+	local tool=${2}
+	local tool_cmd
+
+	if [[ -n ${tool} ]]; then
+		tool_cmd="${tool} -c"
+	else
+		tool_cmd=cat
+	fi
+
+	test_unpack "test-${tool}-1.2.3-1.gpkg.tar" \
+		"test-${tool}-1.2.3-1/image/test.in" "tar ${tool}" \
+		"create_gpkg '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}"
+}
+
 test_reject_junk() {
 	local suffix=${1}
 	local archive=test${1}
@@ -209,6 +246,16 @@ test_deb .xz xz
 test_deb .bz2 bzip2
 test_deb .lzma lzma
 
+test_gpkg
+test_gpkg .gz gzip
+test_gpkg .bz2 bzip2
+test_gpkg .lz4 lz4
+test_gpkg .lz lzip
+test_gpkg .lzma lzma
+test_gpkg .lzo lzop
+test_gpkg .xz xz
+test_gpkg .zst zstd
+
 test_unpack test.zip test.in zip 'zip -q ${archive} ${TESTFILE}'
 # test handling non-adjusted zip with junk prepended
 test_unpack test.zip test.in zip \
diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index a64c5eae18aa..70a46ac19709 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -408,6 +408,42 @@ _unpacker_get_decompressor() {
 	esac
 }
 
+# @FUNCTION: unpack_gpkg
+# @USAGE: <gpkg file>
+# @DESCRIPTION:
+# Unpack the image subarchive of a GPKG package on-the-fly, preserving
+# the original directory structure (i.e. into <gpkg-dir>/image).
+unpack_gpkg() {
+	[[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>"
+
+	local gpkg=$(find_unpackable_file "$1")
+	unpack_banner "${gpkg}"
+
+	local l images=()
+	while read -r l; do
+		case ${l} in
+			*/image.tar*.sig)
+				;;
+			*/image.tar*)
+				images+=( "${l}" )
+				;;
+		esac
+	done < <(tar -tf "${gpkg}" || die "unable to list ${gpkg}")
+
+	if [[ ${#images[@]} -eq 0 ]]; then
+		die "No image.tar found in ${gpkg}"
+	elif [[ ${#images[@]} -gt 1 ]]; then
+		die "More than one image.tar found in ${gpkg}"
+	fi
+
+	local decomp=$(_unpacker_get_decompressor "${images[0]}")
+	local dirname=${images[0]%/*}
+	mkdir -p "${dirname}" || die
+	tar -xOf "${gpkg}" "${images[0]}" | ${decomp:-cat} |
+		tar --no-same-owner -xC "${dirname}"
+	assert "Unpacking ${gpkg} failed"
+}
+
 # @FUNCTION: _unpacker
 # @USAGE: <one archive to unpack>
 # @INTERNAL
@@ -427,6 +463,8 @@ _unpacker() {
 	# then figure out if there are any archiving aspects
 	local arch=""
 	case ${m} in
+	*.gpkg.tar)
+		arch="unpack_gpkg" ;;
 	*.tgz|*.tbz|*.tbz2|*.txz|*.tar.*|*.tar)
 		arch="tar --no-same-owner -xof" ;;
 	*.cpio.*|*.cpio)
-- 
2.37.3

[gentoo-dev] [PATCH 11/15] sys-kernel/gentoo-kernel-bin: Use unpacker.eclass for .gpkg.tar

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 .../gentoo-kernel-bin-5.19.11.ebuild            | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/sys-kernel/gentoo-kernel-bin/gentoo-kernel-bin-5.19.11.ebuild b/sys-kernel/gentoo-kernel-bin/gentoo-kernel-bin-5.19.11.ebuild
index 966fadbe839a..0ba336c5c1aa 100644
--- a/sys-kernel/gentoo-kernel-bin/gentoo-kernel-bin-5.19.11.ebuild
+++ b/sys-kernel/gentoo-kernel-bin/gentoo-kernel-bin-5.19.11.ebuild
@@ -3,7 +3,7 @@
 
 EAPI=8
 
-inherit kernel-install toolchain-funcs
+inherit kernel-install toolchain-funcs unpacker
 
 MY_P=linux-${PV%.*}
 GENPATCHES_P=genpatches-${PV%.*}-$(( ${PV##*.} + 2 ))
@@ -55,11 +55,6 @@ QA_PREBUILT='*'
 KV_LOCALVERSION='-gentoo-dist'
 KPV=${PV}${KV_LOCALVERSION}
 
-src_unpack() {
-	default
-	unpack "${BINPKG}"/image.tar.xz
-}
-
 src_prepare() {
 	local PATCHES=(
 		# meh, genpatches have no directory
@@ -102,22 +97,22 @@ src_configure() {
 	)
 
 	mkdir modprep || die
-	cp "image/usr/src/linux-${KPV}/.config" modprep/ || die
+	cp "${BINPKG}/image/usr/src/linux-${KPV}/.config" modprep/ || die
 	emake -C "${MY_P}" "${makeargs[@]}" modules_prepare
 }
 
 src_test() {
 	kernel-install_test "${KPV}" \
-		"${WORKDIR}/image/usr/src/linux-${KPV}/$(dist-kernel_get_image_path)" \
-		"image/lib/modules/${KPV}"
+		"${WORKDIR}/${BINPKG}/image/usr/src/linux-${KPV}/$(dist-kernel_get_image_path)" \
+		"${BINPKG}/image/lib/modules/${KPV}"
 }
 
 src_install() {
-	mv image/{lib,usr} "${ED}"/ || die
+	mv "${BINPKG}"/image/{lib,usr} "${ED}"/ || die
 
 	# FIXME: requires proper mount-boot
 	if [[ -d boot/dtbs ]]; then
-		mv image/boot "${ED}"/ || die
+		mv "${BINPKG}"/image/boot "${ED}"/ || die
 	fi
 
 	# strip out-of-source build stuffs from modprep
-- 
2.37.3

[gentoo-dev] [PATCH 12/15] unpacker.eclass: decompress xz in parallel

[Michał Górny]

From: Sam James <sam@gentoo.org>

>= xz 5.3.3_alpha supports parallel decompression, so let's use it.

As recently added to Portage (see https://github.com/gentoo/portage/commit/48d107e5c1a103d59a053aebeefa9a5aac5c32ff).

Signed-off-by: Sam James <sam@gentoo.org>
---
 eclass/unpacker.eclass | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index 70a46ac19709..d96b56609869 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -23,7 +23,7 @@ esac
 if [[ -z ${_UNPACKER_ECLASS} ]]; then
 _UNPACKER_ECLASS=1
 
-inherit toolchain-funcs
+inherit multiprocessing toolchain-funcs
 
 # @ECLASS_VARIABLE: UNPACKER_BZ2
 # @USER_VARIABLE
@@ -395,7 +395,7 @@ _unpacker_get_decompressor() {
 	*.z|*.gz|*.tgz)
 		echo "gzip -dc" ;;
 	*.lzma|*.xz|*.txz)
-		echo "xz -dc" ;;
+		echo "xz -T$(makeopts_jobs) -dc" ;;
 	*.lz)
 		: ${UNPACKER_LZIP:=$(type -P plzip || type -P pdlzip || type -P lzip)}
 		echo "${UNPACKER_LZIP} -dc" ;;
-- 
2.37.3

[gentoo-dev] [PATCH 13/15] unpacker.eclass: Support lbzip2 as parallel bz2 decompressor

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index d96b56609869..370f00a83bba 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -30,7 +30,8 @@ inherit multiprocessing toolchain-funcs
 # @DEFAULT_UNSET
 # @DESCRIPTION:
 # Utility to use to decompress bzip2 files.  Will dynamically pick between
-# `pbzip2` and `bzip2`.  Make sure your choice accepts the "-dc" options.
+# `lbzip2`, `pbzip2` and `bzip2`.  Make sure your choice accepts the "-dc"
+# options.
 # Note: this is meant for users to set, not ebuilds.
 
 # @ECLASS_VARIABLE: UNPACKER_LZIP
@@ -387,7 +388,9 @@ unpack_lha() {
 _unpacker_get_decompressor() {
 	case ${1} in
 	*.bz2|*.tbz|*.tbz2)
-		local bzcmd=${PORTAGE_BZIP2_COMMAND:-$(type -P pbzip2 || type -P bzip2)}
+		local bzcmd=${PORTAGE_BZIP2_COMMAND:-$(
+			type -P lbzip2 || type -P pbzip2 || type -P bzip2
+		)}
 		local bzuncmd=${PORTAGE_BUNZIP2_COMMAND:-${bzcmd} -d}
 		: ${UNPACKER_BZ2:=${bzuncmd}}
 		echo "${UNPACKER_BZ2} -c"
-- 
2.37.3

[gentoo-dev] [PATCH 14/15] unpacker.eclass: Fix handling GNU ar archives in hand-weaved impl

[Michał Górny]

Fix the hand-weaved implementation of ar unpacking that is used
on Prefix to handle slash-terminated filenames of GNU ar format
correctly.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/tests/unpacker.sh | 4 ++++
 eclass/unpacker.eclass   | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/eclass/tests/unpacker.sh b/eclass/tests/unpacker.sh
index bbbfa32623ab..e42e656756c4 100755
--- a/eclass/tests/unpacker.sh
+++ b/eclass/tests/unpacker.sh
@@ -134,6 +134,10 @@ test_deb() {
 
 	test_unpack "test-${tool}_1.2.3_noarch.deb" test.in "ar tar ${tool}" \
 		"create_deb '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}"
+	# also test with the hand-weaved implementation used on Prefix
+	EPREFIX=/foo \
+	test_unpack "test_pfx-${tool}_1.2.3_noarch.deb" test.in "ar tar ${tool}" \
+		"create_deb '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}"
 }
 
 create_gpkg() {
diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index 370f00a83bba..100f11428622 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -282,6 +282,8 @@ unpack_deb() {
 			local f timestamp uid gid mode size magic
 			while read f timestamp uid gid mode size magic ; do
 				[[ -n ${f} && -n ${size} ]] || continue # ignore empty lines
+				# GNU ar uses / as filename terminator (and .deb permits that)
+				f=${f%/}
 				if [[ ${f} = "data.tar"* ]] ; then
 					head -c "${size}" > "${f}"
 				else
-- 
2.37.3

[gentoo-dev] [PATCH 15/15] unpacker.eclass: Unpack .deb packages on-the-fly as well

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/unpacker.eclass | 60 +++++++++++++++++++++++-------------------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index 100f11428622..3d23151b636e 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -273,33 +273,39 @@ unpack_deb() {
 
 	unpack_banner "${deb}"
 
-	# on AIX ar doesn't work out as their ar used a different format
-	# from what GNU ar (and thus what .deb files) produce
-	if [[ -n ${EPREFIX} ]] ; then
-		{
-			read # global header
-			[[ ${REPLY} = "!<arch>" ]] || die "${deb} does not seem to be a deb archive"
-			local f timestamp uid gid mode size magic
-			while read f timestamp uid gid mode size magic ; do
-				[[ -n ${f} && -n ${size} ]] || continue # ignore empty lines
-				# GNU ar uses / as filename terminator (and .deb permits that)
-				f=${f%/}
-				if [[ ${f} = "data.tar"* ]] ; then
-					head -c "${size}" > "${f}"
-				else
-					head -c "${size}" > /dev/null # trash it
-				fi
-			done
-		} < "${deb}"
-	else
-		$(tc-getBUILD_AR) x "${deb}" || die
-	fi
-
-	unpacker ./data.tar*
-
-	# Clean things up #458658.  No one seems to actually care about
-	# these, so wait until someone requests to do something else ...
-	rm -f debian-binary {control,data}.tar*
+	{
+		# on AIX ar doesn't work out as their ar used a different format
+		# from what GNU ar (and thus what .deb files) produce
+		if [[ -n ${EPREFIX} ]] ; then
+			{
+				read # global header
+				[[ ${REPLY} = "!<arch>" ]] || die "${deb} does not seem to be a deb archive"
+				local f timestamp uid gid mode size magic
+				while read f timestamp uid gid mode size magic ; do
+					[[ -n ${f} && -n ${size} ]] || continue # ignore empty lines
+					# GNU ar uses / as filename terminator (and .deb permits that)
+					f=${f%/}
+					if [[ ${f} = "data.tar"* ]] ; then
+						local decomp=$(_unpacker_get_decompressor "${f}")
+						head -c "${size}" | ${decomp:-cat}
+						assert "unpacking ${f} from ${deb} failed"
+						break
+					else
+						head -c "${size}" > /dev/null # trash it
+					fi
+				done
+			} < "${deb}"
+		else
+			local f=$(
+				$(tc-getBUILD_AR) t "${deb}" | grep ^data.tar
+				assert "data not found in ${deb}"
+			)
+			local decomp=$(_unpacker_get_decompressor "${f}")
+			$(tc-getBUILD_AR) p "${deb}" "${f}" | ${decomp:-cat}
+			assert "unpacking ${f} from ${deb} failed"
+		fi
+	} | tar --no-same-owner -x
+	assert "unpacking ${deb} failed"
 }
 
 # @FUNCTION: unpack_cpio
-- 
2.37.3