From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 9F488158020 for ; Tue, 11 Oct 2022 19:18:45 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id CD6ACE07D8; Tue, 11 Oct 2022 19:18:44 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id B0F89E07D8 for ; Tue, 11 Oct 2022 19:18:44 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id A15FF341204 for ; Tue, 11 Oct 2022 19:18:43 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 3C37E5B2 for ; Tue, 11 Oct 2022 19:18:42 +0000 (UTC) From: "Sam James" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Sam James" Message-ID: <1665515915.d0fa3d28c5de6a0a34ac87de5e1a463adbe58405.sam@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: lib/_emerge/, /, lib/portage/util/, lib/portage/, bin/, ... X-VCS-Repository: proj/portage X-VCS-Files: NEWS bin/quickpkg lib/_emerge/BinpkgExtractorAsync.py lib/portage/gpkg.py lib/portage/package/ebuild/doebuild.py lib/portage/util/compression_probe.py lib/portage/util/cpuinfo.py X-VCS-Directories: lib/portage/util/ lib/portage/ bin/ lib/portage/package/ebuild/ lib/_emerge/ / X-VCS-Committer: sam X-VCS-Committer-Name: Sam James X-VCS-Revision: d0fa3d28c5de6a0a34ac87de5e1a463adbe58405 X-VCS-Branch: master Date: Tue, 11 Oct 2022 19:18:42 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 8c7683e0-052e-4bfd-9a3c-e4f96ee399cd X-Archives-Hash: 4878bac18fa787c2868dc2c769fd26a6 commit: d0fa3d28c5de6a0a34ac87de5e1a463adbe58405 Author: Sam James gentoo org> AuthorDate: Thu Sep 29 06:37:19 2022 +0000 Commit: Sam James gentoo org> CommitDate: Tue Oct 11 19:18:35 2022 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=d0fa3d28 binpkg: compress/decompress xz in parallel; compress zstd in parallel - As before, needs >= xz-5.3.3_alpha for parallel decompression. - zstd does not support parallel decompression. Closes: https://github.com/gentoo/portage/pull/918 Signed-off-by: Sam James gentoo.org> NEWS | 7 +++++++ bin/quickpkg | 8 +++++++- lib/_emerge/BinpkgExtractorAsync.py | 4 ++++ lib/portage/gpkg.py | 8 +++++++- lib/portage/package/ebuild/doebuild.py | 14 +++++++++++--- lib/portage/util/compression_probe.py | 6 +++--- lib/portage/util/cpuinfo.py | 22 +++++++++++++++++++++- 7 files changed, 60 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 28e8f97ca..a3591e94d 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +portage-3.0.39 (UNRELEASED) +-------------- + +Features: +* gpkg: Call xz and zstd with -T N where N is the number of jobs from ${MAKEOPTS} + for parallel compression (xz & zstd) and decompression (xz with newer versions). + portage-3.0.38.1 (2022-10-04) -------------- diff --git a/bin/quickpkg b/bin/quickpkg index 67eee697a..9ff8d7b5a 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -36,6 +36,7 @@ from portage.exception import ( PermissionDenied, ) from portage.util import ensure_dirs, shlex_split, varexpand, _xattr +from portage.util.cpuinfo import makeopts_to_job_count xattr = _xattr.xattr from portage._sets import load_default_config, SETPREFIX @@ -168,7 +169,12 @@ def quickpkg_atom(options, infos, arg, eout): % (binpkg_compression, missing_package) ) return 1 - cmd = shlex_split(varexpand(compression["compress"], mydict=settings)) + + cmd = compression["compress"].replace( + "{JOBS}", + str(makeopts_to_job_count(settings.get("MAKEOPTS", "1"))), + ) + cmd = shlex_split(varexpand(cmd, mydict=settings)) # Filter empty elements that make Popen fail cmd = [x for x in cmd if x != ""] with open(binpkg_tmpfile, "wb") as fobj: diff --git a/lib/_emerge/BinpkgExtractorAsync.py b/lib/_emerge/BinpkgExtractorAsync.py index 919837fc1..65b383998 100644 --- a/lib/_emerge/BinpkgExtractorAsync.py +++ b/lib/_emerge/BinpkgExtractorAsync.py @@ -10,6 +10,7 @@ from portage.util.compression_probe import ( compression_probe, _compressors, ) +from portage.util.cpuinfo import makeopts_to_job_count from portage.process import find_binary from portage.util import ( shlex_split, @@ -53,6 +54,9 @@ class BinpkgExtractorAsync(SpawnProcess): decomp = _compressors.get(compression_probe(self.pkg_path)) if decomp is not None: decomp_cmd = decomp.get("decompress") + decomp_cmd = decomp_cmd.replace( + "{JOBS}", str(makeopts_to_job_count(self.env.get("MAKEOPTS", "1"))) + ) elif tarfile.is_tarfile( portage._unicode_encode( self.pkg_path, encoding=portage._encodings["fs"], errors="strict" diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py index 7e1b98370..1f8fa1fd3 100644 --- a/lib/portage/gpkg.py +++ b/lib/portage/gpkg.py @@ -38,6 +38,7 @@ from portage.util._urlopen import urlopen from portage.util import writemsg from portage.util import shlex_split, varexpand from portage.util.compression_probe import _compressors +from portage.util.cpuinfo import makeopts_to_job_count from portage.process import find_binary from portage.const import MANIFEST2_HASH_DEFAULTS, HASHING_BLOCKSIZE @@ -1787,7 +1788,12 @@ class gpkg: if mode not in compressor: raise InvalidCompressionMethod("{}: {}".format(compression, mode)) - cmd = shlex_split(varexpand(compressor[mode], mydict=self.settings)) + cmd = compressor[mode] + cmd = cmd.replace( + "{JOBS}", str(makeopts_to_job_count(self.settings.get("MAKEOPTS", "1"))) + ) + cmd = shlex_split(varexpand(cmd, mydict=self.settings)) + # Filter empty elements that make Popen fail cmd = [x for x in cmd if x != ""] diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index d0d134b39..4f7049c21 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -111,7 +111,7 @@ from portage.util import ( writemsg_stdout, write_atomic, ) -from portage.util.cpuinfo import get_cpu_count +from portage.util.cpuinfo import get_cpu_count, makeopts_to_job_count from portage.util.lafilefixer import rewrite_lafile from portage.util.compression_probe import _compressors from portage.util.futures import asyncio @@ -667,8 +667,12 @@ def doebuild_environment( mysettings["PORTAGE_COMPRESSION_COMMAND"] = "cat" else: try: + compression_binary = compression["compress"].replace( + "{JOBS}", + str(makeopts_to_job_count(mysettings.get("MAKEOPTS", "1"))), + ) compression_binary = shlex_split( - varexpand(compression["compress"], mydict=settings) + varexpand(compression_binary, mydict=settings) )[0] except IndexError as e: writemsg( @@ -683,9 +687,13 @@ def doebuild_environment( % (binpkg_compression, missing_package) ) else: + compression_binary = compression["compress"].replace( + "{JOBS}", + str(makeopts_to_job_count(mysettings.get("MAKEOPTS", "1"))), + ) cmd = [ varexpand(x, mydict=settings) - for x in shlex_split(compression["compress"]) + for x in shlex_split(compression_binary) ] # Filter empty elements cmd = [x for x in cmd if x != ""] diff --git a/lib/portage/util/compression_probe.py b/lib/portage/util/compression_probe.py index 423e786bf..bbff5a3c6 100644 --- a/lib/portage/util/compression_probe.py +++ b/lib/portage/util/compression_probe.py @@ -37,12 +37,12 @@ _compressors = { "package": "app-arch/lzop", }, "xz": { - "compress": "xz ${BINPKG_COMPRESS_FLAGS}", - "decompress": "xz -d", + "compress": "xz -T{JOBS} ${BINPKG_COMPRESS_FLAGS}", + "decompress": "xz -T{JOBS} -d", "package": "app-arch/xz-utils", }, "zstd": { - "compress": "zstd ${BINPKG_COMPRESS_FLAGS}", + "compress": "zstd -T{JOBS} ${BINPKG_COMPRESS_FLAGS}", # If the compression windowLog was larger than the default of 27, # then --long=windowLog needs to be passed to the decompressor. # Therefore, pass a larger --long=31 value to the decompressor diff --git a/lib/portage/util/cpuinfo.py b/lib/portage/util/cpuinfo.py index 3cbc5b650..77529bcda 100644 --- a/lib/portage/util/cpuinfo.py +++ b/lib/portage/util/cpuinfo.py @@ -1,7 +1,9 @@ # Copyright 2015-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -__all__ = ["get_cpu_count"] +import re + +__all__ = ["get_cpu_count", "makeopts_to_job_count"] # Before you set out to change this function, figure out what you're really # asking: @@ -48,3 +50,21 @@ def get_cpu_count(): return multiprocessing.cpu_count() except (ImportError, NotImplementedError): return None + + +def makeopts_to_job_count(makeopts): + """ + Parse the job count (-jN) from MAKEOPTS. Python version of + bin/isolated-functions.sh's ___makeopts_jobs(). + + @return: Number of jobs to run or number of CPUs if none set. + """ + if not makeopts: + return get_cpu_count() + + jobs = re.match(r".*(j|--jobs=\s)\s*([0-9]+)", makeopts) + + if not jobs: + return get_cpu_count() + + return jobs.groups()[1]