[gentoo-commits] proj/portage:master commit in: lib/_emerge/, /, lib/portage/util/, lib/portage/, bin/, ...

[gentoo-commits] proj/portage:master commit in: lib/_emerge/, /, lib/portage/util/, lib/portage/, bin/, ...

[Sam James]

commit:     d0fa3d28c5de6a0a34ac87de5e1a463adbe58405
Author:     Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Thu Sep 29 06:37:19 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Oct 11 19:18:35 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d0fa3d28

binpkg: compress/decompress xz in parallel; compress zstd in parallel

- As before, needs >= xz-5.3.3_alpha for parallel decompression.
- zstd does not support parallel decompression.

Closes: https://github.com/gentoo/portage/pull/918
Signed-off-by: Sam James <sam <AT> gentoo.org>

 NEWS                                   |  7 +++++++
 bin/quickpkg                           |  8 +++++++-
 lib/_emerge/BinpkgExtractorAsync.py    |  4 ++++
 lib/portage/gpkg.py                    |  8 +++++++-
 lib/portage/package/ebuild/doebuild.py | 14 +++++++++++---
 lib/portage/util/compression_probe.py  |  6 +++---
 lib/portage/util/cpuinfo.py            | 22 +++++++++++++++++++++-
 7 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/NEWS b/NEWS
index 28e8f97ca..a3591e94d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,10 @@
+portage-3.0.39 (UNRELEASED)
+--------------
+
+Features:
+* gpkg: Call xz and zstd with -T N where N is the number of jobs from ${MAKEOPTS}
+  for parallel compression (xz & zstd) and decompression (xz with newer versions).
+
 portage-3.0.38.1 (2022-10-04)
 --------------
 

diff --git a/bin/quickpkg b/bin/quickpkg
index 67eee697a..9ff8d7b5a 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -36,6 +36,7 @@ from portage.exception import (
     PermissionDenied,
 )
 from portage.util import ensure_dirs, shlex_split, varexpand, _xattr
+from portage.util.cpuinfo import makeopts_to_job_count
 
 xattr = _xattr.xattr
 from portage._sets import load_default_config, SETPREFIX
@@ -168,7 +169,12 @@ def quickpkg_atom(options, infos, arg, eout):
                         % (binpkg_compression, missing_package)
                     )
                     return 1
-                cmd = shlex_split(varexpand(compression["compress"], mydict=settings))
+
+                cmd = compression["compress"].replace(
+                    "{JOBS}",
+                    str(makeopts_to_job_count(settings.get("MAKEOPTS", "1"))),
+                )
+                cmd = shlex_split(varexpand(cmd, mydict=settings))
                 # Filter empty elements that make Popen fail
                 cmd = [x for x in cmd if x != ""]
                 with open(binpkg_tmpfile, "wb") as fobj:

diff --git a/lib/_emerge/BinpkgExtractorAsync.py b/lib/_emerge/BinpkgExtractorAsync.py
index 919837fc1..65b383998 100644
--- a/lib/_emerge/BinpkgExtractorAsync.py
+++ b/lib/_emerge/BinpkgExtractorAsync.py
@@ -10,6 +10,7 @@ from portage.util.compression_probe import (
     compression_probe,
     _compressors,
 )
+from portage.util.cpuinfo import makeopts_to_job_count
 from portage.process import find_binary
 from portage.util import (
     shlex_split,
@@ -53,6 +54,9 @@ class BinpkgExtractorAsync(SpawnProcess):
         decomp = _compressors.get(compression_probe(self.pkg_path))
         if decomp is not None:
             decomp_cmd = decomp.get("decompress")
+            decomp_cmd = decomp_cmd.replace(
+                "{JOBS}", str(makeopts_to_job_count(self.env.get("MAKEOPTS", "1")))
+            )
         elif tarfile.is_tarfile(
             portage._unicode_encode(
                 self.pkg_path, encoding=portage._encodings["fs"], errors="strict"

diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 7e1b98370..1f8fa1fd3 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -38,6 +38,7 @@ from portage.util._urlopen import urlopen
 from portage.util import writemsg
 from portage.util import shlex_split, varexpand
 from portage.util.compression_probe import _compressors
+from portage.util.cpuinfo import makeopts_to_job_count
 from portage.process import find_binary
 from portage.const import MANIFEST2_HASH_DEFAULTS, HASHING_BLOCKSIZE
 
@@ -1787,7 +1788,12 @@ class gpkg:
         if mode not in compressor:
             raise InvalidCompressionMethod("{}: {}".format(compression, mode))
 
-        cmd = shlex_split(varexpand(compressor[mode], mydict=self.settings))
+        cmd = compressor[mode]
+        cmd = cmd.replace(
+            "{JOBS}", str(makeopts_to_job_count(self.settings.get("MAKEOPTS", "1")))
+        )
+        cmd = shlex_split(varexpand(cmd, mydict=self.settings))
+
         # Filter empty elements that make Popen fail
         cmd = [x for x in cmd if x != ""]
 

diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
index d0d134b39..4f7049c21 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -111,7 +111,7 @@ from portage.util import (
     writemsg_stdout,
     write_atomic,
 )
-from portage.util.cpuinfo import get_cpu_count
+from portage.util.cpuinfo import get_cpu_count, makeopts_to_job_count
 from portage.util.lafilefixer import rewrite_lafile
 from portage.util.compression_probe import _compressors
 from portage.util.futures import asyncio
@@ -667,8 +667,12 @@ def doebuild_environment(
                 mysettings["PORTAGE_COMPRESSION_COMMAND"] = "cat"
         else:
             try:
+                compression_binary = compression["compress"].replace(
+                    "{JOBS}",
+                    str(makeopts_to_job_count(mysettings.get("MAKEOPTS", "1"))),
+                )
                 compression_binary = shlex_split(
-                    varexpand(compression["compress"], mydict=settings)
+                    varexpand(compression_binary, mydict=settings)
                 )[0]
             except IndexError as e:
                 writemsg(
@@ -683,9 +687,13 @@ def doebuild_environment(
                         % (binpkg_compression, missing_package)
                     )
                 else:
+                    compression_binary = compression["compress"].replace(
+                        "{JOBS}",
+                        str(makeopts_to_job_count(mysettings.get("MAKEOPTS", "1"))),
+                    )
                     cmd = [
                         varexpand(x, mydict=settings)
-                        for x in shlex_split(compression["compress"])
+                        for x in shlex_split(compression_binary)
                     ]
                     # Filter empty elements
                     cmd = [x for x in cmd if x != ""]

diff --git a/lib/portage/util/compression_probe.py b/lib/portage/util/compression_probe.py
index 423e786bf..bbff5a3c6 100644
--- a/lib/portage/util/compression_probe.py
+++ b/lib/portage/util/compression_probe.py
@@ -37,12 +37,12 @@ _compressors = {
         "package": "app-arch/lzop",
     },
     "xz": {
-        "compress": "xz ${BINPKG_COMPRESS_FLAGS}",
-        "decompress": "xz -d",
+        "compress": "xz -T{JOBS} ${BINPKG_COMPRESS_FLAGS}",
+        "decompress": "xz -T{JOBS} -d",
         "package": "app-arch/xz-utils",
     },
     "zstd": {
-        "compress": "zstd ${BINPKG_COMPRESS_FLAGS}",
+        "compress": "zstd -T{JOBS} ${BINPKG_COMPRESS_FLAGS}",
         # If the compression windowLog was larger than the default of 27,
         # then --long=windowLog needs to be passed to the decompressor.
         # Therefore, pass a larger --long=31 value to the decompressor

diff --git a/lib/portage/util/cpuinfo.py b/lib/portage/util/cpuinfo.py
index 3cbc5b650..77529bcda 100644
--- a/lib/portage/util/cpuinfo.py
+++ b/lib/portage/util/cpuinfo.py
@@ -1,7 +1,9 @@
 # Copyright 2015-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-__all__ = ["get_cpu_count"]
+import re
+
+__all__ = ["get_cpu_count", "makeopts_to_job_count"]
 
 # Before you set out to change this function, figure out what you're really
 # asking:
@@ -48,3 +50,21 @@ def get_cpu_count():
         return multiprocessing.cpu_count()
     except (ImportError, NotImplementedError):
         return None
+
+
+def makeopts_to_job_count(makeopts):
+    """
+    Parse the job count (-jN) from MAKEOPTS. Python version of
+    bin/isolated-functions.sh's ___makeopts_jobs().
+
+    @return: Number of jobs to run or number of CPUs if none set.
+    """
+    if not makeopts:
+        return get_cpu_count()
+
+    jobs = re.match(r".*(j|--jobs=\s)\s*([0-9]+)", makeopts)
+
+    if not jobs:
+        return get_cpu_count()
+
+    return jobs.groups()[1]