[gentoo-dev] [PATCH 0/3] multiprocessing.eclass: consider (GNU)MAKEFLAGS

[gentoo-dev] [PATCH 0/3] multiprocessing.eclass: consider (GNU)MAKEFLAGS

[Florian Schmaus]

We'd like portage to provide a sane default with of the "makeopts",
i.e., the maximum number of parallel jobs (--jobs) and the maximum
load average (--load-average) that build systems should try to
establish at once.

However, while --jobs is a pretty standard feature for Make-ish build
systems, --load-average is not. For example, it is not provided by BSD
Make. As such, it was decided that portage should set --load-average
in GNUMAKEFLAGS, instead of MAKEOPTS (see
https://marc.info/?l=gentoo-dev&m=169027605717531&w=2 and
https://github.com/gentoo/portage/pull/1072).

As a result, the multiprocessing.eclass should now also try to extract
the value for --jobs and --load-average not only from MAKEOPTS, but
also from (GNU)MAKEFLAGS.

While we add it, we also overhaul multiprocessing.eclass's API by
providing two new simple methods to extract the values:
get_multiprocessing_jobs() and get_multiprocessing_loadavg().

PR at https://github.com/gentoo/gentoo/pull/32385

Florian Schmaus (3):
  multiprocessing.eclass: consider (GNU)MAKEFLAGS, add
    get_makeopts_{jobs,loadavg}
  ninja-utils.eclass: use get_makeopts_{jobs,loadavg}
  meson.eclass: use get_makeopts_{jobs,loadavg}

 eclass/meson.eclass                           |  4 +-
 eclass/multiprocessing.eclass                 | 40 ++++++++++++++++---
 eclass/ninja-utils.eclass                     |  3 +-
 eclass/tests/multiprocessing_makeopts_jobs.sh | 24 ++++++++++-
 4 files changed, 61 insertions(+), 10 deletions(-)

-- 
2.41.0

[gentoo-dev] [PATCH 1/3] multiprocessing.eclass: consider (GNU)MAKEFLAGS, add get_makeopts_{jobs,loadavg}

[Florian Schmaus]

Since --load-average may not be found in other Make implementations
besides GNU Make, it is potentially found in GNUMAKEFLAGS and not in
MAKEOPTS.

Signed-off-by: Florian Schmaus <flow@gentoo.org>
---
 eclass/multiprocessing.eclass                 | 40 ++++++++++++++++---
 eclass/tests/multiprocessing_makeopts_jobs.sh | 24 ++++++++++-
 2 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/eclass/multiprocessing.eclass b/eclass/multiprocessing.eclass
index e55be636a02c..0768e7cb1e1f 100644
--- a/eclass/multiprocessing.eclass
+++ b/eclass/multiprocessing.eclass
@@ -1,4 +1,4 @@
-# Copyright 1999-2022 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 # @ECLASS: multiprocessing.eclass
@@ -64,17 +64,36 @@ get_nproc() {
 	fi
 }
 
+# @FUNCTION: get_all_makeopts
+# @INTERNAL
+# @DESCRIPTION:
+# Returns ${MAKEOPTS} ${GNUMAKEFLAGS} ${MAKEFLAGS}.
+_get_all_makeopts() {
+	echo "${MAKEOPTS} ${GNUMAKEFLAGS} ${MAKEFLAGS}"
+}
+
+# @FUNCTION: get_makeopts_jobs
+# @USAGE: [default-jobs]
+# @DESCRIPTION:
+# Return the number of jobs extracted from the make options (MAKEOPTS,
+# GNUMAKEFLAGS, MAKEFLAGS). If the make options do not specify a number,
+# then either the provided default is returned, or 1.
+get_makeopts_jobs() {
+	local makeopts="$(_get_all_makeopts)"
+	echo $(makeopts_jobs ${makeopts} ${1:-1})
+}
+
 # @FUNCTION: makeopts_jobs
 # @USAGE: [${MAKEOPTS}] [${inf:-$(( $(get_nproc) + 1 ))}]
 # @DESCRIPTION:
-# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
+# Searches the arguments (defaults are obtained via get_all_makeopts) and extracts the jobs number
 # specified therein.  Useful for running non-make tools in parallel too.
 # i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
 # number as bash normalizes it to [0, 255].  If the flags haven't specified a
 # -j flag, then "1" is shown as that is the default `make` uses.  If the flags
 # specify -j without a number, ${inf} is returned (defaults to nproc).
 makeopts_jobs() {
-	[[ $# -eq 0 ]] && set -- "${MAKEOPTS}"
+	[[ $# -eq 0 ]] && set -- "$(_get_all_makeopts)"
 	# This assumes the first .* will be more greedy than the second .*
 	# since POSIX doesn't specify a non-greedy match (i.e. ".*?").
 	local jobs=$(echo " $* " | sed -r -n \
@@ -83,10 +102,21 @@ makeopts_jobs() {
 	echo ${jobs:-1}
 }
 
+# @FUNCTION: get_makeopts_loadavg
+# @USAGE: [default-loadavg]
+# @DESCRIPTION:
+# Return the value for the load-average extracted from the make options (MAKEOPTS,
+# GNUMAKEFLAGS, MAKEFLAGS).  If the make options do not specify a value, then
+# either the optional provided default is returned, or 999.
+get_makeopts_loadavg() {
+	local makeopts="$(_get_all_makeopts)"
+	echo $(makeopts_loadavg ${makeopts} ${1:-999})
+}
+
 # @FUNCTION: makeopts_loadavg
 # @USAGE: [${MAKEOPTS}] [${inf:-999}]
 # @DESCRIPTION:
-# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the value set
+# Searches the arguments (defaults are obtained via get_all_makeopts()) and extracts the value set
 # for load-average. For make and ninja based builds this will mean new jobs are
 # not only limited by the jobs-value, but also by the current load - which might
 # get excessive due to I/O and not just due to CPU load.
@@ -95,7 +125,7 @@ makeopts_jobs() {
 # If no limit is specified or --load-average is used without a number, ${inf}
 # (defaults to 999) is returned.
 makeopts_loadavg() {
-	[[ $# -eq 0 ]] && set -- "${MAKEOPTS}"
+	[[ $# -eq 0 ]] && set -- "$(get_all_makeopts)"
 	# This assumes the first .* will be more greedy than the second .*
 	# since POSIX doesn't specify a non-greedy match (i.e. ".*?").
 	local lavg=$(echo " $* " | sed -r -n \
diff --git a/eclass/tests/multiprocessing_makeopts_jobs.sh b/eclass/tests/multiprocessing_makeopts_jobs.sh
index 37d5a7257775..56d73ef48b3c 100755
--- a/eclass/tests/multiprocessing_makeopts_jobs.sh
+++ b/eclass/tests/multiprocessing_makeopts_jobs.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 EAPI=7
@@ -9,7 +9,13 @@ inherit multiprocessing
 
 test-makeopts_jobs() {
 	local exp=$1; shift
-	tbegin "makeopts_jobs($1${2+; inf=${2}}) == ${exp}"
+	local targs
+	if [[ -v 1 ]]; then
+		targs="$1${2+; inf=${2}}"
+	else
+		targs="MAKEOPTS=\"${MAKEOPTS}\" GNUMAKEFLAGS=\"${GNUMAKEFLAGS}\" MAKEFLAGS=\"${MAKEFLAGS}\""
+	fi
+	tbegin "makeopts_jobs(${targs}) == ${exp}"
 	local indirect=$(MAKEOPTS="$*" makeopts_jobs)
 	local direct=$(makeopts_jobs "$@")
 	if [[ "${direct}" != "${indirect}" ]] ; then
@@ -50,6 +56,20 @@ for (( i = 0; i < ${#tests[@]}; i += 2 )) ; do
 	test-makeopts_jobs "${tests[i]}" "${tests[i+1]}"
 done
 
+tests=(
+	7 "" "--jobs 7" ""
+	# MAKEFLAGS override GNUMAKEFLAGS
+	8 "" "--jobs 7" "--jobs 8"
+)
+
+for (( i = 0; i < ${#tests[@]}; i += 4 )) ; do
+	MAKEOPTS="${tests[i+1]}"
+	GNUMAKEFLAGS="${tests[i+2]}"
+	MAKEFLAGS="${tests[i+3]}"
+	test-makeopts_jobs "${tests[i]}"
+	unset MAKEOPTS GNUMAKEFLAGS MAKEFLAGS
+done
+
 # test custom inf value
 test-makeopts_jobs 645 "-j" 645
 
-- 
2.41.0

[gentoo-dev] [PATCH 2/3] ninja-utils.eclass: use get_makeopts_{jobs,loadavg}

[Florian Schmaus]

Signed-off-by: Florian Schmaus <flow@gentoo.org>
---
 eclass/ninja-utils.eclass | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eclass/ninja-utils.eclass b/eclass/ninja-utils.eclass
index 5a211e81131d..1af3df9f71e5 100644
--- a/eclass/ninja-utils.eclass
+++ b/eclass/ninja-utils.eclass
@@ -1,3 +1,4 @@
+
 # Copyright 1999-2023 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
@@ -72,7 +73,7 @@ esac
 # Get the value of NINJAOPTS, inferring them from MAKEOPTS if unset.
 get_NINJAOPTS() {
 	if [[ -z ${NINJAOPTS+set} ]]; then
-		NINJAOPTS="-j$(makeopts_jobs "${MAKEOPTS}" 999) -l$(makeopts_loadavg "${MAKEOPTS}" 0)"
+		NINJAOPTS="-j$(get_makeopts_jobs 999) -l$(get_makeopts_loadavg 0)"
 	fi
 	echo "${NINJAOPTS}"
 }
-- 
2.41.0

[gentoo-dev] [PATCH 3/3] meson.eclass: use get_makeopts_{jobs,loadavg}

[Florian Schmaus]

Signed-off-by: Florian Schmaus <flow@gentoo.org>
---
 eclass/meson.eclass | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eclass/meson.eclass b/eclass/meson.eclass
index 5aff3eb58930..4757f3fa5eef 100644
--- a/eclass/meson.eclass
+++ b/eclass/meson.eclass
@@ -388,8 +388,8 @@ meson_src_compile() {
 
 	local mesoncompileargs=(
 		-C "${BUILD_DIR}"
-		--jobs "$(makeopts_jobs "${MAKEOPTS}" 0)"
-		--load-average "$(makeopts_loadavg "${MAKEOPTS}" 0)"
+		--jobs "$(get_makeopts_jobs 0)"
+		--load-average "$(get_makeopts_loadavg 0)"
 	)
 
 	case ${MESON_VERBOSE} in
-- 
2.41.0