[gentoo-dev] [PATCH 0/6] python-utils-r1/distutils-r1: EPYTEST_XDIST support

[gentoo-dev] [PATCH 0/6] python-utils-r1/distutils-r1: EPYTEST_XDIST support

[Michał Górny]

Hi,

Here's a patchset that introduces EPYTEST_XDIST=1 variable to
automatically use dev-python/pytest-xdist to run test suites in parallel
via epytest.  It also takes care of adding the dep if set prior to
distutils_enable_tests (as expected, pkgcheck will also check for that).

I'm also adding an EPYTEST_JOBS variable to allow overriding the job
count independently of MAKEOPTS.

The patches also include some prior refactoring and two example ebuild
conversions.

PR: https://github.com/gentoo/gentoo/pull/33667

-- 
Best regards,
Michał Górny

Michał Górny (6):
  distutils-r1.eclass: Refactor `d_e_t unittest` (NFC)
  distutils-r1.eclass: Refactor d_e_t to permit multiple test packages
  python-utils-r1.eclass: Add EPYTEST_XDIST for epytest
  distutils-r1.eclass: Add pytest-xdist dep if EPYTEST_XDIST is set
  dev-python/pyrate-limiter: Use EPYTEST_XDIST
  dev-python/numpy: Switch to EPYTEST_XDIST

 dev-python/numpy/numpy-1.26.1.ebuild          |  4 +--
 .../pyrate-limiter-3.1.0.ebuild               |  6 +---
 eclass/distutils-r1.eclass                    | 32 ++++++++++---------
 eclass/python-utils-r1.eclass                 | 31 ++++++++++++++++++
 4 files changed, 51 insertions(+), 22 deletions(-)

-- 
2.42.0

[gentoo-dev] [PATCH 1/6] distutils-r1.eclass: Refactor `d_e_t unittest` (NFC)

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/distutils-r1.eclass | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/eclass/distutils-r1.eclass b/eclass/distutils-r1.eclass
index 1cc91110dccf..9dd091921391 100644
--- a/eclass/distutils-r1.eclass
+++ b/eclass/distutils-r1.eclass
@@ -608,6 +608,8 @@ distutils_enable_tests() {
 	esac
 
 	[[ ${#} -eq 1 ]] || die "${FUNCNAME} takes exactly one argument: test-runner"
+
+	local test_deps=${RDEPEND}
 	local test_pkg
 	case ${1} in
 		nose)
@@ -619,7 +621,12 @@ distutils_enable_tests() {
 		setup.py)
 			;;
 		unittest)
-			# dep handled below
+			# unittest-or-fail is needed in py<3.12
+			test_deps+="
+				$(python_gen_cond_dep '
+					dev-python/unittest-or-fail[${PYTHON_USEDEP}]
+				' 3.10 3.11)
+			"
 			;;
 		*)
 			die "${FUNCNAME}: unsupported argument: ${1}"
@@ -628,7 +635,6 @@ distutils_enable_tests() {
 	_DISTUTILS_TEST_RUNNER=${1}
 	python_test() { distutils-r1_python_test; }
 
-	local test_deps=${RDEPEND}
 	if [[ -n ${test_pkg} ]]; then
 		if [[ ! ${DISTUTILS_SINGLE_IMPL} ]]; then
 			test_deps+=" ${test_pkg}[${PYTHON_USEDEP}]"
@@ -637,13 +643,6 @@ distutils_enable_tests() {
 				${test_pkg}[\${PYTHON_USEDEP}]
 			")"
 		fi
-	elif [[ ${1} == unittest ]]; then
-		# unittest-or-fail is needed in py<3.12
-		test_deps+="
-			$(python_gen_cond_dep '
-				dev-python/unittest-or-fail[${PYTHON_USEDEP}]
-			' 3.{9..11})
-		"
 	fi
 	if [[ -n ${test_deps} ]]; then
 		IUSE+=" test"
-- 
2.42.0

[gentoo-dev] [PATCH 2/6] distutils-r1.eclass: Refactor d_e_t to permit multiple test packages

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/distutils-r1.eclass | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/eclass/distutils-r1.eclass b/eclass/distutils-r1.eclass
index 9dd091921391..3d69911d9209 100644
--- a/eclass/distutils-r1.eclass
+++ b/eclass/distutils-r1.eclass
@@ -610,13 +610,13 @@ distutils_enable_tests() {
 	[[ ${#} -eq 1 ]] || die "${FUNCNAME} takes exactly one argument: test-runner"
 
 	local test_deps=${RDEPEND}
-	local test_pkg
+	local test_pkgs
 	case ${1} in
 		nose)
-			test_pkg=">=dev-python/nose-1.3.7_p20221026"
+			test_pkgs='>=dev-python/nose-1.3.7_p20221026[${PYTHON_USEDEP}]'
 			;;
 		pytest)
-			test_pkg=">=dev-python/pytest-7.3.1"
+			test_pkgs='>=dev-python/pytest-7.3.1[${PYTHON_USEDEP}]'
 			;;
 		setup.py)
 			;;
@@ -635,12 +635,12 @@ distutils_enable_tests() {
 	_DISTUTILS_TEST_RUNNER=${1}
 	python_test() { distutils-r1_python_test; }
 
-	if [[ -n ${test_pkg} ]]; then
+	if [[ -n ${test_pkgs} ]]; then
 		if [[ ! ${DISTUTILS_SINGLE_IMPL} ]]; then
-			test_deps+=" ${test_pkg}[${PYTHON_USEDEP}]"
+			test_deps+=" ${test_pkgs//'${PYTHON_USEDEP}'/${PYTHON_USEDEP}}"
 		else
 			test_deps+=" $(python_gen_cond_dep "
-				${test_pkg}[\${PYTHON_USEDEP}]
+				${test_pkgs}
 			")"
 		fi
 	fi
-- 
2.42.0

[gentoo-dev] [PATCH 3/6] python-utils-r1.eclass: Add EPYTEST_XDIST for epytest

[Michał Górny]

Add an `EPYTEST_XDIST` variable that can be used to enable running
the test suite in parallel via the dev-python/pytest-xdist plugin.
This also includes user-facing `EPYTEST_JOBS` to control the job count
independently of `MAKEOPTS`.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/python-utils-r1.eclass | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/eclass/python-utils-r1.eclass b/eclass/python-utils-r1.eclass
index 4a538f9942f6..f2a8d4d0f65e 100644
--- a/eclass/python-utils-r1.eclass
+++ b/eclass/python-utils-r1.eclass
@@ -1307,6 +1307,19 @@ _python_check_occluded_packages() {
 # parameter, when calling epytest.  The listed files will be entirely
 # skipped from test collection.
 
+# @VARIABLE: EPYTEST_XDIST
+# @DEFAULT_UNSET
+# @DESCRIPTION:
+# If set to a non-empty value, enables running tests in parallel
+# via pytest-xdist plugin.
+
+# @VARIABLE: EPYTEST_JOBS
+# @USER_VARIABLE
+# @DEFAULT_UNSET
+# @DESCRIPTION:
+# Specifies the number of jobs for parallel (pytest-xdist) test runs.
+# When unset, defaults to -j from MAKEOPTS, or the current nproc.
+
 # @FUNCTION: epytest
 # @USAGE: [<args>...]
 # @DESCRIPTION:
@@ -1371,6 +1384,22 @@ epytest() {
 		-p no:plus
 		-p no:tavern
 	)
+
+	if [[ ${EPYTEST_XDIST} ]]; then
+		local jobs=${EPYTEST_JOBS:-$(makeopts_jobs)}
+		if [[ ${jobs} -gt 1 ]]; then
+			args+=(
+				# explicitly enable the plugin, in case the ebuild was using
+				# PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
+				-p xdist
+				-n "${jobs}"
+				# worksteal ensures that workers don't end up idle when heavy
+				# jobs are unevenly distributed
+				--dist=worksteal
+			)
+		fi
+	fi
+
 	local x
 	for x in "${EPYTEST_DESELECT[@]}"; do
 		args+=( --deselect "${x}" )
-- 
2.42.0

[gentoo-dev] [PATCH 4/6] distutils-r1.eclass: Add pytest-xdist dep if EPYTEST_XDIST is set

[Michał Górny]

Make `distutils_enable_tests pytest` automatically add
the `dev-python/pytest-xdist` dependency if `EPYTEST_XDIST` is set.

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 eclass/distutils-r1.eclass    | 3 +++
 eclass/python-utils-r1.eclass | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/eclass/distutils-r1.eclass b/eclass/distutils-r1.eclass
index 3d69911d9209..3d756eaad556 100644
--- a/eclass/distutils-r1.eclass
+++ b/eclass/distutils-r1.eclass
@@ -617,6 +617,9 @@ distutils_enable_tests() {
 			;;
 		pytest)
 			test_pkgs='>=dev-python/pytest-7.3.1[${PYTHON_USEDEP}]'
+			if [[ ${EPYTEST_XDIST} ]]; then
+				test_pkgs+=' dev-python/pytest-xdist[${PYTHON_USEDEP}]'
+			fi
 			;;
 		setup.py)
 			;;
diff --git a/eclass/python-utils-r1.eclass b/eclass/python-utils-r1.eclass
index f2a8d4d0f65e..c61f9e2ecb94 100644
--- a/eclass/python-utils-r1.eclass
+++ b/eclass/python-utils-r1.eclass
@@ -1311,7 +1311,9 @@ _python_check_occluded_packages() {
 # @DEFAULT_UNSET
 # @DESCRIPTION:
 # If set to a non-empty value, enables running tests in parallel
-# via pytest-xdist plugin.
+# via pytest-xdist plugin.  If this variable is set prior to calling
+# distutils_enable_tests in distutils-r1, a test dependency
+# on dev-python/pytest-xdist is added automatically.
 
 # @VARIABLE: EPYTEST_JOBS
 # @USER_VARIABLE
-- 
2.42.0

[gentoo-dev] [PATCH 5/6] dev-python/pyrate-limiter: Use EPYTEST_XDIST

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild b/dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild
index 47947453dc89..a00cb916428e 100644
--- a/dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild
+++ b/dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild
@@ -26,7 +26,6 @@ RDEPEND="
 BDEPEND="
 	test? (
 		dev-python/pytest-asyncio[${PYTHON_USEDEP}]
-		dev-python/pytest-xdist[${PYTHON_USEDEP}]
 		dev-python/pyyaml[${PYTHON_USEDEP}]
 	)
 "
@@ -37,6 +36,7 @@ EPYTEST_DESELECT=(
 	# Python 3.11 is slightly faster, leading to a non-critical failure here
 	"tests/test_concurrency.py::test_concurrency[ProcessPoolExecutor-SQLiteBucket]"
 )
+EPYTEST_XDIST=1
 
 # TODO: package sphinx-copybutton
 # distutils_enable_sphinx docs \
@@ -67,7 +67,3 @@ src_test() {
 	# Clean up afterwards
 	kill "$(<"${redis_pid}")" || die
 }
-
-python_test() {
-	epytest -n "$(makeopts_jobs)" --dist=worksteal
-}
-- 
2.42.0

[gentoo-dev] [PATCH 6/6] dev-python/numpy: Switch to EPYTEST_XDIST

[Michał Górny]

Signed-off-by: Michał Górny <mgorny@gentoo.org>
---
 dev-python/numpy/numpy-1.26.1.ebuild | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev-python/numpy/numpy-1.26.1.ebuild b/dev-python/numpy/numpy-1.26.1.ebuild
index 3255d05e2c00..92564017966e 100644
--- a/dev-python/numpy/numpy-1.26.1.ebuild
+++ b/dev-python/numpy/numpy-1.26.1.ebuild
@@ -46,11 +46,11 @@ BDEPEND="
 		' 'python*')
 		dev-python/charset-normalizer[${PYTHON_USEDEP}]
 		>=dev-python/hypothesis-5.8.0[${PYTHON_USEDEP}]
-		dev-python/pytest-xdist[${PYTHON_USEDEP}]
 		>=dev-python/pytz-2019.3[${PYTHON_USEDEP}]
 	)
 "
 
+EPYTEST_XDIST=1
 distutils_enable_tests pytest
 
 python_prepare_all() {
@@ -143,7 +143,7 @@ python_test() {
 	esac
 
 	rm -rf numpy || die
-	epytest -n "$(makeopts_jobs)" --pyargs numpy
+	epytest --pyargs numpy
 }
 
 python_install_all() {
-- 
2.42.0