From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 0C40615808B for ; Tue, 13 Feb 2024 09:34:16 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 17D93E2A1A; Tue, 13 Feb 2024 09:33:58 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [140.211.166.183]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id EB79BE2A1A for ; Tue, 13 Feb 2024 09:33:57 +0000 (UTC) From: Zac Medico To: gentoo-portage-dev@lists.gentoo.org Cc: Zac Medico Subject: [gentoo-portage-dev] [PATCH 9/9] EbuildPhase: async_check_locale Date: Tue, 13 Feb 2024 01:33:13 -0800 Message-ID: <20240213093313.3467599-10-zmedico@gentoo.org> X-Mailer: git-send-email 2.41.0 In-Reply-To: <20240213093313.3467599-1-zmedico@gentoo.org> References: <20240213093313.3467599-1-zmedico@gentoo.org> Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Archives-Salt: f8961d9f-6c6f-476e-b0ea-5f90e417ecd5 X-Archives-Hash: 39dc8773a788ffe86137afb5d08d730f Change config.environ() check_locale calls to async_check_locale calls in the EbuildPhase _async_start method in order to eliminate synchronous waiting for child processes in the main event loop thread. Bug: https://bugs.gentoo.org/923841 Signed-off-by: Zac Medico --- lib/_emerge/EbuildMetadataPhase.py | 4 +++ lib/_emerge/EbuildPhase.py | 28 ++++++++++++++++++- lib/portage/package/ebuild/config.py | 26 ++++++++--------- lib/portage/util/futures/_asyncio/__init__.py | 9 ++++++ lib/portage/util/locale.py | 28 +++++++++++++------ 5 files changed, 70 insertions(+), 25 deletions(-) diff --git a/lib/_emerge/EbuildMetadataPhase.py b/lib/_emerge/EbuildMetadataPhase.py index 9fcdabe840..90a3ea05aa 100644 --- a/lib/_emerge/EbuildMetadataPhase.py +++ b/lib/_emerge/EbuildMetadataPhase.py @@ -8,6 +8,7 @@ import portage portage.proxy.lazyimport.lazyimport( globals(), + "_emerge.EbuildPhase:_setup_locale", "portage.package.ebuild._metadata_invalid:eapi_invalid", ) from portage import os @@ -83,6 +84,9 @@ class EbuildMetadataPhase(SubProcess): settings.setcpv(self.cpv) settings.configdict["pkg"]["EAPI"] = parsed_eapi + # This requires above setcpv and EAPI setup. + await _setup_locale(self.settings) + debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py index c81bf54a81..c8caf73722 100644 --- a/lib/_emerge/EbuildPhase.py +++ b/lib/_emerge/EbuildPhase.py @@ -1,4 +1,4 @@ -# Copyright 1999-2021 Gentoo Authors +# Copyright 1999-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools @@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import ( _prepare_fake_distdir, _prepare_fake_filesdir, ) +from portage.eapi import _get_eapi_attrs from portage.util import writemsg, ensure_dirs from portage.util._async.AsyncTaskFuture import AsyncTaskFuture from portage.util._async.BuildLogger import BuildLogger @@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport( + "_post_src_install_write_metadata," + "_preinst_bsdflags", "portage.util.futures.unix_events:_set_nonblocking", + "portage.util.locale:async_check_locale,split_LC_ALL", ) from portage import os from portage import _encodings from portage import _unicode_encode +async def _setup_locale(settings): + eapi_attrs = _get_eapi_attrs(settings["EAPI"]) + if eapi_attrs.posixish_locale: + split_LC_ALL(settings) + settings["LC_COLLATE"] = "C" + # check_locale() returns None when check can not be executed. + if await async_check_locale(silent=True, env=settings.environ()) is False: + # try another locale + for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"): + settings["LC_CTYPE"] = l + if await async_check_locale(silent=True, env=settings.environ()): + # TODO: output the following only once + # writemsg( + # _("!!! LC_CTYPE unsupported, using %s instead\n") + # % self.settings["LC_CTYPE"] + # ) + break + else: + raise AssertionError("C locale did not pass the test!") + + class EbuildPhase(CompositeTask): __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + ("_ebuild_lock",) @@ -94,6 +117,9 @@ class EbuildPhase(CompositeTask): self._start_task(AsyncTaskFuture(future=future), self._async_start_exit) async def _async_start(self): + + await _setup_locale(self.settings) + need_builddir = self.phase not in EbuildProcess._phases_without_builddir if need_builddir: diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py index d7b0ca5676..35c77486ec 100644 --- a/lib/portage/package/ebuild/config.py +++ b/lib/portage/package/ebuild/config.py @@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport( "portage.dbapi.vartree:vartree", "portage.package.ebuild.doebuild:_phase_func_map", "portage.util.compression_probe:_compressors", - "portage.util.locale:check_locale,split_LC_ALL", ) from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode from portage.const import ( @@ -3368,20 +3367,17 @@ class config: mydict["EBUILD_PHASE_FUNC"] = phase_func if eapi_attrs.posixish_locale: - split_LC_ALL(mydict) - mydict["LC_COLLATE"] = "C" - # check_locale() returns None when check can not be executed. - if check_locale(silent=True, env=mydict) is False: - # try another locale - for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"): - mydict["LC_CTYPE"] = l - if check_locale(silent=True, env=mydict): - # TODO: output the following only once - # writemsg(_("!!! LC_CTYPE unsupported, using %s instead\n") - # % mydict["LC_CTYPE"]) - break - else: - raise AssertionError("C locale did not pass the test!") + if mydict.get("LC_ALL"): + # Sometimes this method is called for processes + # that are not ebuild phases, so only raise + # AssertionError for actual ebuild phases. + if phase and phase not in ("clean", "cleanrm", "fetch"): + raise AssertionError( + f"LC_ALL={mydict['LC_ALL']} for posixish locale. It seems that split_LC_ALL was not called for phase {phase}?" + ) + elif "LC_ALL" in mydict: + # Delete placeholder from split_LC_ALL. + del mydict["LC_ALL"] if not eapi_attrs.exports_PORTDIR: mydict.pop("PORTDIR", None) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index b6481c281e..22241f335d 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -16,6 +16,7 @@ __all__ = ( "set_child_watcher", "get_event_loop_policy", "set_event_loop_policy", + "run", "shield", "sleep", "Task", @@ -109,6 +110,14 @@ def set_child_watcher(watcher): return get_event_loop_policy().set_child_watcher(watcher) +# Emulate run since it's the preferred python API. +def run(coro): + return _safe_loop().run_until_complete(coro) + + +run.__doc__ = _real_asyncio.run.__doc__ + + def create_subprocess_exec(*args, **kwargs): """ Create a subprocess. diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py index b5da8d949b..b6a41e7655 100644 --- a/lib/portage/util/locale.py +++ b/lib/portage/util/locale.py @@ -17,6 +17,7 @@ import traceback import portage from portage.util import _unicode_decode, writemsg_level from portage.util._ctypes import find_library, LoadLibrary +from portage.util.futures import asyncio locale_categories = ( @@ -121,7 +122,10 @@ def check_locale(silent=False, env=None): warning and returns False if it is not. Returns None if the check can not be executed due to platform limitations. """ + return asyncio.run(async_check_locale(silent=silent, env=env)) + +async def async_check_locale(silent=False, env=None): if env is not None: for v in ("LC_ALL", "LC_CTYPE", "LANG"): if v in env: @@ -135,20 +139,17 @@ def check_locale(silent=False, env=None): except KeyError: pass - # TODO: Make async version of check_locale and call it from - # EbuildPhase instead of config.environ(), since it's bad to - # synchronously wait for the process in the main event loop - # thread where config.environ() tends to be called. proc = multiprocessing.Process( target=_set_and_check_locale, args=(silent, env, None if env is None else portage._native_string(mylocale)), ) proc.start() - proc.join() + proc = portage.process.MultiprocessingProcess(proc) + await proc.wait() pyret = None - if proc.exitcode >= 0: - ret = proc.exitcode + if proc.returncode >= 0: + ret = proc.returncode if ret != 2: pyret = ret == 0 @@ -157,13 +158,22 @@ def check_locale(silent=False, env=None): return pyret +async_check_locale.__doc__ = check_locale.__doc__ +async_check_locale.__doc__ += """ + This function is a coroutine. +""" + + def split_LC_ALL(env): """ Replace LC_ALL with split-up LC_* variables if it is defined. Works on the passed environment (or settings instance). """ lc_all = env.get("LC_ALL") - if lc_all is not None: + if lc_all: for c in locale_categories: env[c] = lc_all - del env["LC_ALL"] + # Set empty so that config.reset() can restore LC_ALL state, + # since del can permanently delete variables which are not + # stored in the config's backupenv. + env["LC_ALL"] = "" -- 2.41.0