[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/

[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/

[Zac Medico]

commit:     a69c1b853a47346192950c91b088163490287350
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb  3 21:27:45 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb  7 00:49:26 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a69c1b85

process.spawn: Use multiprocessing.Process for returnproc

Use multiprocessing.Process for returnproc, so that
fork will stop being used when python makes "spawn"
the default multiprocessing start method.

Continue to use _start_fork when returnproc is not
enabled, for backward compatibility. Ultimately,
it can be removed at the same time as the returnpid
parameter.

The _setup_pipes_after_fork wrapper prevents a
"Bad file descriptor" error by making fd_pipes
inheritable on exec for bug 923755. ForkProcess
does not handle this because its target function
does not necessarily exec.

Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/923755
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/_emerge/SpawnProcess.py |  4 +-
 lib/portage/process.py      | 92 ++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 82 insertions(+), 14 deletions(-)

diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 716e94d665..b63afae01c 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -241,7 +241,9 @@ class SpawnProcess(SubProcess):
         got_pty, master_fd, slave_fd = _create_pty_or_pipe(copy_term_size=stdout_pipe)
         return (master_fd, slave_fd)
 
-    def _spawn(self, args: list[str], **kwargs) -> portage.process.Process:
+    def _spawn(
+        self, args: list[str], **kwargs
+    ) -> portage.process.MultiprocessingProcess:
         spawn_func = portage.process.spawn
 
         if self._selinux_type is not None:

diff --git a/lib/portage/process.py b/lib/portage/process.py
index b223ecb887..20327b38bc 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -19,7 +19,7 @@ import warnings
 
 from dataclasses import dataclass
 from functools import lru_cache
-from typing import Any, Optional, Callable
+from typing import Any, Optional, Callable, Union
 
 from portage import os
 from portage import _encodings
@@ -28,6 +28,7 @@ import portage
 
 portage.proxy.lazyimport.lazyimport(
     globals(),
+    "portage.util._async.ForkProcess:ForkProcess",
     "portage.util._eventloop.global_event_loop:global_event_loop",
     "portage.util.futures:asyncio",
     "portage.util:dump_traceback,writemsg,writemsg_level",
@@ -296,12 +297,19 @@ class AbstractProcess:
 
 class Process(AbstractProcess):
     """
-    An object that wraps OS processes created by spawn.
-    In the future, spawn will return objects of a different type
-    but with a compatible interface to this class, in order
-    to encapsulate implementation-dependent objects like
-    multiprocessing.Process which are designed to manage
-    the process lifecycle and need to persist until it exits.
+    An object that wraps OS processes which do not have an
+    associated multiprocessing.Process instance. Ultimately,
+    we need to stop using os.fork() to create these processes
+    because it is unsafe for threaded processes as discussed
+    in https://github.com/python/cpython/issues/84559.
+
+    Note that if subprocess.Popen is used without pass_fds
+    or preexec_fn parameters, then it avoids using os.fork()
+    by instead using posix_spawn. This approach is not used
+    by spawn because it needs to execute python code prior
+    to exec, so it instead uses multiprocessing.Process,
+    which only uses os.fork() when the multiprocessing start
+    method is fork.
     """
 
     def __init__(self, pid: int):
@@ -461,7 +469,7 @@ def spawn(
     unshare_mount=False,
     unshare_pid=False,
     warn_on_large_env=False,
-):
+) -> Union[int, MultiprocessingProcess, list[int]]:
     """
     Spawns a given command.
 
@@ -479,8 +487,8 @@ def spawn(
     @param returnpid: Return the Process IDs for a successful spawn.
     NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
     @type returnpid: Boolean
-    @param returnproc: Return a Process object for a successful spawn (conflicts with logfile parameter).
-    NOTE: This requires the caller to asynchronously wait for the Process.
+    @param returnproc: Return a MultiprocessingProcess instance (conflicts with logfile parameter).
+    NOTE: This requires the caller to asynchronously wait for the MultiprocessingProcess instance.
     @type returnproc: Boolean
     @param uid: User ID to spawn as; useful for dropping privilages
     @type uid: Integer
@@ -626,7 +634,9 @@ def spawn(
     # fork, so that the result is cached in the main process.
     bool(groups)
 
-    pid = _start_fork(
+    start_func = _start_proc if returnproc else _start_fork
+
+    pid = start_func(
         _exec_wrapper,
         args=(
             binary,
@@ -652,6 +662,10 @@ def spawn(
         close_fds=close_fds,
     )
 
+    if returnproc:
+        # _start_proc returns a MultiprocessingProcess instance.
+        return pid
+
     if not isinstance(pid, int):
         raise AssertionError(f"fork returned non-integer: {repr(pid)}")
 
@@ -673,8 +687,6 @@ def spawn(
                 stacklevel=1,
             )
         return mypids
-    if returnproc:
-        return Process(mypids[0])
 
     # Otherwise we clean them up.
     while mypids:
@@ -1373,6 +1385,60 @@ def _start_fork(
     return pid
 
 
+class _setup_pipes_after_fork:
+    def __init__(self, target, fd_pipes):
+        self._target = target
+        self._fd_pipes = fd_pipes
+
+    def __call__(self, *args, **kwargs):
+        for fd in set(self._fd_pipes.values()):
+            os.set_inheritable(fd, True)
+        _setup_pipes(self._fd_pipes, close_fds=False, inheritable=True)
+        return self._target(*args, **kwargs)
+
+
+def _start_proc(
+    target: Callable[..., None],
+    args: Optional[tuple[Any, ...]] = (),
+    kwargs: Optional[dict[str, Any]] = {},
+    fd_pipes: Optional[dict[int, int]] = None,
+    close_fds: Optional[bool] = False,
+) -> MultiprocessingProcess:
+    """
+    Execute the target function using multiprocess.Process.
+    If the close_fds parameter is True then NotImplementedError
+    is raised, since it is risky to forcefully close file
+    descriptors that have references (bug 374335), and PEP 446
+    should ensure that any relevant file descriptors are
+    non-inheritable and therefore automatically closed on exec.
+    """
+    if close_fds:
+        raise NotImplementedError(
+            "close_fds is not supported (since file descriptors are non-inheritable by default for exec)"
+        )
+
+    # Manage fd_pipes inheritance for spawn/exec (bug 923755),
+    # which ForkProcess does not handle because its target
+    # function does not necessarily exec.
+    if fd_pipes and multiprocessing.get_start_method() == "fork":
+        target = _setup_pipes_after_fork(target, fd_pipes)
+        fd_pipes = None
+
+    proc = ForkProcess(
+        scheduler=global_event_loop(),
+        target=target,
+        args=args,
+        kwargs=kwargs,
+        fd_pipes=fd_pipes,
+        create_pipe=False,  # Pipe creation is delegated to the caller (see bug 923750).
+    )
+    proc.start()
+
+    # ForkProcess conveniently holds a MultiprocessingProcess
+    # instance that is suitable to return here.
+    return proc._proc
+
+
 def find_binary(binary):
     """
     Given a binary name, find the binary in PATH

[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/

[Sam James]

commit:     7bfddc6b9ce724d141713223669490d4b46f76da
Author:     steering7253 <steering7253 <AT> proton <DOT> me>
AuthorDate: Sat Jul  5 09:56:01 2025 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Thu Sep 11 00:52:56 2025 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7bfddc6b

Hostname support in titles for all titles

Move hostname support to the xtermTitle function, instead of (some of) the places that call it.
Notably, this adds the hostname to the initial static "emerge" display when syncing or calculating dependencies.
It also removes duplicated code.

Signed-off-by: John Runyon <steering7253 <AT> proton.me>
Part-of: https://github.com/gentoo/portage/pull/1448
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/_emerge/JobStatusDisplay.py | 6 ------
 lib/_emerge/emergelog.py        | 2 --
 lib/portage/output.py           | 3 +++
 3 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/lib/_emerge/JobStatusDisplay.py b/lib/_emerge/JobStatusDisplay.py
index b98d4a8327..eefb72c49b 100644
--- a/lib/_emerge/JobStatusDisplay.py
+++ b/lib/_emerge/JobStatusDisplay.py
@@ -296,11 +296,5 @@ class JobStatusDisplay:
             self._update(color_output.getvalue())
 
         if self.xterm_titles:
-            # If the HOSTNAME variable is exported, include it
-            # in the xterm title, just like emergelog() does.
-            # See bug #390699.
             title_str = " ".join(plain_output.split())
-            hostname = os.environ.get("HOSTNAME")
-            if hostname is not None:
-                title_str = f"{hostname}: {title_str}"
             xtermTitle(title_str)

diff --git a/lib/_emerge/emergelog.py b/lib/_emerge/emergelog.py
index 30e27a8114..0b73ce98d2 100644
--- a/lib/_emerge/emergelog.py
+++ b/lib/_emerge/emergelog.py
@@ -27,8 +27,6 @@ def emergelog(xterm_titles, mystr, short_msg=None):
         short_msg = _unicode_decode(short_msg)
 
     if xterm_titles and short_msg:
-        if "HOSTNAME" in os.environ:
-            short_msg = os.environ["HOSTNAME"] + ": " + short_msg
         xtermTitle(short_msg)
     try:
         file_path = os.path.join(_emerge_log_dir, "emerge.log")

diff --git a/lib/portage/output.py b/lib/portage/output.py
index 42eb5062cf..27da869b86 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -282,6 +282,9 @@ def xtermTitle(mystr, raw=False):
         )
 
     if dotitles and not _disable_xtermTitle:
+        if "HOSTNAME" in os.environ and not raw:
+            hostname = os.environ["HOSTNAME"]
+            mystr = f"{hostname}: {mystr}"
         # If the title string is too big then the terminal can
         # misbehave. Therefore, truncate it if it's too big.
         if len(mystr) > _max_xtermTitle_len: