* [gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
@ 2021-03-07 15:17 Zac Medico
0 siblings, 0 replies; 4+ messages in thread
From: Zac Medico @ 2021-03-07 15:17 UTC (permalink / raw
To: gentoo-commits
commit: 77e545e0955930606eb7e73d3970d8c0706fd8a8
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 7 14:55:49 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar 7 15:12:42 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=77e545e0
EbuildPhase: Use async and await syntax
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/_emerge/EbuildPhase.py | 28 ++++++++++++---------------
lib/portage/util/_async/SchedulerInterface.py | 10 ++++------
2 files changed, 16 insertions(+), 22 deletions(-)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index e4c0428a6..26c770d29 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -21,7 +21,6 @@ from portage.util._dyn_libs.soname_deps_qa import (
)
from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir,
_prepare_fake_distdir, _prepare_fake_filesdir)
-from portage.util.futures.compat_coroutine import coroutine
from portage.util import writemsg
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
@@ -70,11 +69,10 @@ class EbuildPhase(CompositeTask):
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
def _start(self):
- future = asyncio.ensure_future(self._async_start(loop=self.scheduler), loop=self.scheduler)
+ future = asyncio.ensure_future(self._async_start(), loop=self.scheduler)
self._start_task(AsyncTaskFuture(future=future), self._async_start_exit)
- @coroutine
- def _async_start(self, loop=None):
+ async def _async_start(self):
need_builddir = self.phase not in EbuildProcess._phases_without_builddir
@@ -132,7 +130,7 @@ class EbuildPhase(CompositeTask):
# Force background=True for this header since it's intended
# for the log and it doesn't necessarily need to be visible
# elsewhere.
- yield self._elog('einfo', msg, background=True, loop=self.scheduler)
+ await self._elog('einfo', msg, background=True)
if self.phase == 'package':
if 'PORTAGE_BINPKG_TMPFILE' not in self.settings:
@@ -402,8 +400,7 @@ class EbuildPhase(CompositeTask):
self.returncode = 1
self.wait()
- @coroutine
- def _elog(self, elog_funcname, lines, background=None, loop=None):
+ async def _elog(self, elog_funcname, lines, background=None):
if background is None:
background = self.background
out = io.StringIO()
@@ -434,12 +431,12 @@ class EbuildPhase(CompositeTask):
_set_nonblocking(build_logger.stdin.fileno())
log_file = build_logger.stdin
- yield self.scheduler.async_output(msg, log_file=log_file,
- background=background, loop=self.scheduler)
+ await self.scheduler.async_output(msg, log_file=log_file,
+ background=background)
if build_logger is not None:
build_logger.stdin.close()
- yield build_logger.async_wait()
+ await build_logger.async_wait()
except asyncio.CancelledError:
if build_logger is not None:
build_logger.cancel()
@@ -487,7 +484,7 @@ class _PostPhaseCommands(CompositeTask):
if 'qa-unresolved-soname-deps' in self.settings.features:
# This operates on REQUIRES metadata generated by the above function call.
- future = asyncio.ensure_future(self._soname_deps_qa(loop=self.scheduler), loop=self.scheduler)
+ future = asyncio.ensure_future(self._soname_deps_qa(), loop=self.scheduler)
# If an unexpected exception occurs, then this will raise it.
future.add_done_callback(lambda future: future.cancelled() or future.result())
self._start_task(AsyncTaskFuture(future=future), self._default_final_exit)
@@ -496,12 +493,11 @@ class _PostPhaseCommands(CompositeTask):
else:
self._default_final_exit(task)
- @coroutine
- def _soname_deps_qa(self, loop=None):
+ async def _soname_deps_qa(self):
vardb = QueryCommand.get_db()[self.settings['EROOT']]['vartree'].dbapi
- all_provides = (yield self.scheduler.run_in_executor(ForkExecutor(loop=self.scheduler), _get_all_provides, vardb))
+ all_provides = (await self.scheduler.run_in_executor(ForkExecutor(loop=self.scheduler), _get_all_provides, vardb))
unresolved = _get_unresolved_soname_deps(os.path.join(self.settings['PORTAGE_BUILDDIR'], 'build-info'), all_provides)
@@ -512,4 +508,4 @@ class _PostPhaseCommands(CompositeTask):
qa_msg.extend("\t%s: %s" % (filename, " ".join(sorted(soname_deps)))
for filename, soname_deps in unresolved)
qa_msg.append("")
- yield self.elog("eqawarn", qa_msg, loop=self.scheduler)
+ await self.elog("eqawarn", qa_msg)
diff --git a/lib/portage/util/_async/SchedulerInterface.py b/lib/portage/util/_async/SchedulerInterface.py
index 2865266eb..c2d1be51f 100644
--- a/lib/portage/util/_async/SchedulerInterface.py
+++ b/lib/portage/util/_async/SchedulerInterface.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2020 Gentoo Authors
+# Copyright 2012-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import gzip
@@ -8,7 +8,6 @@ from portage import _encodings
from portage import _unicode_encode
from portage.util import writemsg_level
from portage.util.futures._asyncio.streams import _writer
-from portage.util.futures.compat_coroutine import coroutine
from ..SlotObject import SlotObject
class SchedulerInterface(SlotObject):
@@ -55,9 +54,8 @@ class SchedulerInterface(SlotObject):
def _return_false():
return False
- @coroutine
- def async_output(self, msg, log_file=None, background=None,
- level=0, noiselevel=-1, loop=None):
+ async def async_output(self, msg, log_file=None, background=None,
+ level=0, noiselevel=-1):
"""
Output a msg to stdio (if not in background) and to a log file
if provided.
@@ -81,7 +79,7 @@ class SchedulerInterface(SlotObject):
writemsg_level(msg, level=level, noiselevel=noiselevel)
if log_file is not None:
- yield _writer(log_file, _unicode_encode(msg), loop=loop)
+ await _writer(log_file, _unicode_encode(msg))
def output(self, msg, log_path=None, background=None,
level=0, noiselevel=-1):
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
@ 2021-09-21 5:51 Zac Medico
0 siblings, 0 replies; 4+ messages in thread
From: Zac Medico @ 2021-09-21 5:51 UTC (permalink / raw
To: gentoo-commits
commit: 5d61d0f8d72b27213896f052ef4abb2337b922bf
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 21 04:53:52 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 21 05:33:50 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=5d61d0f8
Binpkg: convert compat coroutine to async
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/_emerge/Binpkg.py | 10 ++++------
lib/portage/util/_async/AsyncTaskFuture.py | 4 +++-
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index 0f37063f0..c7dde69bd 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2019 Gentoo Authors
+# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -15,7 +15,6 @@ from _emerge.SpawnProcess import SpawnProcess
from portage.eapi import eapi_exports_replace_vars
from portage.util import ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
-from portage.util.futures.compat_coroutine import coroutine
import portage
from portage import os
from portage import shutil
@@ -305,8 +304,7 @@ class Binpkg(CompositeTask):
self._unpack_metadata_exit,
)
- @coroutine
- def _unpack_metadata(self, loop=None):
+ async def _unpack_metadata(self, loop=None):
dir_path = self.settings["PORTAGE_BUILDDIR"]
@@ -327,7 +325,7 @@ class Binpkg(CompositeTask):
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
- yield self._bintree.dbapi.unpack_metadata(
+ await self._bintree.dbapi.unpack_metadata(
self.settings, infloc, loop=self.scheduler
)
check_missing_metadata = ("CATEGORY", "PF")
@@ -378,7 +376,7 @@ class Binpkg(CompositeTask):
background=self.background, scheduler=self.scheduler, settings=self.settings
)
env_extractor.start()
- yield env_extractor.async_wait()
+ await env_extractor.async_wait()
if env_extractor.returncode != os.EX_OK:
raise portage.exception.PortageException(
"failed to extract environment for {}".format(self.pkg.cpv)
diff --git a/lib/portage/util/_async/AsyncTaskFuture.py b/lib/portage/util/_async/AsyncTaskFuture.py
index f87a7f90a..0cd034c97 100644
--- a/lib/portage/util/_async/AsyncTaskFuture.py
+++ b/lib/portage/util/_async/AsyncTaskFuture.py
@@ -1,10 +1,11 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2021 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import os
import signal
from _emerge.AsynchronousTask import AsynchronousTask
+from portage.util.futures import asyncio
class AsyncTaskFuture(AsynchronousTask):
@@ -16,6 +17,7 @@ class AsyncTaskFuture(AsynchronousTask):
__slots__ = ("future",)
def _start(self):
+ self.future = asyncio.ensure_future(self.future, self.scheduler)
self.future.add_done_callback(self._done_callback)
def _cancel(self):
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
@ 2024-02-06 2:24 Zac Medico
0 siblings, 0 replies; 4+ messages in thread
From: Zac Medico @ 2024-02-06 2:24 UTC (permalink / raw
To: gentoo-commits
commit: e8b31c86eaed645a8740fb2844e2935aee161e43
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 5 05:55:11 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 6 01:30:21 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=e8b31c86
ForkProcess: Prevent redundant pipe and set_term_size recursion
When process.spawn is updated to call ForkProcess
for bug 916566, it needs to avoid recursion via
set_term_size.
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/923750
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/_emerge/SpawnProcess.py | 31 ++++++++++++++++++++++++-------
lib/portage/util/_async/ForkProcess.py | 28 +++++++++++++++++++++++-----
2 files changed, 47 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 7f4a23892b..716e94d665 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -41,7 +41,7 @@ class SpawnProcess(SubProcess):
)
__slots__ = (
- ("args", "log_filter_file")
+ ("args", "create_pipe", "log_filter_file")
+ _spawn_kwarg_names
+ (
"_main_task",
@@ -60,15 +60,30 @@ class SpawnProcess(SubProcess):
else:
self.fd_pipes = self.fd_pipes.copy()
fd_pipes = self.fd_pipes
+ log_file_path = None
if fd_pipes or self.logfile or not self.background:
- master_fd, slave_fd = self._pipe(fd_pipes)
+ if self.create_pipe is not False:
+ master_fd, slave_fd = self._pipe(fd_pipes)
- can_log = self._can_log(slave_fd)
- if can_log:
- log_file_path = self.logfile
+ can_log = self._can_log(slave_fd)
+ if can_log:
+ log_file_path = self.logfile
else:
- log_file_path = None
+ if self.logfile:
+ raise NotImplementedError(
+ "logfile conflicts with create_pipe=False"
+ )
+ # When called via process.spawn and ForkProcess._start,
+ # SpawnProcess will have created a pipe earlier, so it
+ # would be redundant to do it here (it could also trigger
+ # spawn recursion via set_term_size as in bug 923750).
+ # Use /dev/null for master_fd, triggering early return
+ # of _main, followed by _async_waitpid.
+ # TODO: Optimize away the need for master_fd here.
+ master_fd = os.open(os.devnull, os.O_RDONLY)
+ slave_fd = None
+ can_log = False
null_input = None
if not self.background or 0 in fd_pipes:
@@ -97,7 +112,9 @@ class SpawnProcess(SubProcess):
fd_pipes_orig = fd_pipes.copy()
- if log_file_path is not None or self.background:
+ if slave_fd is None:
+ pass
+ elif log_file_path is not None or self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
diff --git a/lib/portage/util/_async/ForkProcess.py b/lib/portage/util/_async/ForkProcess.py
index 3acbe34fc6..cb240d0712 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -75,12 +75,29 @@ class ForkProcess(SpawnProcess):
self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
self.fd_pipes.setdefault(1, sys.__stdout__.fileno())
self.fd_pipes.setdefault(2, sys.__stderr__.fileno())
- stdout_fd = os.dup(self.fd_pipes[1])
+ if self.create_pipe is not False:
+ stdout_fd = os.dup(self.fd_pipes[1])
if self._HAVE_SEND_HANDLE:
- master_fd, slave_fd = self._pipe(self.fd_pipes)
- self.fd_pipes[1] = slave_fd
- self.fd_pipes[2] = slave_fd
+ if self.create_pipe is not False:
+ master_fd, slave_fd = self._pipe(self.fd_pipes)
+ self.fd_pipes[1] = slave_fd
+ self.fd_pipes[2] = slave_fd
+ else:
+ if self.logfile:
+ raise NotImplementedError(
+ "logfile conflicts with create_pipe=False"
+ )
+ # When called via process.spawn, SpawnProcess
+ # will have created a pipe earlier, so it would be
+ # redundant to do it here (it could also trigger spawn
+ # recursion via set_term_size as in bug 923750). Use
+ # /dev/null for master_fd, triggering early return
+ # of _main, followed by _async_waitpid.
+ # TODO: Optimize away the need for master_fd here.
+ master_fd = os.open(os.devnull, os.O_RDONLY)
+ slave_fd = None
+
self._files = self._files_dict(connection=connection, slave_fd=slave_fd)
# Create duplicate file descriptors in self._fd_pipes
@@ -167,7 +184,8 @@ class ForkProcess(SpawnProcess):
self._files.connection.close()
del self._files.connection
if hasattr(self._files, "slave_fd"):
- os.close(self._files.slave_fd)
+ if self._files.slave_fd is not None:
+ os.close(self._files.slave_fd)
del self._files.slave_fd
await super()._main(build_logger, pipe_logger, loop=loop)
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
@ 2024-03-04 15:31 Zac Medico
0 siblings, 0 replies; 4+ messages in thread
From: Zac Medico @ 2024-03-04 15:31 UTC (permalink / raw
To: gentoo-commits
commit: 6f9a10d38259dd61b948837e193b047464791845
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 4 05:31:08 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Mar 4 05:33:44 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=6f9a10d3
SpawnProcess: Optimize away null input for create_pipe=False
When create_pipe=False support was added in commit e8b31c86eaed,
a null input file descriptor was used for PipeLogger and BuildLogger
instances. Optimize this away, eliminating the unnecessary loggers.
Fixes: e8b31c86eaed ("ForkProcess: Prevent redundant pipe and set_term_size recursion")
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/_emerge/SpawnProcess.py | 51 +++++++++++++++++-----------------
lib/portage/util/_async/ForkProcess.py | 7 ++---
2 files changed, 28 insertions(+), 30 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 9fc12c42e5..513a7b2fe4 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -79,10 +79,7 @@ class SpawnProcess(SubProcess):
# SpawnProcess will have created a pipe earlier, so it
# would be redundant to do it here (it could also trigger
# spawn recursion via set_term_size as in bug 923750).
- # Use /dev/null for master_fd, triggering early return
- # of _main, followed by _async_waitpid.
- # TODO: Optimize away the need for master_fd here.
- master_fd = os.open(os.devnull, os.O_RDONLY)
+ master_fd = None
slave_fd = None
can_log = False
@@ -166,23 +163,27 @@ class SpawnProcess(SubProcess):
self._registered = True
def _start_main_task(self, pr, log_file_path=None, stdout_fd=None):
- build_logger = BuildLogger(
- env=self.env,
- log_path=log_file_path,
- log_filter_file=self.log_filter_file,
- scheduler=self.scheduler,
- )
- build_logger.start()
-
- pipe_logger = PipeLogger(
- background=self.background,
- scheduler=self.scheduler,
- input_fd=pr,
- log_file_path=build_logger.stdin,
- stdout_fd=stdout_fd,
- )
-
- pipe_logger.start()
+ if pr is None:
+ build_logger = None
+ pipe_logger = None
+ else:
+ build_logger = BuildLogger(
+ env=self.env,
+ log_path=log_file_path,
+ log_filter_file=self.log_filter_file,
+ scheduler=self.scheduler,
+ )
+ build_logger.start()
+
+ pipe_logger = PipeLogger(
+ background=self.background,
+ scheduler=self.scheduler,
+ input_fd=pr,
+ log_file_path=build_logger.stdin,
+ stdout_fd=stdout_fd,
+ )
+
+ pipe_logger.start()
self._main_task_cancel = functools.partial(
self._main_cancel, build_logger, pipe_logger
@@ -198,18 +199,18 @@ class SpawnProcess(SubProcess):
await self._pty_ready
self._pty_ready = None
try:
- if pipe_logger.poll() is None:
+ if pipe_logger is not None and pipe_logger.poll() is None:
await pipe_logger.async_wait()
- if build_logger.poll() is None:
+ if build_logger is not None and build_logger.poll() is None:
await build_logger.async_wait()
except asyncio.CancelledError:
self._main_cancel(build_logger, pipe_logger)
raise
def _main_cancel(self, build_logger, pipe_logger):
- if pipe_logger.poll() is None:
+ if pipe_logger is not None and pipe_logger.poll() is None:
pipe_logger.cancel()
- if build_logger.poll() is None:
+ if build_logger is not None and build_logger.poll() is None:
build_logger.cancel()
def _main_exit(self, main_task):
diff --git a/lib/portage/util/_async/ForkProcess.py b/lib/portage/util/_async/ForkProcess.py
index ebcbd94107..e6cfdefb88 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -91,11 +91,8 @@ class ForkProcess(SpawnProcess):
# When called via process.spawn, SpawnProcess
# will have created a pipe earlier, so it would be
# redundant to do it here (it could also trigger spawn
- # recursion via set_term_size as in bug 923750). Use
- # /dev/null for master_fd, triggering early return
- # of _main, followed by _async_waitpid.
- # TODO: Optimize away the need for master_fd here.
- master_fd = os.open(os.devnull, os.O_RDONLY)
+ # recursion via set_term_size as in bug 923750).
+ master_fd = None
slave_fd = None
self._files = self._files_dict(connection=connection, slave_fd=slave_fd)
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2024-03-04 15:31 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-02-06 2:24 [gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/ Zac Medico
-- strict thread matches above, loose matches on Subject: below --
2024-03-04 15:31 Zac Medico
2021-09-21 5:51 Zac Medico
2021-03-07 15:17 Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox