[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     117eeee2de26723a90ff072b7c795b1b0766898b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct  3 10:18:46 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Oct  3 10:18:46 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=117eeee2

AsyncScheduler: source_remove in _wait

---
 pym/portage/util/_async/AsyncScheduler.py |    8 ++++++++
 1 files changed, 8 insertions(+), 0 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py
index cae45fd..130331e 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -80,6 +80,14 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 		while self._is_work_scheduled():
 			self.sched_iface.iteration()
 
+		if self._term_check_id is not None:
+			self.sched_iface.source_remove(self._term_check_id)
+			self._term_check_id = None
+
+		if self._loadavg_check_id is not None:
+			self.sched_iface.source_remove(self._loadavg_check_id)
+			self._loadavg_check_id = None
+
 		if self._error_count > 0:
 			self.returncode = 1
 		else:

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     359cf128a70f7153cb800d7a22c519ec7838f02b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct  3 17:38:45 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Oct  3 17:38:45 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=359cf128

AsyncScheduler: use _termination_check

The _terminate_tasks method is always supposed to be called via
_termination_check.

---
 pym/portage/util/_async/AsyncScheduler.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py
index 130331e..a452c0c 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -27,7 +27,7 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 
 	def _cancel(self):
 		self._terminated.set()
-		self._terminate_tasks()
+		self._termination_check()
 
 	def _terminate_tasks(self):
 		for task in list(self._running_tasks):

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     3453c38128c8af3e0178dce7d2d64d7cfb899141
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct  3 18:39:11 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Oct  3 18:39:11 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3453c381

AsyncScheduler: implement _poll

---
 pym/portage/util/_async/AsyncScheduler.py |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py
index a452c0c..3fb079c 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -25,6 +25,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 		self._term_check_id = None
 		self._loadavg_check_id = None
 
+	def _poll(self):
+		if not (self._is_work_scheduled() or self._keep_scheduling()):
+			self.wait()
+		return self.returncode
+
 	def _cancel(self):
 		self._terminated.set()
 		self._termination_check()

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     5de3c48a4221ec1f4a044a2e424b113770db5386
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct  3 18:55:50 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Oct  3 18:55:50 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5de3c48a

AsyncScheduler: poll in _schedule_tasks

Triggers cleanup and exit listeners if there's nothing left to do.

---
 pym/portage/util/_async/AsyncScheduler.py |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py
index 3fb079c..f6285b7 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -59,6 +59,9 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 				task.addExitListener(self._task_exit)
 				task.start()
 
+		# Triggers cleanup and exit listeners if there's nothing left to do.
+		self.poll()
+
 	def _task_exit(self, task):
 		self._running_tasks.discard(task)
 		if task.returncode != os.EX_OK:

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     ed13db998ed501c710a5417e451aebed1eb24f6c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Oct  6 01:54:16 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Oct  6 01:54:16 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ed13db99

AsyncScheduler: remove useless __slots__

It's not allowed for AsyncScheduler to inherit from two classes that
define __slots__, so it useless to define __slots__ for AsyncScheduler
and subclasses.

---
 pym/portage/util/_async/AsyncScheduler.py |    4 ----
 pym/portage/util/_async/TaskScheduler.py  |    2 --
 2 files changed, 0 insertions(+), 6 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py
index f6285b7..0648220 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -7,10 +7,6 @@ from _emerge.PollScheduler import PollScheduler
 
 class AsyncScheduler(AsynchronousTask, PollScheduler):
 
-	__slots__ = ('_error_count', '_loadavg_check_id',
-		'_max_jobs', '_max_load',
-		'_remaining_tasks', '_running_tasks', '_term_check_id')
-
 	def __init__(self, max_jobs=None, max_load=None, **kwargs):
 		AsynchronousTask.__init__(self)
 		PollScheduler.__init__(self, **kwargs)

diff --git a/pym/portage/util/_async/TaskScheduler.py b/pym/portage/util/_async/TaskScheduler.py
index b0ec7af..35b3875 100644
--- a/pym/portage/util/_async/TaskScheduler.py
+++ b/pym/portage/util/_async/TaskScheduler.py
@@ -5,8 +5,6 @@ from .AsyncScheduler import AsyncScheduler
 
 class TaskScheduler(AsyncScheduler):
 
-	__slots__ = ('_task_iter',)
-
 	"""
 	A simple way to handle scheduling of AbstractPollTask instances. Simply
 	pass a task iterator into the constructor and call start(). Use the

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     70f580e22e82dc26831cd759c56501c72a8db00f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Oct  8 13:48:55 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Oct  8 13:48:55 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=70f580e2

ForkProcess: increase scope of try/finally/_exit

This minimizes the probability of triggering irrelevant finally blocks
from earlier in the call stack (bug #345289).

---
 pym/portage/util/_async/ForkProcess.py |   20 +++++++++++---------
 1 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/pym/portage/util/_async/ForkProcess.py b/pym/portage/util/_async/ForkProcess.py
index 6fcd662..96ce3d9 100644
--- a/pym/portage/util/_async/ForkProcess.py
+++ b/pym/portage/util/_async/ForkProcess.py
@@ -25,17 +25,19 @@ class ForkProcess(SpawnProcess):
 			portage.process.spawned_pids.append(pid)
 			return [pid]
 
-		portage.locks._close_fds()
-		# Disable close_fds since we don't exec (see _setup_pipes docstring).
-		portage.process._setup_pipes(fd_pipes, close_fds=False)
-
-		# Use default signal handlers in order to avoid problems
-		# killing subprocesses as reported in bug #353239.
-		signal.signal(signal.SIGINT, signal.SIG_DFL)
-		signal.signal(signal.SIGTERM, signal.SIG_DFL)
-
 		rval = 1
 		try:
+
+			# Use default signal handlers in order to avoid problems
+			# killing subprocesses as reported in bug #353239.
+			signal.signal(signal.SIGINT, signal.SIG_DFL)
+			signal.signal(signal.SIGTERM, signal.SIG_DFL)
+
+			portage.locks._close_fds()
+			# We don't exec, so use close_fds=False
+			# (see _setup_pipes docstring).
+			portage.process._setup_pipes(fd_pipes, close_fds=False)
+
 			rval = self._run()
 		except SystemExit:
 			raise

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     eeef84317758b01ae2c1ba8f343343179aebc01e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 16 06:51:35 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct 16 06:51:35 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=eeef8431

Add missing PopenProcess class from last commit.

---
 pym/portage/util/_async/PopenProcess.py |   15 +++++++++++++++
 1 files changed, 15 insertions(+), 0 deletions(-)

diff --git a/pym/portage/util/_async/PopenProcess.py b/pym/portage/util/_async/PopenProcess.py
new file mode 100644
index 0000000..7c42c85
--- /dev/null
+++ b/pym/portage/util/_async/PopenProcess.py
@@ -0,0 +1,15 @@
+# Copyright 2012 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from _emerge.SubProcess import SubProcess
+
+class PopenProcess(SubProcess):
+
+	__slots__ = ("proc",)
+
+	def __init__(self, **kwargs):
+		SubProcess.__init__(self, **kwargs)
+		self.pid = self.proc.pid
+
+	def _start(self):
+		pass

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     b0d85ce135496bb241d2d5714ac330710dfa0b72
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 18 01:23:11 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 18 01:23:11 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b0d85ce1

PopenProcess: child_watch_add if no pipe_reader

---
 pym/portage/util/_async/PopenProcess.py |   10 ++++++----
 1 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/_async/PopenProcess.py b/pym/portage/util/_async/PopenProcess.py
index 37d32f2..2fc56d2 100644
--- a/pym/portage/util/_async/PopenProcess.py
+++ b/pym/portage/util/_async/PopenProcess.py
@@ -7,13 +7,15 @@ class PopenProcess(SubProcess):
 
 	__slots__ = ("pipe_reader", "proc",)
 
-	def __init__(self, **kwargs):
-		SubProcess.__init__(self, **kwargs)
+	def _start(self):
+
 		self.pid = self.proc.pid
 		self._registered = True
 
-	def _start(self):
-		if self.pipe_reader is not None:
+		if self.pipe_reader is None:
+			self._reg_id = self.scheduler.child_watch_add(
+				self.pid, self._child_watch_cb)
+		else:
 			try:
 				self.pipe_reader.scheduler = self.scheduler
 			except AttributeError:

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     c7b3b1fbffb23f30c07fe57946221b62c54c1af9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 29 07:18:47 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Dec 29 07:30:07 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c7b3b1fb

Add run_main_scheduler helper function.

---
 pym/portage/util/_async/run_main_scheduler.py |   41 +++++++++++++++++++++++++
 1 files changed, 41 insertions(+), 0 deletions(-)

diff --git a/pym/portage/util/_async/run_main_scheduler.py b/pym/portage/util/_async/run_main_scheduler.py
new file mode 100644
index 0000000..10fed34
--- /dev/null
+++ b/pym/portage/util/_async/run_main_scheduler.py
@@ -0,0 +1,41 @@
+
+import signal
+
+def run_main_scheduler(scheduler):
+	"""
+	Start and run an AsyncScheduler (or compatible object), and handle
+	SIGINT or SIGTERM by calling its terminate() method and waiting
+	for it to clean up after itself. If SIGINT or SIGTERM is received,
+	return signum, else return None. Any previous SIGINT or SIGTERM
+	signal handlers are automatically saved and restored before
+	returning.
+	"""
+
+	received_signal = []
+
+	def sighandler(signum, frame):
+		signal.signal(signal.SIGINT, signal.SIG_IGN)
+		signal.signal(signal.SIGTERM, signal.SIG_IGN)
+		received_signal.append(signum)
+		scheduler.terminate()
+
+	earlier_sigint_handler = signal.signal(signal.SIGINT, sighandler)
+	earlier_sigterm_handler = signal.signal(signal.SIGTERM, sighandler)
+
+	try:
+		scheduler.start()
+		scheduler.wait()
+	finally:
+		# Restore previous handlers
+		if earlier_sigint_handler is not None:
+			signal.signal(signal.SIGINT, earlier_sigint_handler)
+		else:
+			signal.signal(signal.SIGINT, signal.SIG_DFL)
+		if earlier_sigterm_handler is not None:
+			signal.signal(signal.SIGTERM, earlier_sigterm_handler)
+		else:
+			signal.signal(signal.SIGTERM, signal.SIG_DFL)
+
+	if received_signal:
+		return received_signal[0]
+	return None

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     730572c343879be1c96b884c39ebb47b03d7ad70
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jan  5 03:40:45 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jan  5 03:40:45 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=730572c3

SchedulerInterface: use loop for proxied attrs

---
 pym/portage/util/_async/SchedulerInterface.py |   25 +++++++++----------------
 1 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py
index 731f521..2ab668e 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -1,4 +1,4 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import gzip
@@ -11,9 +11,12 @@ from ..SlotObject import SlotObject
 
 class SchedulerInterface(SlotObject):
 
-	__slots__ = ("IO_ERR", "IO_HUP", "IO_IN", "IO_NVAL", "IO_OUT", "IO_PRI",
-		"child_watch_add", "idle_add", "io_add_watch", "iteration",
-		"source_remove", "timeout_add", "_event_loop", "_is_background")
+	_event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN",
+		"IO_NVAL", "IO_OUT", "IO_PRI",
+		"child_watch_add", "idle_add", "io_add_watch",
+		"iteration", "source_remove", "timeout_add")
+
+	__slots__ = _event_loop_attrs + ("_event_loop", "_is_background")
 
 	def __init__(self, event_loop, is_background=None, **kwargs):
 		SlotObject.__init__(self, **kwargs)
@@ -21,18 +24,8 @@ class SchedulerInterface(SlotObject):
 		if is_background is None:
 			is_background = self._return_false
 		self._is_background = is_background
-		self.IO_ERR = event_loop.IO_ERR
-		self.IO_HUP = event_loop.IO_HUP
-		self.IO_IN = event_loop.IO_IN
-		self.IO_NVAL = event_loop.IO_NVAL
-		self.IO_OUT = event_loop.IO_OUT
-		self.IO_PRI = event_loop.IO_PRI
-		self.child_watch_add = event_loop.child_watch_add
-		self.idle_add = event_loop.idle_add
-		self.io_add_watch = event_loop.io_add_watch
-		self.iteration = event_loop.iteration
-		self.source_remove = event_loop.source_remove
-		self.timeout_add = event_loop.timeout_add
+		for k in self._event_loop_attrs:
+			setattr(self, k, getattr(event_loop, k))
 
 	@staticmethod
 	def _return_false():

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     99ca26fb15414379ae3a70cc446a225f2b8d532b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jan  5 15:18:05 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jan  5 15:18:05 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=99ca26fb

Add async FileDigester class.

---
 pym/portage/util/_async/FileDigester.py |   73 +++++++++++++++++++++++++++++++
 1 files changed, 73 insertions(+), 0 deletions(-)

diff --git a/pym/portage/util/_async/FileDigester.py b/pym/portage/util/_async/FileDigester.py
new file mode 100644
index 0000000..881c692
--- /dev/null
+++ b/pym/portage/util/_async/FileDigester.py
@@ -0,0 +1,73 @@
+# Copyright 2013 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage import os
+from portage.checksum import perform_multiple_checksums
+from portage.util._async.ForkProcess import ForkProcess
+from _emerge.PipeReader import PipeReader
+
+class FileDigester(ForkProcess):
+	"""
+	Asynchronously generate file digests. Pass in file_path and
+	hash_names, and after successful execution, the digests
+	attribute will be a dict containing all of the requested
+	digests.
+	"""
+
+	__slots__ = ('file_path', 'digests', 'hash_names',
+		'_digest_pipe_reader', '_digest_pw')
+
+	def _start(self):
+		pr, pw = os.pipe()
+		self.fd_pipes = {}
+		self.fd_pipes[pw] = pw
+		self._digest_pw = pw
+		self._digest_pipe_reader = PipeReader(
+			input_files={"input":pr},
+			scheduler=self.scheduler)
+		self._digest_pipe_reader.addExitListener(self._digest_pipe_reader_exit)
+		self._digest_pipe_reader.start()
+		ForkProcess._start(self)
+		os.close(pw)
+
+	def _run(self):
+		digests = perform_multiple_checksums(self.file_path,
+			hashes=self.hash_names)
+
+		buf = "".join("%s=%s\n" % item
+			for item in digests.items()).encode('utf_8')
+
+		while buf:
+			buf = buf[os.write(self._digest_pw, buf):]
+
+		return os.EX_OK
+
+	def _parse_digests(self, data):
+
+		digests = {}
+		for line in data.decode('utf_8').splitlines():
+			parts = line.split('=', 1)
+			if len(parts) == 2:
+				digests[parts[0]] = parts[1]
+
+		self.digests = digests
+
+	def _pipe_logger_exit(self, pipe_logger):
+		# Ignore this event, since we want to ensure that we
+		# exit only after _digest_pipe_reader has reached EOF.
+		self._pipe_logger = None
+
+	def _digest_pipe_reader_exit(self, pipe_reader):
+		self._parse_digests(pipe_reader.getvalue())
+		self._digest_pipe_reader = None
+		self._unregister()
+		self.wait()
+
+	def _unregister(self):
+		ForkProcess._unregister(self)
+
+		pipe_reader = self._digest_pipe_reader
+		if pipe_reader is not None:
+			self._digest_pipe_reader = None
+			pipe_reader.removeExitListener(self._digest_pipe_reader_exit)
+			pipe_reader.cancel()

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     1aae13adbc8648d3763e1570ddeffb2536c5d826
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul  7 02:16:56 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul  7 02:16:56 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1aae13ad

ForkProcess: flush stderr for traceback

---
 pym/portage/util/_async/ForkProcess.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_async/ForkProcess.py b/pym/portage/util/_async/ForkProcess.py
index be856d0..25f72d3 100644
--- a/pym/portage/util/_async/ForkProcess.py
+++ b/pym/portage/util/_async/ForkProcess.py
@@ -1,7 +1,8 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import signal
+import sys
 import traceback
 
 import portage
@@ -46,6 +47,8 @@ class ForkProcess(SpawnProcess):
 				raise
 			except:
 				traceback.print_exc()
+				# os._exit() skips stderr flush!
+				sys.stderr.flush()
 			finally:
 				os._exit(rval)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     855c8761f85323bbe570ae61e87c57dc8ea5ea0e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Feb  8 09:08:07 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb  8 09:10:49 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=855c8761

PopenProcess: suppress ResourceWarning subprocess "still running" (bug 608594)

Override the _set_returncode method to set the Popen.returncode
attribute, in order to suppress Python 3.6 ResourceWarnings which
erroneously report that the subprocess is still running.

X-Gentoo-Bug: 608594
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=608594

 pym/portage/util/_async/PopenProcess.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_async/PopenProcess.py b/pym/portage/util/_async/PopenProcess.py
index 2fc56d295..4344b1c9d 100644
--- a/pym/portage/util/_async/PopenProcess.py
+++ b/pym/portage/util/_async/PopenProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2017 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from _emerge.SubProcess import SubProcess
@@ -31,3 +31,10 @@ class PopenProcess(SubProcess):
 		self._reg_id = None
 		self._waitpid_cb(pid, condition)
 		self.wait()
+
+	def _set_returncode(self, wait_retval):
+		SubProcess._set_returncode(self, wait_retval)
+		if self.proc.returncode is None:
+			# Suppress warning messages like this:
+			# ResourceWarning: subprocess 1234 is still running
+			self.proc.returncode = self.returncode

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     128c13ff8bd9a1685fb844f7a83469b2271f5e0f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 19 06:19:00 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Apr 19 06:25:32 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=128c13ff

AsyncTaskFuture: override _wait method

If the _wait method is called before the future is done, then wait for
it to complete.

 pym/portage/util/_async/AsyncTaskFuture.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pym/portage/util/_async/AsyncTaskFuture.py b/pym/portage/util/_async/AsyncTaskFuture.py
index ee39183fe..c2316deeb 100644
--- a/pym/portage/util/_async/AsyncTaskFuture.py
+++ b/pym/portage/util/_async/AsyncTaskFuture.py
@@ -29,3 +29,8 @@ class AsyncTaskFuture(AsynchronousTask):
 		else:
 			self.returncode = 1
 		self.wait()
+
+	def _wait(self):
+		if self.returncode is None:
+			self.scheduler.run_until_complete(self.future)
+		return self.returncode

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     d836b0d32797f1431ebeaa20e2a18ea5cb7c3f7c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 22 15:56:29 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 22 17:48:52 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d836b0d3

AsyncFunction: use _async_waitpid (bug 591760)

When pid exit status is not yet available, use the Subprocess
_async_waitpid() method to avoid event loop recursion.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/AsyncFunction.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/AsyncFunction.py b/pym/portage/util/_async/AsyncFunction.py
index 40f6c5e75..ad3d8333f 100644
--- a/pym/portage/util/_async/AsyncFunction.py
+++ b/pym/portage/util/_async/AsyncFunction.py
@@ -57,8 +57,11 @@ class AsyncFunction(ForkProcess):
 			# and returned an unsuccessful returncode.
 			pass
 		self._async_func_reader = None
-		self._unregister()
-		self.wait()
+		if self.returncode is None:
+			self._async_waitpid()
+		else:
+			self._unregister()
+			self._async_wait()
 
 	def _unregister(self):
 		ForkProcess._unregister(self)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     3eb46b05eb098507ea74108559bf77006e0f8142
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 23 00:49:15 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 23 00:49:15 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3eb46b05

FileDigester: use _async_waitpid (bug 591760)

When pid exit status is not yet available, use the Subprocess
_async_waitpid() method to avoid event loop recursion.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/FileDigester.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/FileDigester.py b/pym/portage/util/_async/FileDigester.py
index 881c69280..72f06759c 100644
--- a/pym/portage/util/_async/FileDigester.py
+++ b/pym/portage/util/_async/FileDigester.py
@@ -60,8 +60,11 @@ class FileDigester(ForkProcess):
 	def _digest_pipe_reader_exit(self, pipe_reader):
 		self._parse_digests(pipe_reader.getvalue())
 		self._digest_pipe_reader = None
-		self._unregister()
-		self.wait()
+		if self.pid is None:
+			self._unregister()
+			self._async_wait()
+		else:
+			self._async_waitpid()
 
 	def _unregister(self):
 		ForkProcess._unregister(self)

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     384b61dd24a21aaccd7c643e58b07ea56c3cbdfc
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 26 05:31:54 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Apr 26 05:46:30 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=384b61dd

PipeLogger._output_handler: use _async_wait() (bug 591760)

Use _async_wait() to avoid event loop recursion.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/PipeLogger.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/PipeLogger.py b/pym/portage/util/_async/PipeLogger.py
index aa605d94d..ed1202edf 100644
--- a/pym/portage/util/_async/PipeLogger.py
+++ b/pym/portage/util/_async/PipeLogger.py
@@ -1,4 +1,4 @@
-# Copyright 2008-2013 Gentoo Foundation
+# Copyright 2008-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import fcntl
@@ -89,7 +89,8 @@ class PipeLogger(AbstractPollTask):
 			if not buf:
 				# EOF
 				self._unregister()
-				self.wait()
+				self.returncode = self.returncode or os.EX_OK
+				self._async_wait()
 				break
 
 			else:

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     d16206ccd315edbd4bd7a46c0b1aa9e59a46db43
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 26 07:45:32 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Apr 26 08:02:28 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d16206cc

AsyncScheduler._poll(): fix event loop recursion (bug 591760)

Call self._cleanup() and set the returncode, in order to avoid
event loop recursion in self.wait().

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/AsyncScheduler.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py
index b89b57dab..1b870c771 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -29,7 +29,12 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 
 	def _poll(self):
 		if not (self._is_work_scheduled() or self._keep_scheduling()):
-			self.wait()
+			self._cleanup()
+
+			if self._error_count > 0:
+				self.returncode = 1
+			else:
+				self.returncode = os.EX_OK
 		return self.returncode
 
 	def _cancel(self):

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     a65d93bdeb4d964603cd0ce9b0a75a571c9bdefa
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 26 09:19:22 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Apr 26 09:31:09 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a65d93bd

SchedulerInterface: expose AbstractEventLoop methods (bug 591760)

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/SchedulerInterface.py | 34 +++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py
index 21420ae41..ab2a70852 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -13,10 +13,36 @@ class SchedulerInterface(SlotObject):
 
 	_event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN",
 		"IO_NVAL", "IO_OUT", "IO_PRI",
-		"call_soon", "call_soon_threadsafe",
-		"child_watch_add", "create_future",
-		"idle_add", "io_add_watch", "iteration", "run_until_complete",
-		"source_remove", "timeout_add")
+		"child_watch_add",
+		"idle_add",
+		"io_add_watch",
+		"iteration",
+		"source_remove",
+		"timeout_add",
+
+		"add_reader",
+		"add_writer",
+		"call_at",
+		"call_exception_handler",
+		"call_later",
+		"call_soon",
+		"call_soon_threadsafe",
+		"close",
+		"create_future",
+		"default_exception_handler",
+		"get_debug",
+		"is_closed",
+		"is_running",
+		"remove_reader",
+		"remove_writer",
+		"run_in_executor",
+		"run_until_complete",
+		"set_debug",
+		"time",
+
+		"_asyncio_child_watcher",
+		"_asyncio_wrapper",
+	)
 
 	__slots__ = _event_loop_attrs + ("_event_loop", "_is_background")
 

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     3d81af6440cfd20cd007568779d6129083f1de24
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 28 22:45:56 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 28 22:45:56 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d81af64

AsyncScheduler: fix deprecated _wait usage (bug 653856)

Override AsynchronousTask._async_wait() for cleanup, since
AsynchronousTask._wait() is deprecated.

Bug: https://bugs.gentoo.org/653856

 pym/portage/util/_async/AsyncScheduler.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py
index 1b870c771..2ddbaa885 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -29,12 +29,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 
 	def _poll(self):
 		if not (self._is_work_scheduled() or self._keep_scheduling()):
-			self._cleanup()
-
 			if self._error_count > 0:
 				self.returncode = 1
 			else:
 				self.returncode = os.EX_OK
+			self._async_wait()
 		return self.returncode
 
 	def _cancel(self):
@@ -91,7 +90,17 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 			self._event_loop.source_remove(self._loadavg_check_id)
 			self._loadavg_check_id = None
 
+	def _async_wait(self):
+		"""
+		Override _async_wait to call self._cleanup().
+		"""
+		self._cleanup()
+		super(AsyncScheduler, self)._async_wait()
+
 	def _wait(self):
+		"""
+		Deprecated. Use _async_wait() instead.
+		"""
 		# Loop while there are jobs to be scheduled.
 		while self._keep_scheduling():
 			self._event_loop.iteration()

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     98e5852117cbda19b4f4266eb71a053dced64a88
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 28 23:17:30 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 28 23:17:30 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=98e58521

AsyncTaskFuture: fix deprecated _wait usage (bug 653856)

Fix _done_callback to use _async_wait() instead of wait().

Bug: https://bugs.gentoo.org/653856

 pym/portage/util/_async/AsyncTaskFuture.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/_async/AsyncTaskFuture.py b/pym/portage/util/_async/AsyncTaskFuture.py
index ce881ba8d..cdc842b05 100644
--- a/pym/portage/util/_async/AsyncTaskFuture.py
+++ b/pym/portage/util/_async/AsyncTaskFuture.py
@@ -28,9 +28,12 @@ class AsyncTaskFuture(AsynchronousTask):
 			self.returncode = os.EX_OK
 		else:
 			self.returncode = 1
-		self.wait()
+		self._async_wait()
 
 	def _wait(self):
+		"""
+		Deprecated. Use _async_wait() instead.
+		"""
 		if self.returncode is None:
 			self.scheduler.run_until_complete(self.future)
 		return self.returncode

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     532366feaf8e3cb04e0d19d90498cfe0b9361ab1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 28 23:27:49 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 28 23:27:49 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=532366fe

PipeReaderBlockingIO: fix deprecated _wait usage (bug 653856)

Use _async_wait() instead of wait().

Bug: https://bugs.gentoo.org/653856

 pym/portage/util/_async/PipeReaderBlockingIO.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/pym/portage/util/_async/PipeReaderBlockingIO.py b/pym/portage/util/_async/PipeReaderBlockingIO.py
index b06adf6ed..b426f7695 100644
--- a/pym/portage/util/_async/PipeReaderBlockingIO.py
+++ b/pym/portage/util/_async/PipeReaderBlockingIO.py
@@ -1,4 +1,4 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 try:
@@ -63,7 +63,7 @@ class PipeReaderBlockingIO(AbstractPollTask):
 		self._registered = False
 		if self.returncode is None:
 			self.returncode = os.EX_OK
-		self.wait()
+		self._async_wait()
 		return False
 
 	def _cancel(self):
@@ -71,9 +71,12 @@ class PipeReaderBlockingIO(AbstractPollTask):
 		self._registered = False
 		if self.returncode is None:
 			self.returncode = self._cancelled_returncode
-		self.wait()
+		self._async_wait()
 
 	def _wait(self):
+		"""
+		Deprecated. Use _async_wait() instead.
+		"""
 		if self.returncode is not None:
 			return self.returncode
 		self._wait_loop()

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     2ab6eed08c9eafa29de6100b19c9c072b17b9167
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 29 04:06:25 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 29 04:06:25 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2ab6eed0

PipeReaderBlockingIO._reader_thread: asyncio compat (bug 653856)

Use call_soon_threadsafe for asyncio compatibility, and also
remove idle_add from SchedulerInterface since there are no more
consumers.

Bug: https://bugs.gentoo.org/653856

 pym/portage/util/_async/PipeReaderBlockingIO.py | 3 +--
 pym/portage/util/_async/SchedulerInterface.py   | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/pym/portage/util/_async/PipeReaderBlockingIO.py b/pym/portage/util/_async/PipeReaderBlockingIO.py
index b426f7695..0727d2c6a 100644
--- a/pym/portage/util/_async/PipeReaderBlockingIO.py
+++ b/pym/portage/util/_async/PipeReaderBlockingIO.py
@@ -55,7 +55,7 @@ class PipeReaderBlockingIO(AbstractPollTask):
 					del self._threads[f]
 					if not self._threads:
 						# Thread-safe callback to EventLoop
-						self.scheduler.idle_add(self._eof)
+						self.scheduler.call_soon_threadsafe(self._eof)
 					break
 		f.close()
 
@@ -64,7 +64,6 @@ class PipeReaderBlockingIO(AbstractPollTask):
 		if self.returncode is None:
 			self.returncode = os.EX_OK
 		self._async_wait()
-		return False
 
 	def _cancel(self):
 		self._terminate.set()

diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py
index ab2a70852..acacf7c31 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2013 Gentoo Foundation
+# Copyright 2012-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import gzip
@@ -14,7 +14,6 @@ class SchedulerInterface(SlotObject):
 	_event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN",
 		"IO_NVAL", "IO_OUT", "IO_PRI",
 		"child_watch_add",
-		"idle_add",
 		"io_add_watch",
 		"iteration",
 		"source_remove",

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     be800bf0153a28ce034277d103a2021f93ac8b2e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 29 05:10:27 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 29 05:58:58 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=be800bf0

PopenProcess: add_child_handler asyncio compat (bug 591760)

Migrate to asyncio's AbstractChildWatcher.add_child_handler
interface.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/PopenProcess.py | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/pym/portage/util/_async/PopenProcess.py b/pym/portage/util/_async/PopenProcess.py
index 4344b1c9d..3fb60d527 100644
--- a/pym/portage/util/_async/PopenProcess.py
+++ b/pym/portage/util/_async/PopenProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2017 Gentoo Foundation
+# Copyright 2012-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from _emerge.SubProcess import SubProcess
@@ -13,8 +13,7 @@ class PopenProcess(SubProcess):
 		self._registered = True
 
 		if self.pipe_reader is None:
-			self._reg_id = self.scheduler.child_watch_add(
-				self.pid, self._child_watch_cb)
+			self._async_waitpid()
 		else:
 			try:
 				self.pipe_reader.scheduler = self.scheduler
@@ -24,17 +23,25 @@ class PopenProcess(SubProcess):
 			self.pipe_reader.start()
 
 	def _pipe_reader_exit(self, pipe_reader):
-		self._reg_id = self.scheduler.child_watch_add(
-			self.pid, self._child_watch_cb)
+		self._async_waitpid()
 
-	def _child_watch_cb(self, pid, condition, user_data=None):
-		self._reg_id = None
-		self._waitpid_cb(pid, condition)
-		self.wait()
+	def _async_waitpid(self):
+		if self.returncode is None:
+			self.scheduler._asyncio_child_watcher.\
+				add_child_handler(self.pid, self._async_waitpid_cb)
+		else:
+			self._unregister()
+			self._async_wait()
 
-	def _set_returncode(self, wait_retval):
-		SubProcess._set_returncode(self, wait_retval)
+	def _async_waitpid_cb(self, pid, returncode):
 		if self.proc.returncode is None:
 			# Suppress warning messages like this:
 			# ResourceWarning: subprocess 1234 is still running
-			self.proc.returncode = self.returncode
+			self.proc.returncode = returncode
+		self._unregister()
+		self.returncode = returncode
+		self._async_wait()
+
+	def _poll(self):
+		# Simply rely on _async_waitpid_cb to set the returncode.
+		return self.returncode

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     abf4b7abcdc53bbbd86eb6f79d45417b54ed64d2
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 29 22:12:13 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 29 22:18:28 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=abf4b7ab

AsyncScheduler: call_later asyncio compat (bug 591760)

Use call_later for asyncio compatibility. Also remove the
timeout_add method from SchedulerInterface, since there are
no more consumers.

Bug: https://bugs.gentoo.org/591760

 pym/portage/util/_async/AsyncScheduler.py     | 9 +++++++--
 pym/portage/util/_async/SchedulerInterface.py | 1 -
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/pym/portage/util/_async/AsyncScheduler.py b/pym/portage/util/_async/AsyncScheduler.py
index 90e803eba..c6b523eaa 100644
--- a/pym/portage/util/_async/AsyncScheduler.py
+++ b/pym/portage/util/_async/AsyncScheduler.py
@@ -65,6 +65,11 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 				task.addExitListener(self._task_exit)
 				task.start()
 
+		if self._loadavg_check_id is not None:
+			self._loadavg_check_id.cancel()
+			self._loadavg_check_id = self._event_loop.call_later(
+				self._loadavg_latency, self._schedule)
+
 		# Triggers cleanup and exit listeners if there's nothing left to do.
 		self.poll()
 
@@ -80,14 +85,14 @@ class AsyncScheduler(AsynchronousTask, PollScheduler):
 			(self._max_jobs is True or self._max_jobs > 1):
 			# We have to schedule periodically, in case the load
 			# average has changed since the last call.
-			self._loadavg_check_id = self._event_loop.timeout_add(
+			self._loadavg_check_id = self._event_loop.call_later(
 				self._loadavg_latency, self._schedule)
 		self._schedule()
 
 	def _cleanup(self):
 		super(AsyncScheduler, self)._cleanup()
 		if self._loadavg_check_id is not None:
-			self._event_loop.source_remove(self._loadavg_check_id)
+			self._loadavg_check_id.cancel()
 			self._loadavg_check_id = None
 
 	def _async_wait(self):

diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py
index ff39bc587..56b844616 100644
--- a/pym/portage/util/_async/SchedulerInterface.py
+++ b/pym/portage/util/_async/SchedulerInterface.py
@@ -15,7 +15,6 @@ class SchedulerInterface(SlotObject):
 		"IO_NVAL", "IO_OUT", "IO_PRI",
 		"io_add_watch",
 		"source_remove",
-		"timeout_add",
 
 		"add_reader",
 		"add_writer",

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     2495fe6ff060e2ed8ee54e08a4dec132de1f4984
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 30 05:52:07 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 30 06:20:01 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2495fe6f

PipeLogger: add_reader asyncio compat (bug 654382)

Use add_reader for asyncio compatibility.

Bug: https://bugs.gentoo.org/654382

 pym/portage/util/_async/PipeLogger.py | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/pym/portage/util/_async/PipeLogger.py b/pym/portage/util/_async/PipeLogger.py
index 02de74b16..61000271d 100644
--- a/pym/portage/util/_async/PipeLogger.py
+++ b/pym/portage/util/_async/PipeLogger.py
@@ -21,7 +21,7 @@ class PipeLogger(AbstractPollTask):
 	"""
 
 	__slots__ = ("input_fd", "log_file_path", "stdout_fd") + \
-		("_log_file", "_log_file_real", "_reg_id")
+		("_log_file", "_log_file_real")
 
 	def _start(self):
 
@@ -57,8 +57,7 @@ class PipeLogger(AbstractPollTask):
 				fcntl.fcntl(fd, fcntl.F_SETFD,
 					fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
 
-		self._reg_id = self.scheduler.io_add_watch(fd,
-			self._registered_events, self._output_handler)
+		self.scheduler.add_reader(fd, self._output_handler, fd)
 		self._registered = True
 
 	def _cancel(self):
@@ -66,14 +65,14 @@ class PipeLogger(AbstractPollTask):
 		if self.returncode is None:
 			self.returncode = self._cancelled_returncode
 
-	def _output_handler(self, fd, event):
+	def _output_handler(self, fd):
 
 		background = self.background
 		stdout_fd = self.stdout_fd
 		log_file = self._log_file 
 
 		while True:
-			buf = self._read_buf(fd, event)
+			buf = self._read_buf(fd, None)
 
 			if buf is None:
 				# not a POLLIN event, EAGAIN, etc...
@@ -124,20 +123,13 @@ class PipeLogger(AbstractPollTask):
 					log_file.write(buf)
 					log_file.flush()
 
-		self._unregister_if_appropriate(event)
-
-		return True
-
 	def _unregister(self):
-
-		if self._reg_id is not None:
-			self.scheduler.source_remove(self._reg_id)
-			self._reg_id = None
-
 		if self.input_fd is not None:
 			if isinstance(self.input_fd, int):
+				self.scheduler.remove_reader(self.input_fd)
 				os.close(self.input_fd)
 			else:
+				self.scheduler.remove_reader(self.input_fd.fileno())
 				self.input_fd.close()
 			self.input_fd = None
 

[gentoo-commits] proj/portage:master commit in: pym/portage/util/_async/

[Zac Medico]

commit:     b28b01a78e4cc0199111e804af5b56a33319356d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 14 15:33:15 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 14 15:35:53 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b28b01a7

ForkProcess: unregister SIGCHLD and wakeup_fd (bug 655656)

In order to prevent forked processes from invoking the parent process's
SIGCHLD handler and writing to wakeup_fd (triggering BlockingIOError),
unregister the SIGCHLD and wakeup_fd.

Bug: https://bugs.gentoo.org/655656
Reported-by: Helmut Jarausch <jarausch <AT> igpm.rwth-aachen.de>

 pym/portage/util/_async/ForkProcess.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/pym/portage/util/_async/ForkProcess.py b/pym/portage/util/_async/ForkProcess.py
index 25f72d308..d84e93833 100644
--- a/pym/portage/util/_async/ForkProcess.py
+++ b/pym/portage/util/_async/ForkProcess.py
@@ -37,6 +37,16 @@ class ForkProcess(SpawnProcess):
 				signal.signal(signal.SIGINT, signal.SIG_DFL)
 				signal.signal(signal.SIGTERM, signal.SIG_DFL)
 
+				# Unregister SIGCHLD handler and wakeup_fd for the parent
+				# process's event loop (bug 655656).
+				signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+				try:
+					wakeup_fd = signal.set_wakeup_fd(-1)
+					if wakeup_fd > 0:
+						os.close(wakeup_fd)
+				except (ValueError, OSError):
+					pass
+
 				portage.locks._close_fds()
 				# We don't exec, so use close_fds=False
 				# (see _setup_pipes docstring).