* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-09 7:25 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-09 7:25 UTC (permalink / raw
To: gentoo-commits
commit: e6cf0c2689b219ce894d1b18be4e514acac508a7
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 9 07:25:09 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 9 07:25:09 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e6cf0c26
EventLoop: fix inverted may_block logic
This caused emerge to consume 100% cpu.
---
pym/portage/util/_eventloop/EventLoop.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index c2ba1f2..3f4d354 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -157,9 +157,9 @@ class EventLoop(object):
if not self._poll_event_queue:
if may_block:
- timeout = 0
- else:
timeout = None
+ else:
+ timeout = 0
self._poll(timeout=timeout)
try:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-10 0:17 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-10 0:17 UTC (permalink / raw
To: gentoo-commits
commit: 2ea1c3f0326de46e81c064688004a01408b9d8b2
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 10 00:16:39 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 10 00:16:39 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2ea1c3f0
EventLoop: inherit PollConstants
---
pym/portage/util/_eventloop/EventLoop.py | 2 +-
pym/portage/util/_eventloop/GlibEventLoop.py | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index aa78ccd..21969e6 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -11,7 +11,7 @@ from _emerge.SlotObject import SlotObject
from _emerge.PollConstants import PollConstants
from _emerge.PollSelectAdapter import PollSelectAdapter
-class EventLoop(object):
+class EventLoop(PollConstants):
supports_multiprocessing = True
diff --git a/pym/portage/util/_eventloop/GlibEventLoop.py b/pym/portage/util/_eventloop/GlibEventLoop.py
index b35772e..37b78aa 100644
--- a/pym/portage/util/_eventloop/GlibEventLoop.py
+++ b/pym/portage/util/_eventloop/GlibEventLoop.py
@@ -1,12 +1,14 @@
# Copyright 2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from _emerge.PollConstants import PollConstants
+
import portage
portage.proxy.lazyimport.lazyimport(globals(),
'glib',
)
-class GlibEventLoop(object):
+class GlibEventLoop(PollConstants):
# TODO: Support multiprocessing by using a separate glib.MainContext
# instance for each process.
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-10 0:20 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-10 0:20 UTC (permalink / raw
To: gentoo-commits
commit: 067138de6e4c5ca5890969b2f0681b0b178a24ed
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 10 00:20:31 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 10 00:20:31 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=067138de
EventLoop: inherit PollConstants
---
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index aa78ccd..21969e6 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -11,7 +11,7 @@ from _emerge.SlotObject import SlotObject
from _emerge.PollConstants import PollConstants
from _emerge.PollSelectAdapter import PollSelectAdapter
-class EventLoop(object):
+class EventLoop(PollConstants):
supports_multiprocessing = True
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-10 0:42 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-10 0:42 UTC (permalink / raw
To: gentoo-commits
commit: 963fa653fc0d7aff25bb4756df8398b88193547c
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 10 00:41:14 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 10 00:41:14 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=963fa653
EventLoop: name poll constants like glib's
It's conceivable the constants could be implementation dependent, so
use glib's generic IO_* names rather than the POLL* names which are
associated specifically with python's select.poll object.
---
pym/portage/util/_eventloop/EventLoop.py | 9 ++++++++-
pym/portage/util/_eventloop/GlibEventLoop.py | 6 ++++++
2 files changed, 14 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 21969e6..02c3cec 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -11,7 +11,7 @@ from _emerge.SlotObject import SlotObject
from _emerge.PollConstants import PollConstants
from _emerge.PollSelectAdapter import PollSelectAdapter
-class EventLoop(PollConstants):
+class EventLoop(object):
supports_multiprocessing = True
@@ -37,6 +37,13 @@ class EventLoop(PollConstants):
self._poll_obj = create_poll_instance()
self._polling = False
+ self.IO_ERR = PollConstants.POLLERR
+ self.IO_HUP = PollConstants.POLLHUP
+ self.IO_IN = PollConstants.POLLIN
+ self.IO_NVAL = PollConstants.POLLNVAL
+ self.IO_OUT = PollConstants.POLLOUT
+ self.IO_PRI = PollConstants.POLLPRI
+
def _poll(self, timeout=None):
if self._polling:
return
diff --git a/pym/portage/util/_eventloop/GlibEventLoop.py b/pym/portage/util/_eventloop/GlibEventLoop.py
index b35772e..1e064eb 100644
--- a/pym/portage/util/_eventloop/GlibEventLoop.py
+++ b/pym/portage/util/_eventloop/GlibEventLoop.py
@@ -13,6 +13,12 @@ class GlibEventLoop(object):
supports_multiprocessing = False
def __init__(self):
+ self.IO_ERR = glib.IO_ERR
+ self.IO_HUP = glib.IO_HUP
+ self.IO_IN = glib.IO_IN
+ self.IO_NVAL = glib.IO_NVAL
+ self.IO_OUT = glib.IO_OUT
+ self.IO_PRI = glib.IO_PRI
self.iteration = glib.main_context_default().iteration
self.idle_add = glib.idle_add
self.io_add_watch = glib.io_add_watch
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-11 2:59 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-11 2:59 UTC (permalink / raw
To: gentoo-commits
commit: a79b320609b56ee222eca71b1bd45a203333f6d0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 11 02:55:19 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 11 02:55:19 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a79b3206
GlibEventLoop: use gi.repository.GLib
---
pym/portage/util/_eventloop/GlibEventLoop.py | 6 +-----
1 files changed, 1 insertions(+), 5 deletions(-)
diff --git a/pym/portage/util/_eventloop/GlibEventLoop.py b/pym/portage/util/_eventloop/GlibEventLoop.py
index 1e064eb..0d33175 100644
--- a/pym/portage/util/_eventloop/GlibEventLoop.py
+++ b/pym/portage/util/_eventloop/GlibEventLoop.py
@@ -1,11 +1,6 @@
# Copyright 2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-import portage
-portage.proxy.lazyimport.lazyimport(globals(),
- 'glib',
-)
-
class GlibEventLoop(object):
# TODO: Support multiprocessing by using a separate glib.MainContext
@@ -13,6 +8,7 @@ class GlibEventLoop(object):
supports_multiprocessing = False
def __init__(self):
+ import gi.repository.GLib as glib
self.IO_ERR = glib.IO_ERR
self.IO_HUP = glib.IO_HUP
self.IO_IN = glib.IO_IN
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-11 19:35 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-11 19:35 UTC (permalink / raw
To: gentoo-commits
commit: dcb9fab8463996542d06d29bc383f5933bf0d677
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 11 19:34:36 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 11 19:34:36 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=dcb9fab8
EventLoop: shorten iteration poll if timeouts
This fixes EventLoop.iteration() so that it doesn't poll too long when
there are timeouts registered, since it's not nice to spend a long time
polling for an IO event that might never arrive even though a timeout
may have changed some state that would terminate a timeout loop like
in QueueScheduler.run() or AbstractPollTask._wait_loop(). This makes
EventLoop.iteration() behave more like GlibEventLoop.iteration().
---
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 02c3cec..a7161db 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -166,7 +166,7 @@ class EventLoop(object):
if not self._poll_event_queue:
if may_block:
- timeout = None
+ timeout = self._timeout_interval
else:
timeout = 0
self._poll(timeout=timeout)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-11 21:11 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-11 21:11 UTC (permalink / raw
To: gentoo-commits
commit: 517b791999843b1749c1c8d84c5908634524dc5c
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 11 21:11:10 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 11 21:11:10 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=517b7919
EventLoop.iteration: run timeouts if no fds
---
pym/portage/util/_eventloop/EventLoop.py | 10 +++++++++-
1 files changed, 9 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index a7161db..07d9bec 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -162,7 +162,15 @@ class EventLoop(object):
events_handled = 0
if not event_handlers:
- return bool(events_handled)
+ if not self._polling:
+ self._polling = True
+ try:
+ if self._run_timeouts():
+ events_handled += 1
+ finally:
+ self._polling = False
+ if not event_handlers:
+ return bool(events_handled)
if not self._poll_event_queue:
if may_block:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-11 21:41 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-11 21:41 UTC (permalink / raw
To: gentoo-commits
commit: f406dba6540a6873a08c331bd4040d6a33cd3538
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 11 21:41:18 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 11 21:41:18 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f406dba6
EventLoop: remove obsolete timeout_add comment
---
pym/portage/util/_eventloop/EventLoop.py | 7 -------
1 files changed, 0 insertions(+), 7 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 07d9bec..c059197 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -227,13 +227,6 @@ class EventLoop(object):
should return False to stop being called, or True to continue
being called. Any additional positional arguments given here
are passed to your function when it's called.
-
- NOTE: Timeouts registered by this function currently do not
- keep the main loop running when there are no remaining callbacks
- registered for IO events. This is not an issue if the purpose of
- the timeout is to place an upper limit on the time allowed for
- a particular IO event to occur, since the handler associated with
- the IO event will serve to keep the main loop running.
"""
self._event_handler_id += 1
source_id = self._event_handler_id
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-11 23:56 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-11 23:56 UTC (permalink / raw
To: gentoo-commits
commit: ea8cb012d563bdce3eeeec6f85874503b0048928
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 11 23:56:07 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 11 23:56:07 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ea8cb012
EventLoop.iteration: handle _poll StopIteration
---
pym/portage/util/_eventloop/EventLoop.py | 8 +++++++-
1 files changed, 7 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index c059197..0da388d 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -177,7 +177,13 @@ class EventLoop(object):
timeout = self._timeout_interval
else:
timeout = 0
- self._poll(timeout=timeout)
+ try:
+ self._poll(timeout=timeout)
+ except StopIteration:
+ # This could happen if there are no IO event handlers
+ # after _poll() calls _run_timeouts(), due to them
+ # being removed by timeout or idle callbacks.
+ events_handled += 1
try:
while event_handlers and self._poll_event_queue:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-14 1:23 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-14 1:23 UTC (permalink / raw
To: gentoo-commits
commit: 6afd0e508eaf1f9040a20ed670cd6cf7a3a07517
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 14 01:22:56 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 14 01:22:56 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6afd0e50
EventLoop: make _poll/_run_timeouts re-entrant
This fixes infinite loops triggered by Ctrl-C, where timeout calls
would exhaust the poll event queue because _poll was not re-entrant.
Now, re-entrance is only prohibited for individual callback functions,
in order to protect against infinite recursion.
---
pym/portage/util/_eventloop/EventLoop.py | 135 ++++++++++++++++--------------
1 files changed, 73 insertions(+), 62 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 0da388d..6c6a1b7 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -16,13 +16,13 @@ class EventLoop(object):
supports_multiprocessing = True
class _idle_callback_class(SlotObject):
- __slots__ = ("args", "callback", "source_id")
+ __slots__ = ("args", "callback", "calling", "source_id")
class _io_handler_class(SlotObject):
- __slots__ = ("args", "callback", "fd", "source_id")
+ __slots__ = ("args", "callback", "calling", "fd", "source_id")
class _timeout_handler_class(SlotObject):
- __slots__ = ("args", "function", "interval", "source_id",
+ __slots__ = ("args", "function", "calling", "interval", "source_id",
"timestamp")
def __init__(self):
@@ -35,7 +35,6 @@ class EventLoop(object):
self._timeout_handlers = {}
self._timeout_interval = None
self._poll_obj = create_poll_instance()
- self._polling = False
self.IO_ERR = PollConstants.POLLERR
self.IO_HUP = PollConstants.POLLHUP
@@ -45,53 +44,47 @@ class EventLoop(object):
self.IO_PRI = PollConstants.POLLPRI
def _poll(self, timeout=None):
- if self._polling:
- return
- self._polling = True
- try:
- if self._timeout_interval is None:
+ if self._timeout_interval is None:
+ self._run_timeouts()
+ self._do_poll(timeout=timeout)
+
+ elif timeout is None:
+ while True:
self._run_timeouts()
- self._do_poll(timeout=timeout)
+ previous_count = len(self._poll_event_queue)
+ self._do_poll(timeout=self._timeout_interval)
+ if previous_count != len(self._poll_event_queue):
+ break
- elif timeout is None:
- while True:
- self._run_timeouts()
- previous_count = len(self._poll_event_queue)
- self._do_poll(timeout=self._timeout_interval)
- if previous_count != len(self._poll_event_queue):
- break
+ elif timeout <= self._timeout_interval:
+ self._run_timeouts()
+ self._do_poll(timeout=timeout)
- elif timeout <= self._timeout_interval:
+ else:
+ remaining_timeout = timeout
+ start_time = time.time()
+ while True:
self._run_timeouts()
- self._do_poll(timeout=timeout)
-
- else:
- remaining_timeout = timeout
- start_time = time.time()
- while True:
- self._run_timeouts()
- # _timeout_interval can change each time
- # _run_timeouts is called
- min_timeout = remaining_timeout
- if self._timeout_interval is not None and \
- self._timeout_interval < min_timeout:
- min_timeout = self._timeout_interval
-
- previous_count = len(self._poll_event_queue)
- self._do_poll(timeout=min_timeout)
- if previous_count != len(self._poll_event_queue):
- break
- elapsed_time = time.time() - start_time
- if elapsed_time < 0:
- # The system clock has changed such that start_time
- # is now in the future, so just assume that the
- # timeout has already elapsed.
- break
- remaining_timeout = timeout - 1000 * elapsed_time
- if remaining_timeout <= 0:
- break
- finally:
- self._polling = False
+ # _timeout_interval can change each time
+ # _run_timeouts is called
+ min_timeout = remaining_timeout
+ if self._timeout_interval is not None and \
+ self._timeout_interval < min_timeout:
+ min_timeout = self._timeout_interval
+
+ previous_count = len(self._poll_event_queue)
+ self._do_poll(timeout=min_timeout)
+ if previous_count != len(self._poll_event_queue):
+ break
+ elapsed_time = time.time() - start_time
+ if elapsed_time < 0:
+ # The system clock has changed such that start_time
+ # is now in the future, so just assume that the
+ # timeout has already elapsed.
+ break
+ remaining_timeout = timeout - 1000 * elapsed_time
+ if remaining_timeout <= 0:
+ break
def _do_poll(self, timeout=None):
"""
@@ -162,13 +155,8 @@ class EventLoop(object):
events_handled = 0
if not event_handlers:
- if not self._polling:
- self._polling = True
- try:
- if self._run_timeouts():
- events_handled += 1
- finally:
- self._polling = False
+ if self._run_timeouts():
+ events_handled += 1
if not event_handlers:
return bool(events_handled)
@@ -189,9 +177,16 @@ class EventLoop(object):
while event_handlers and self._poll_event_queue:
f, event = self._next_poll_event()
x = event_handlers[f]
- if not x.callback(f, event, *x.args):
- self.source_remove(x.source_id)
+ if x.calling:
+ # don't call it recursively
+ continue
events_handled += 1
+ x.calling = True
+ try:
+ if not x.callback(f, event, *x.args):
+ self.source_remove(x.source_id)
+ finally:
+ x.calling = False
except StopIteration:
events_handled += 1
@@ -223,8 +218,15 @@ class EventLoop(object):
if x.source_id not in self._idle_callbacks:
# it got cancelled while executing another callback
continue
- if not x.callback(*x.args):
- self.source_remove(x.source_id)
+ if x.calling:
+ # don't call it recursively
+ continue
+ x.calling = True
+ try:
+ if not x.callback(*x.args):
+ self.source_remove(x.source_id)
+ finally:
+ x.calling = False
def timeout_add(self, interval, function, *args):
"""
@@ -262,15 +264,24 @@ class EventLoop(object):
# Iterate of our local list, since self._timeout_handlers can be
# modified during the exection of these callbacks.
+ calls = 0
for x in ready_timeouts:
if x.source_id not in self._timeout_handlers:
# it got cancelled while executing another timeout
continue
- x.timestamp = time.time()
- if not x.function(*x.args):
- self.source_remove(x.source_id)
+ if x.calling:
+ # don't call it recursively
+ continue
+ calls += 1
+ x.calling = True
+ try:
+ x.timestamp = time.time()
+ if not x.function(*x.args):
+ self.source_remove(x.source_id)
+ finally:
+ x.calling = False
- return bool(ready_timeouts)
+ return bool(calls)
def io_add_watch(self, f, condition, callback, *args):
"""
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-14 3:06 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-14 3:06 UTC (permalink / raw
To: gentoo-commits
commit: fe3960b69c326bc779bdf5ec34d56630b3e188ae
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 14 03:06:29 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 14 03:06:29 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fe3960b6
EventLoop._do_poll: tweak EINTR handling
Silently handle EINTR, which is normal when we have received a signal
such as SIGINT. Also, raise StopIteration in order to break out of our
current iteration and respond appropriately to the signal as soon as
possible.
---
pym/portage/util/_eventloop/EventLoop.py | 27 ++++++++++++++-------------
1 files changed, 14 insertions(+), 13 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 6c6a1b7..37839ab 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -1,6 +1,7 @@
# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+import errno
import logging
import select
import time
@@ -100,24 +101,23 @@ class EventLoop(object):
raise StopIteration(
"timeout is None and there are no poll() event handlers")
- # The following error is known to occur with Linux kernel versions
- # less than 2.6.24:
- #
- # select.error: (4, 'Interrupted system call')
- #
- # This error has been observed after a SIGSTOP, followed by SIGCONT.
- # Treat it similar to EAGAIN if timeout is None, otherwise just return
- # without any events.
while True:
try:
self._poll_event_queue.extend(self._poll_obj.poll(timeout))
break
except select.error as e:
- writemsg_level("\n!!! select error: %s\n" % (e,),
- level=logging.ERROR, noiselevel=-1)
+ # Silently handle EINTR, which is normal when we have
+ # received a signal such as SIGINT.
+ if not (e.args and e.args[0] == errno.EINTR):
+ writemsg_level("\n!!! select error: %s\n" % (e,),
+ level=logging.ERROR, noiselevel=-1)
del e
- if timeout is not None:
- break
+
+ # This typically means that we've received a SIGINT, so
+ # raise StopIteration in order to break out of our current
+ # iteration and respond appropriately to the signal as soon
+ # as possible.
+ raise StopIteration("interrupted")
def _next_poll_event(self, timeout=None):
"""
@@ -170,7 +170,8 @@ class EventLoop(object):
except StopIteration:
# This could happen if there are no IO event handlers
# after _poll() calls _run_timeouts(), due to them
- # being removed by timeout or idle callbacks.
+ # being removed by timeout or idle callbacks. It can
+ # also be triggered by EINTR which is caused by signals.
events_handled += 1
try:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-14 16:27 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-14 16:27 UTC (permalink / raw
To: gentoo-commits
commit: 608c76a4eb76b88221ba24ca1765765bef827fe1
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 14 16:27:01 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 14 16:27:01 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=608c76a4
EventLoop: allow IO event handler re-entrance
IO event handlers may be re-entrant, in case something like
AbstractPollTask._wait_loop(), needs to be called inside a handler for
some reason.
---
pym/portage/util/_eventloop/EventLoop.py | 17 ++++++-----------
1 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 37839ab..af0c6a5 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -20,7 +20,7 @@ class EventLoop(object):
__slots__ = ("args", "callback", "calling", "source_id")
class _io_handler_class(SlotObject):
- __slots__ = ("args", "callback", "calling", "fd", "source_id")
+ __slots__ = ("args", "callback", "fd", "source_id")
class _timeout_handler_class(SlotObject):
__slots__ = ("args", "function", "calling", "interval", "source_id",
@@ -178,16 +178,11 @@ class EventLoop(object):
while event_handlers and self._poll_event_queue:
f, event = self._next_poll_event()
x = event_handlers[f]
- if x.calling:
- # don't call it recursively
- continue
- events_handled += 1
- x.calling = True
- try:
- if not x.callback(f, event, *x.args):
- self.source_remove(x.source_id)
- finally:
- x.calling = False
+ # NOTE: IO event handlers may be re-entrant, in case something
+ # like AbstractPollTask._wait_loop(), needs to be called inside
+ # a handler for some reason.
+ if not x.callback(f, event, *x.args):
+ self.source_remove(x.source_id)
except StopIteration:
events_handled += 1
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-16 4:41 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-16 4:41 UTC (permalink / raw
To: gentoo-commits
commit: 611ec3edf3695453639f2436049d0005f6e6f769
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 16 04:41:16 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 16 04:41:16 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=611ec3ed
EventLoop.timeout_add: fix inverted min interval
---
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index af0c6a5..f6f9fc5 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -238,7 +238,7 @@ class EventLoop(object):
self._timeout_handler_class(
interval=interval, function=function, args=args,
source_id=source_id, timestamp=time.time())
- if self._timeout_interval is None or self._timeout_interval < interval:
+ if self._timeout_interval is None or self._timeout_interval > interval:
self._timeout_interval = interval
return source_id
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-16 4:58 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-16 4:58 UTC (permalink / raw
To: gentoo-commits
commit: 5697972e9f2a3021ca6ed99b24e23e32f7515bf9
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 16 04:58:10 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 16 04:58:10 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5697972e
EventLoop.iteration: sleep if no IO handlers
Sleep so that we don't waste cpu time by looping too quickly. This
makes EventLoop useful for code that needs to wait for timeout
callbacks regardless of whether or not any IO handlers are currently
registered.
---
pym/portage/util/_eventloop/EventLoop.py | 14 +++++++++++++-
1 files changed, 13 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index f6f9fc5..24ba140 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -158,7 +158,19 @@ class EventLoop(object):
if self._run_timeouts():
events_handled += 1
if not event_handlers:
- return bool(events_handled)
+ if not events_handled and may_block and \
+ self._timeout_interval is not None:
+ # Sleep so that we don't waste cpu time by looping too
+ # quickly. This makes EventLoop useful for code that needs
+ # to wait for timeout callbacks regardless of whether or
+ # not any IO handlers are currently registered.
+ time.sleep(self._timeout_interval/1000)
+ if self._run_timeouts():
+ events_handled += 1
+ if not event_handlers:
+ return bool(events_handled)
+ else:
+ return bool(events_handled)
if not self._poll_event_queue:
if may_block:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-16 21:35 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-16 21:35 UTC (permalink / raw
To: gentoo-commits
commit: 47cfc4f2bf1a72ff1ab7ec3178c927a2975aa0c5
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 16 21:35:23 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 16 21:35:23 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=47cfc4f2
EventLoop: fix _sigchld_io_cb to return True
This fixes an infinite loop since commit
1979a6cdfcd8c6bae4565982d82d862be07ba5be.
---
pym/portage/util/_eventloop/EventLoop.py | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 994952e..3123cab 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -263,6 +263,7 @@ class EventLoop(object):
# read until EAGAIN
pass
self._poll_child_processes()
+ return True
def _poll_child_processes(self):
if not self._child_handlers:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-16 21:43 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-16 21:43 UTC (permalink / raw
To: gentoo-commits
commit: f5e04fa4d74fc404cb067ecedf1adb7e5fc8846f
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 16 21:42:59 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 16 21:42:59 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f5e04fa4
EventLoop: fix signal race in _sigchld_init
---
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 3123cab..9ca529f 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -242,11 +242,11 @@ class EventLoop(object):
return source_id
def _sigchld_init(self):
- signal.signal(signal.SIGCHLD, self._sigchld_sig_cb)
self._sigchld_read, self._sigchld_write = os.pipe()
fcntl.fcntl(self._sigchld_read, fcntl.F_SETFL,
fcntl.fcntl(self._sigchld_read, fcntl.F_GETFL) | os.O_NONBLOCK)
self.io_add_watch(self._sigchld_read, self.IO_IN, self._sigchld_io_cb)
+ signal.signal(signal.SIGCHLD, self._sigchld_sig_cb)
def _sigchld_sig_cb(self, signum, frame):
# If this signal handler was not installed by the
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-17 3:08 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-17 3:08 UTC (permalink / raw
To: gentoo-commits
commit: bbcfd4e2ad7f9d97ee7bb027b5345c2d5bbb3997
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 03:08:07 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 03:08:07 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=bbcfd4e2
EventLoop: wakeup poll loop to receive sigchild
TODO: Find out why SIGCHLD signals aren't delivered during poll calls,
forcing us to wakeup in order to receive them. This fixes random hangs
in poll calls since commit 1979a6cdfcd8c6bae4565982d82d862be07ba5be.
---
pym/portage/util/_eventloop/EventLoop.py | 13 ++++++++++++-
1 files changed, 12 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 9ca529f..e381342 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -19,6 +19,10 @@ class EventLoop(object):
supports_multiprocessing = True
+ # TODO: Find out why SIGCHLD signals aren't delivered during poll
+ # calls, forcing us to wakeup in order to receive them.
+ _sigchld_interval = 250
+
class _child_callback_class(SlotObject):
__slots__ = ("callback", "data", "pid", "source_id")
@@ -185,7 +189,14 @@ class EventLoop(object):
if not self._poll_event_queue:
if may_block:
- timeout = self._timeout_interval
+ if self._child_handlers:
+ if self._timeout_interval is None:
+ timeout = self._sigchld_interval
+ else:
+ timeout = min(self._sigchld_interval,
+ self._timeout_interval)
+ else:
+ timeout = self._timeout_interval
else:
timeout = 0
try:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-17 6:04 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-17 6:04 UTC (permalink / raw
To: gentoo-commits
commit: ed9125487af39181bf3a00ba46a2bda7ea1d68d2
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 06:03:30 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 06:03:30 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ed912548
EventLoop.child_watch_add: dynamic IO watch
The IO watch is dynamically registered and unregistered as needed,
since we don't want to consider it as a valid source of events when
there are no child listeners. It's important to distinguish when there
are no valid sources of IO events, in order to avoid an endless poll
call if there's no timeout. This fixes possbible endless poll calls
since commit 1979a6cdfcd8c6bae4565982d82d862be07ba5be.
---
pym/portage/util/_eventloop/EventLoop.py | 32 ++++++++++++++++++++---------
1 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index e381342..7f171fb 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -57,6 +57,7 @@ class EventLoop(object):
self._child_handlers = {}
self._sigchld_read = None
self._sigchld_write = None
+ self._sigchld_src_id = None
self._pid = os.getpid()
def _poll(self, timeout=None):
@@ -246,19 +247,26 @@ class EventLoop(object):
source_id = self._event_handler_id
self._child_handlers[source_id] = self._child_callback_class(
callback=callback, data=data, pid=pid, source_id=source_id)
+
if self._sigchld_read is None:
- self._sigchld_init()
+ self._sigchld_read, self._sigchld_write = os.pipe()
+ fcntl.fcntl(self._sigchld_read, fcntl.F_SETFL,
+ fcntl.fcntl(self._sigchld_read, fcntl.F_GETFL) | os.O_NONBLOCK)
+
+ # The IO watch is dynamically registered and unregistered as
+ # needed, since we don't want to consider it as a valid source
+ # of events when there are no child listeners. It's important
+ # to distinguish when there are no valid sources of IO events,
+ # in order to avoid an endless poll call if there's no timeout.
+ if self._sigchld_src_id is None:
+ self._sigchld_src_id = self.io_add_watch(
+ self._sigchld_read, self.IO_IN, self._sigchld_io_cb)
+ signal.signal(signal.SIGCHLD, self._sigchld_sig_cb)
+
# poll now, in case the SIGCHLD has already arrived
self._poll_child_processes()
return source_id
- def _sigchld_init(self):
- self._sigchld_read, self._sigchld_write = os.pipe()
- fcntl.fcntl(self._sigchld_read, fcntl.F_SETFL,
- fcntl.fcntl(self._sigchld_read, fcntl.F_GETFL) | os.O_NONBLOCK)
- self.io_add_watch(self._sigchld_read, self.IO_IN, self._sigchld_io_cb)
- signal.signal(signal.SIGCHLD, self._sigchld_sig_cb)
-
def _sigchld_sig_cb(self, signum, frame):
# If this signal handler was not installed by the
# current process then the signal doesn't belong to
@@ -292,14 +300,14 @@ class EventLoop(object):
if e.errno != errno.ECHILD:
raise
del e
- self._child_handlers.pop(x.source_id, None)
+ self.source_remove(x.source_id)
else:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if wait_retval[0] != 0:
calls += 1
- self._child_handlers.pop(x.source_id, None)
+ self.source_remove(x.source_id)
x.callback(x.pid, wait_retval[1], x.data)
return bool(calls)
@@ -429,6 +437,10 @@ class EventLoop(object):
"""
x = self._child_handlers.pop(reg_id, None)
if x is not None:
+ if not self._child_handlers:
+ signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+ self.source_remove(self._sigchld_src_id)
+ self._sigchld_src_id = None
return True
idle_callback = self._idle_callbacks.pop(reg_id, None)
if idle_callback is not None:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-17 6:29 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-17 6:29 UTC (permalink / raw
To: gentoo-commits
commit: 8dd6814bd387ddfba484bf22c429103b885a58c6
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 06:29:17 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 06:29:17 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8dd6814b
EventLoop.iteration: poll for blocking, not sleep
The effect is be mostly the same, but it's more conistent to use
_do_poll for all blocking, plus it has EINTR handling.
---
pym/portage/util/_eventloop/EventLoop.py | 7 +++++--
1 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 7f171fb..37a971e 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -176,11 +176,14 @@ class EventLoop(object):
if not event_handlers:
if not events_handled and may_block and \
self._timeout_interval is not None:
- # Sleep so that we don't waste cpu time by looping too
+ # Block so that we don't waste cpu time by looping too
# quickly. This makes EventLoop useful for code that needs
# to wait for timeout callbacks regardless of whether or
# not any IO handlers are currently registered.
- time.sleep(self._timeout_interval/1000)
+ try:
+ self._do_poll(timeout=self._timeout_interval)
+ except StopIteration:
+ pass
if self._run_timeouts():
events_handled += 1
if not event_handlers:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-17 9:29 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-17 9:29 UTC (permalink / raw
To: gentoo-commits
commit: f9b33e065f2065c900c978b4515a52f9869e89f1
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 09:29:15 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 09:29:15 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f9b33e06
EventLoop: prune obsolete code
---
pym/portage/util/_eventloop/EventLoop.py | 88 ++++++------------------------
1 files changed, 18 insertions(+), 70 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 37a971e..808c83d 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -61,49 +61,6 @@ class EventLoop(object):
self._pid = os.getpid()
def _poll(self, timeout=None):
- if self._timeout_interval is None:
- self._run_timeouts()
- self._do_poll(timeout=timeout)
-
- elif timeout is None:
- while True:
- self._run_timeouts()
- previous_count = len(self._poll_event_queue)
- self._do_poll(timeout=self._timeout_interval)
- if previous_count != len(self._poll_event_queue):
- break
-
- elif timeout <= self._timeout_interval:
- self._run_timeouts()
- self._do_poll(timeout=timeout)
-
- else:
- remaining_timeout = timeout
- start_time = time.time()
- while True:
- self._run_timeouts()
- # _timeout_interval can change each time
- # _run_timeouts is called
- min_timeout = remaining_timeout
- if self._timeout_interval is not None and \
- self._timeout_interval < min_timeout:
- min_timeout = self._timeout_interval
-
- previous_count = len(self._poll_event_queue)
- self._do_poll(timeout=min_timeout)
- if previous_count != len(self._poll_event_queue):
- break
- elapsed_time = time.time() - start_time
- if elapsed_time < 0:
- # The system clock has changed such that start_time
- # is now in the future, so just assume that the
- # timeout has already elapsed.
- break
- remaining_timeout = timeout - 1000 * elapsed_time
- if remaining_timeout <= 0:
- break
-
- def _do_poll(self, timeout=None):
"""
All poll() calls pass through here. The poll events
are added directly to self._poll_event_queue.
@@ -135,20 +92,6 @@ class EventLoop(object):
# as possible.
raise StopIteration("interrupted")
- def _next_poll_event(self, timeout=None):
- """
- Since iteration() can be called recursively, maintain
- a central event queue to share events from a single
- poll() call. In order to avoid endless blocking, this
- raises StopIteration if timeout is None and there are
- no file descriptors to poll.
- """
- if not self._poll_event_queue:
- self._poll(timeout)
- if not self._poll_event_queue:
- raise StopIteration()
- return self._poll_event_queue.pop()
-
def iteration(self, *args):
"""
Like glib.MainContext.iteration(), runs a single iteration.
@@ -167,6 +110,7 @@ class EventLoop(object):
"expected at most 1 argument (%s given)" % len(args))
may_block = args[0]
+ event_queue = self._poll_event_queue
event_handlers = self._poll_event_handlers
events_handled = 0
@@ -181,7 +125,7 @@ class EventLoop(object):
# to wait for timeout callbacks regardless of whether or
# not any IO handlers are currently registered.
try:
- self._do_poll(timeout=self._timeout_interval)
+ self._poll(timeout=self._timeout_interval)
except StopIteration:
pass
if self._run_timeouts():
@@ -191,7 +135,8 @@ class EventLoop(object):
else:
return bool(events_handled)
- if not self._poll_event_queue:
+ if not event_queue:
+
if may_block:
if self._child_handlers:
if self._timeout_interval is None:
@@ -203,26 +148,29 @@ class EventLoop(object):
timeout = self._timeout_interval
else:
timeout = 0
+
+ if self._run_timeouts():
+ events_handled += 1
+
try:
+
self._poll(timeout=timeout)
except StopIteration:
# This could happen if there are no IO event handlers
# after _poll() calls _run_timeouts(), due to them
# being removed by timeout or idle callbacks. It can
# also be triggered by EINTR which is caused by signals.
- events_handled += 1
+ pass
- try:
- while event_handlers and self._poll_event_queue:
- f, event = self._next_poll_event()
- x = event_handlers[f]
- # NOTE: IO event handlers may be re-entrant, in case something
- # like AbstractPollTask._wait_loop(), needs to be called inside
- # a handler for some reason.
- if not x.callback(f, event, *x.args):
- self.source_remove(x.source_id)
- except StopIteration:
+ # NOTE: IO event handlers may be re-entrant, in case something
+ # like AbstractPollTask._wait_loop() needs to be called inside
+ # a handler for some reason.
+ while event_queue:
events_handled += 1
+ f, event = event_queue.pop()
+ x = event_handlers[f]
+ if not x.callback(f, event, *x.args):
+ self.source_remove(x.source_id)
return bool(events_handled)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-17 10:20 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-17 10:20 UTC (permalink / raw
To: gentoo-commits
commit: 9e5cda69b593c37151d0e4014c7025ff415ae831
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 10:20:35 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 10:20:35 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9e5cda69
EventLoop.iteration: run timeouts last
Run timeouts last, in order to minimize latency in termination of
iteration loops that they may control.
---
pym/portage/util/_eventloop/EventLoop.py | 16 ++++++++++------
1 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 808c83d..255527f 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -130,9 +130,11 @@ class EventLoop(object):
pass
if self._run_timeouts():
events_handled += 1
- if not event_handlers:
- return bool(events_handled)
- else:
+
+ # If any timeouts have executed, then return immediately,
+ # in order to minimize latency in termination of iteration
+ # loops that they may control.
+ if events_handled or not event_handlers:
return bool(events_handled)
if not event_queue:
@@ -149,9 +151,6 @@ class EventLoop(object):
else:
timeout = 0
- if self._run_timeouts():
- events_handled += 1
-
try:
self._poll(timeout=timeout)
@@ -172,6 +171,11 @@ class EventLoop(object):
if not x.callback(f, event, *x.args):
self.source_remove(x.source_id)
+ # Run timeouts last, in order to minimize latency in
+ # termination of iteration loops that they may control.
+ if self._run_timeouts():
+ events_handled += 1
+
return bool(events_handled)
def child_watch_add(self, pid, callback, data=None):
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-17 10:31 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-17 10:31 UTC (permalink / raw
To: gentoo-commits
commit: 4a858889202448e93cb43ec2559e9f66c30b09c9
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 10:20:35 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 10:31:07 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4a858889
EventLoop.iteration: run timeouts last
Run timeouts last, in order to minimize latency in termination of
iteration loops that they may control.
---
pym/portage/util/_eventloop/EventLoop.py | 24 ++++++++++++------------
1 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 808c83d..475a4a9 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -130,10 +130,12 @@ class EventLoop(object):
pass
if self._run_timeouts():
events_handled += 1
- if not event_handlers:
- return bool(events_handled)
- else:
- return bool(events_handled)
+
+ # If any timeouts have executed, then return immediately,
+ # in order to minimize latency in termination of iteration
+ # loops that they may control.
+ if events_handled or not event_handlers:
+ return bool(events_handled)
if not event_queue:
@@ -149,17 +151,10 @@ class EventLoop(object):
else:
timeout = 0
- if self._run_timeouts():
- events_handled += 1
-
try:
-
self._poll(timeout=timeout)
except StopIteration:
- # This could happen if there are no IO event handlers
- # after _poll() calls _run_timeouts(), due to them
- # being removed by timeout or idle callbacks. It can
- # also be triggered by EINTR which is caused by signals.
+ # This can be triggered by EINTR which is caused by signals.
pass
# NOTE: IO event handlers may be re-entrant, in case something
@@ -172,6 +167,11 @@ class EventLoop(object):
if not x.callback(f, event, *x.args):
self.source_remove(x.source_id)
+ # Run timeouts last, in order to minimize latency in
+ # termination of iteration loops that they may control.
+ if self._run_timeouts():
+ events_handled += 1
+
return bool(events_handled)
def child_watch_add(self, pid, callback, data=None):
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-17 22:17 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-17 22:17 UTC (permalink / raw
To: gentoo-commits
commit: d6081e461bc6d14fb8d63e6346fdfda7fc0cc188
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 22:16:59 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 22:16:59 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d6081e46
EventLoop: fix _io_handler_class "f" attribute
---
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 475a4a9..9e4ac28 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -30,7 +30,7 @@ class EventLoop(object):
__slots__ = ("args", "callback", "calling", "source_id")
class _io_handler_class(SlotObject):
- __slots__ = ("args", "callback", "fd", "source_id")
+ __slots__ = ("args", "callback", "f", "source_id")
class _timeout_handler_class(SlotObject):
__slots__ = ("args", "function", "calling", "interval", "source_id",
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-17 23:23 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-17 23:23 UTC (permalink / raw
To: gentoo-commits
commit: 52ec1135694dca47662ed066f747b945b71e3177
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 23:23:39 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 23:23:39 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=52ec1135
PollSelectAdapter: don't inherit PollConstants
---
pym/portage/util/_eventloop/PollSelectAdapter.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/PollSelectAdapter.py b/pym/portage/util/_eventloop/PollSelectAdapter.py
index 55f08f0..b91b22a 100644
--- a/pym/portage/util/_eventloop/PollSelectAdapter.py
+++ b/pym/portage/util/_eventloop/PollSelectAdapter.py
@@ -4,7 +4,7 @@
from portage.util._eventloop.PollConstants import PollConstants
import select
-class PollSelectAdapter(PollConstants):
+class PollSelectAdapter(object):
"""
Use select to emulate a poll object, for
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-02-18 4:04 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-02-18 4:04 UTC (permalink / raw
To: gentoo-commits
commit: 9ed6365f4c6e65f5c38f23505fe704ac5f81feaa
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 18 04:03:41 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 18 04:03:41 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9ed6365f
_eventloop: use explicit relative imports
---
pym/portage/util/_eventloop/EventLoop.py | 6 +++---
pym/portage/util/_eventloop/PollSelectAdapter.py | 2 +-
pym/portage/util/_eventloop/global_event_loop.py | 5 ++---
3 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 478faab..a2d4cd4 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -10,9 +10,9 @@ import signal
import time
from portage.util import writemsg_level
-from portage.util.SlotObject import SlotObject
-from portage.util._eventloop.PollConstants import PollConstants
-from portage.util._eventloop.PollSelectAdapter import PollSelectAdapter
+from ..SlotObject import SlotObject
+from .PollConstants import PollConstants
+from .PollSelectAdapter import PollSelectAdapter
class EventLoop(object):
diff --git a/pym/portage/util/_eventloop/PollSelectAdapter.py b/pym/portage/util/_eventloop/PollSelectAdapter.py
index b91b22a..17e63d9 100644
--- a/pym/portage/util/_eventloop/PollSelectAdapter.py
+++ b/pym/portage/util/_eventloop/PollSelectAdapter.py
@@ -1,7 +1,7 @@
# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-from portage.util._eventloop.PollConstants import PollConstants
+from .PollConstants import PollConstants
import select
class PollSelectAdapter(object):
diff --git a/pym/portage/util/_eventloop/global_event_loop.py b/pym/portage/util/_eventloop/global_event_loop.py
index 0f0c53f..502dab8 100644
--- a/pym/portage/util/_eventloop/global_event_loop.py
+++ b/pym/portage/util/_eventloop/global_event_loop.py
@@ -3,11 +3,10 @@
import os
-from portage.util._eventloop.EventLoop import EventLoop
+from .EventLoop import EventLoop
_default_constructor = EventLoop
-#from portage.util._eventloop.GlibEventLoop \
-# import GlibEventLoop as _default_constructor
+#from .GlibEventLoop import GlibEventLoop as _default_constructor
# If _default_constructor doesn't support multiprocessing,
# then _multiprocessing_constructor is used in subprocesses.
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-08-22 5:39 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-08-22 5:39 UTC (permalink / raw
To: gentoo-commits
commit: b986bcdd49c5523ffe6972377071d556a819c776
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 22 05:38:57 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Aug 22 05:38:57 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b986bcdd
EventLoop: use epoll when available
This will fix bug #432024.
---
pym/portage/util/_eventloop/EventLoop.py | 59 ++++++++++++++++++++++++++----
1 files changed, 52 insertions(+), 7 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index bbbce52..eed68fe 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -52,14 +52,25 @@ class EventLoop(object):
self._idle_callbacks = {}
self._timeout_handlers = {}
self._timeout_interval = None
- self._poll_obj = create_poll_instance()
- self.IO_ERR = PollConstants.POLLERR
- self.IO_HUP = PollConstants.POLLHUP
- self.IO_IN = PollConstants.POLLIN
- self.IO_NVAL = PollConstants.POLLNVAL
- self.IO_OUT = PollConstants.POLLOUT
- self.IO_PRI = PollConstants.POLLPRI
+ try:
+ select.epoll
+ except AttributeError:
+ self._poll_obj = create_poll_instance()
+ self.IO_ERR = PollConstants.POLLERR
+ self.IO_HUP = PollConstants.POLLHUP
+ self.IO_IN = PollConstants.POLLIN
+ self.IO_NVAL = PollConstants.POLLNVAL
+ self.IO_OUT = PollConstants.POLLOUT
+ self.IO_PRI = PollConstants.POLLPRI
+ else:
+ self._poll_obj = _epoll_adapter(select.epoll())
+ self.IO_ERR = select.EPOLLERR
+ self.IO_HUP = select.EPOLLHUP
+ self.IO_IN = select.EPOLLIN
+ self.IO_NVAL = 0
+ self.IO_OUT = select.EPOLLOUT
+ self.IO_PRI = select.EPOLLPRI
self._child_handlers = {}
self._sigchld_read = None
@@ -488,3 +499,37 @@ def create_poll_instance():
if can_poll_device():
return select.poll()
return PollSelectAdapter()
+
+class _epoll_adapter(object):
+ """
+ Wraps a select.epoll instance in order to make it compatible
+ with select.poll instances. This is necessary since epoll instances
+ interpret timeout arguments differently. Note that the file descriptor
+ that is associated with an epoll instance will close automatically when
+ it is garbage collected, so it's not necessary to close it explicitly.
+ """
+ __slots__ = ('_epoll_obj',)
+
+ def __init__(self, epoll_obj):
+ self._epoll_obj = epoll_obj
+
+ def register(self, fd, *args):
+ self._epoll_obj.register(fd, *args)
+
+ def unregister(self, fd):
+ self._epoll_obj.unregister(fd)
+
+ def poll(self, *args):
+ if len(args) > 1:
+ raise TypeError(
+ "poll expected at most 2 arguments, got " + \
+ repr(1 + len(args)))
+ timeout = -1
+ if args:
+ timeout = args[0]
+ if timeout is None or timeout < 0:
+ timeout = -1
+ elif timeout != 0:
+ timeout = timeout / 1000
+
+ return self._epoll_obj.poll(timeout)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-08-22 16:23 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-08-22 16:23 UTC (permalink / raw
To: gentoo-commits
commit: 7a55f93d76bab41f025b77fc4e356d6a5b4b9385
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 22 16:23:18 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Aug 22 16:23:18 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7a55f93d
EventLoop: handle EINTR IOError for epoll
---
pym/portage/util/_eventloop/EventLoop.py | 6 ++++--
1 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index eed68fe..44a3c26 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -96,9 +96,11 @@ class EventLoop(object):
try:
self._poll_event_queue.extend(self._poll_obj.poll(timeout))
break
- except select.error as e:
+ except (IOError, select.error) as e:
# Silently handle EINTR, which is normal when we have
- # received a signal such as SIGINT.
+ # received a signal such as SIGINT (epoll objects may
+ # raise IOError rather than select.error, at least in
+ # Python 3.2).
if not (e.args and e.args[0] == errno.EINTR):
writemsg_level("\n!!! select error: %s\n" % (e,),
level=logging.ERROR, noiselevel=-1)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-09-26 2:26 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-09-26 2:26 UTC (permalink / raw
To: gentoo-commits
commit: 2b2580d9dac62aa720e5d996fa5102ee5caeffe7
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Sep 26 02:25:49 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Sep 26 02:25:49 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2b2580d9
EventLoop: fix busy loop waiting for child pid
This fixes a case where EventLoop could consume 100% CPU while waiting
for a child process. It also fixes timeout calculations in python 2.x,
where it was using integer division instead of float.
---
pym/portage/util/_eventloop/EventLoop.py | 30 ++++++++++++---------
pym/portage/util/_eventloop/PollSelectAdapter.py | 2 +-
2 files changed, 18 insertions(+), 14 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 44a3c26..15d4ef2 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -137,15 +137,15 @@ class EventLoop(object):
if not event_handlers:
if self._run_timeouts():
events_handled += 1
- if not event_handlers:
- if not events_handled and may_block and \
- self._timeout_interval is not None:
+ if not event_handlers and not events_handled and may_block:
+ timeout = self._get_poll_timeout()
+ if timeout is not None:
# Block so that we don't waste cpu time by looping too
# quickly. This makes EventLoop useful for code that needs
# to wait for timeout callbacks regardless of whether or
# not any IO handlers are currently registered.
try:
- self._poll(timeout=self._timeout_interval)
+ self._poll(timeout=timeout)
except StopIteration:
pass
if self._run_timeouts():
@@ -160,14 +160,7 @@ class EventLoop(object):
if not event_queue:
if may_block:
- if self._child_handlers:
- if self._timeout_interval is None:
- timeout = self._sigchld_interval
- else:
- timeout = min(self._sigchld_interval,
- self._timeout_interval)
- else:
- timeout = self._timeout_interval
+ timeout = self._get_poll_timeout()
else:
timeout = 0
@@ -194,6 +187,17 @@ class EventLoop(object):
return bool(events_handled)
+ def _get_poll_timeout(self):
+ if self._child_handlers:
+ if self._timeout_interval is None:
+ timeout = self._sigchld_interval
+ else:
+ timeout = min(self._sigchld_interval,
+ self._timeout_interval)
+ else:
+ timeout = self._timeout_interval
+ return timeout
+
def child_watch_add(self, pid, callback, data=None):
"""
Like glib.child_watch_add(), sets callback to be called with the
@@ -532,6 +536,6 @@ class _epoll_adapter(object):
if timeout is None or timeout < 0:
timeout = -1
elif timeout != 0:
- timeout = timeout / 1000
+ timeout = float(timeout) / 1000
return self._epoll_obj.poll(timeout)
diff --git a/pym/portage/util/_eventloop/PollSelectAdapter.py b/pym/portage/util/_eventloop/PollSelectAdapter.py
index 17e63d9..244788c 100644
--- a/pym/portage/util/_eventloop/PollSelectAdapter.py
+++ b/pym/portage/util/_eventloop/PollSelectAdapter.py
@@ -64,7 +64,7 @@ class PollSelectAdapter(object):
if timeout is not None and timeout < 0:
timeout = None
if timeout is not None:
- select_args.append(timeout / 1000)
+ select_args.append(float(timeout) / 1000)
select_events = select.select(*select_args)
poll_events = []
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-11-22 12:23 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-11-22 12:23 UTC (permalink / raw
To: gentoo-commits
commit: 4cc3db2b37e3a0ee123f9c9c81aba56be543b383
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 22 12:23:20 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Nov 22 12:23:20 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4cc3db2b
EventLoop: handle Linux 2.4 errno 38 for epoll
---
pym/portage/util/_eventloop/EventLoop.py | 27 +++++++++++++++++++--------
1 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 15d4ef2..17a468f 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -53,9 +53,28 @@ class EventLoop(object):
self._timeout_handlers = {}
self._timeout_interval = None
+ self._poll_obj = None
try:
select.epoll
except AttributeError:
+ pass
+ else:
+ try:
+ epoll_obj = select.epoll()
+ except IOError:
+ # This happens with Linux 2.4 kernels:
+ # IOError: [Errno 38] Function not implemented
+ pass
+ else:
+ self._poll_obj = _epoll_adapter(epoll_obj)
+ self.IO_ERR = select.EPOLLERR
+ self.IO_HUP = select.EPOLLHUP
+ self.IO_IN = select.EPOLLIN
+ self.IO_NVAL = 0
+ self.IO_OUT = select.EPOLLOUT
+ self.IO_PRI = select.EPOLLPRI
+
+ if self._poll_obj is None:
self._poll_obj = create_poll_instance()
self.IO_ERR = PollConstants.POLLERR
self.IO_HUP = PollConstants.POLLHUP
@@ -63,14 +82,6 @@ class EventLoop(object):
self.IO_NVAL = PollConstants.POLLNVAL
self.IO_OUT = PollConstants.POLLOUT
self.IO_PRI = PollConstants.POLLPRI
- else:
- self._poll_obj = _epoll_adapter(select.epoll())
- self.IO_ERR = select.EPOLLERR
- self.IO_HUP = select.EPOLLHUP
- self.IO_IN = select.EPOLLIN
- self.IO_NVAL = 0
- self.IO_OUT = select.EPOLLOUT
- self.IO_PRI = select.EPOLLPRI
self._child_handlers = {}
self._sigchld_read = None
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-12-27 2:31 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-12-27 2:31 UTC (permalink / raw
To: gentoo-commits
commit: 323bfee21f553b2c3b088bf6398b8d9e6b963aaf
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 27 02:31:18 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Dec 27 02:31:18 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=323bfee2
EventLoop: thread-safe idle_add and timeout_add
This may be useful for using threads to handle blocking IO with Jython,
since Jython lacks the fcntl module which is needed for non-blocking IO
(see http://bugs.jython.org/issue1074).
---
pym/portage/util/_eventloop/EventLoop.py | 145 +++++++++++++++++------------
1 files changed, 85 insertions(+), 60 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 17a468f..6f464df 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -9,12 +9,23 @@ import select
import signal
import time
+try:
+ import threading
+except ImportError:
+ import dummy_threading as threading
+
from portage.util import writemsg_level
from ..SlotObject import SlotObject
from .PollConstants import PollConstants
from .PollSelectAdapter import PollSelectAdapter
class EventLoop(object):
+ """
+ An event loop, intended to be compatible with the GLib event loop.
+ Call the iteration method in order to execute one iteration of the
+ loop. The idle_add and timeout_add methods serve as thread-safe
+ means to interact with the loop's thread.
+ """
supports_multiprocessing = True
@@ -44,6 +55,7 @@ class EventLoop(object):
@type main: bool
"""
self._use_signal = main
+ self._thread_rlock = threading.RLock()
self._poll_event_queue = []
self._poll_event_handlers = {}
self._poll_event_handler_ids = {}
@@ -89,6 +101,14 @@ class EventLoop(object):
self._sigchld_src_id = None
self._pid = os.getpid()
+ def _new_source_id(self):
+ """
+ Generate a new source id. This method is thread-safe.
+ """
+ with self._thread_rlock:
+ self._event_handler_id += 1
+ return self._event_handler_id
+
def _poll(self, timeout=None):
"""
All poll() calls pass through here. The poll events
@@ -229,8 +249,7 @@ class EventLoop(object):
@rtype: int
@return: an integer ID
"""
- self._event_handler_id += 1
- source_id = self._event_handler_id
+ source_id = self._new_source_id()
self._child_handlers[source_id] = self._child_callback_class(
callback=callback, data=data, pid=pid, source_id=source_id)
@@ -304,20 +323,21 @@ class EventLoop(object):
"""
Like glib.idle_add(), if callback returns False it is
automatically removed from the list of event sources and will
- not be called again.
+ not be called again. This method is thread-safe.
@type callback: callable
@param callback: a function to call
@rtype: int
@return: an integer ID
"""
- self._event_handler_id += 1
- source_id = self._event_handler_id
- self._idle_callbacks[source_id] = self._idle_callback_class(
- args=args, callback=callback, source_id=source_id)
+ with self._thread_rlock:
+ source_id = self._new_source_id()
+ self._idle_callbacks[source_id] = self._idle_callback_class(
+ args=args, callback=callback, source_id=source_id)
return source_id
def _run_idle_callbacks(self):
+ # assumes caller has acquired self._thread_rlock
if not self._idle_callbacks:
return
# Iterate of our local list, since self._idle_callbacks can be
@@ -344,14 +364,15 @@ class EventLoop(object):
being called. Any additional positional arguments given here
are passed to your function when it's called.
"""
- self._event_handler_id += 1
- source_id = self._event_handler_id
- self._timeout_handlers[source_id] = \
- self._timeout_handler_class(
- interval=interval, function=function, args=args,
- source_id=source_id, timestamp=time.time())
- if self._timeout_interval is None or self._timeout_interval > interval:
- self._timeout_interval = interval
+ with self._thread_rlock:
+ source_id = self._new_source_id()
+ self._timeout_handlers[source_id] = \
+ self._timeout_handler_class(
+ interval=interval, function=function, args=args,
+ source_id=source_id, timestamp=time.time())
+ if self._timeout_interval is None or \
+ self._timeout_interval > interval:
+ self._timeout_interval = interval
return source_id
def _run_timeouts(self):
@@ -361,37 +382,39 @@ class EventLoop(object):
if self._poll_child_processes():
calls += 1
- self._run_idle_callbacks()
-
- if not self._timeout_handlers:
- return bool(calls)
-
- ready_timeouts = []
- current_time = time.time()
- for x in self._timeout_handlers.values():
- elapsed_seconds = current_time - x.timestamp
- # elapsed_seconds < 0 means the system clock has been adjusted
- if elapsed_seconds < 0 or \
- (x.interval - 1000 * elapsed_seconds) <= 0:
- ready_timeouts.append(x)
-
- # Iterate of our local list, since self._timeout_handlers can be
- # modified during the exection of these callbacks.
- for x in ready_timeouts:
- if x.source_id not in self._timeout_handlers:
- # it got cancelled while executing another timeout
- continue
- if x.calling:
- # don't call it recursively
- continue
- calls += 1
- x.calling = True
- try:
- x.timestamp = time.time()
- if not x.function(*x.args):
- self.source_remove(x.source_id)
- finally:
- x.calling = False
+ with self._thread_rlock:
+
+ self._run_idle_callbacks()
+
+ if not self._timeout_handlers:
+ return bool(calls)
+
+ ready_timeouts = []
+ current_time = time.time()
+ for x in self._timeout_handlers.values():
+ elapsed_seconds = current_time - x.timestamp
+ # elapsed_seconds < 0 means the system clock has been adjusted
+ if elapsed_seconds < 0 or \
+ (x.interval - 1000 * elapsed_seconds) <= 0:
+ ready_timeouts.append(x)
+
+ # Iterate of our local list, since self._timeout_handlers can be
+ # modified during the exection of these callbacks.
+ for x in ready_timeouts:
+ if x.source_id not in self._timeout_handlers:
+ # it got cancelled while executing another timeout
+ continue
+ if x.calling:
+ # don't call it recursively
+ continue
+ calls += 1
+ x.calling = True
+ try:
+ x.timestamp = time.time()
+ if not x.function(*x.args):
+ self.source_remove(x.source_id)
+ finally:
+ x.calling = False
return bool(calls)
@@ -413,8 +436,7 @@ class EventLoop(object):
"""
if f in self._poll_event_handlers:
raise AssertionError("fd %d is already registered" % f)
- self._event_handler_id += 1
- source_id = self._event_handler_id
+ source_id = self._new_source_id()
self._poll_event_handler_ids[source_id] = f
self._poll_event_handlers[f] = self._io_handler_class(
args=args, callback=callback, f=f, source_id=source_id)
@@ -434,18 +456,21 @@ class EventLoop(object):
self.source_remove(self._sigchld_src_id)
self._sigchld_src_id = None
return True
- idle_callback = self._idle_callbacks.pop(reg_id, None)
- if idle_callback is not None:
- return True
- timeout_handler = self._timeout_handlers.pop(reg_id, None)
- if timeout_handler is not None:
- if timeout_handler.interval == self._timeout_interval:
- if self._timeout_handlers:
- self._timeout_interval = \
- min(x.interval for x in self._timeout_handlers.values())
- else:
- self._timeout_interval = None
- return True
+
+ with self._thread_rlock:
+ idle_callback = self._idle_callbacks.pop(reg_id, None)
+ if idle_callback is not None:
+ return True
+ timeout_handler = self._timeout_handlers.pop(reg_id, None)
+ if timeout_handler is not None:
+ if timeout_handler.interval == self._timeout_interval:
+ if self._timeout_handlers:
+ self._timeout_interval = min(x.interval
+ for x in self._timeout_handlers.values())
+ else:
+ self._timeout_interval = None
+ return True
+
f = self._poll_event_handler_ids.pop(reg_id, None)
if f is None:
return False
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-12-27 2:39 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-12-27 2:39 UTC (permalink / raw
To: gentoo-commits
commit: 5c9bd5c9893e6f852b8dc38c2463f3f7f43122e4
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 27 02:31:18 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Dec 27 02:39:20 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5c9bd5c9
EventLoop: thread-safe idle_add and timeout_add
This may be useful for using threads to handle blocking IO with Jython,
since Jython lacks the fcntl module which is needed for non-blocking IO
(see http://bugs.jython.org/issue1074).
---
pym/portage/util/_eventloop/EventLoop.py | 165 +++++++++++++++++------------
1 files changed, 97 insertions(+), 68 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 17a468f..37e6007 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -9,12 +9,23 @@ import select
import signal
import time
+try:
+ import threading
+except ImportError:
+ import dummy_threading as threading
+
from portage.util import writemsg_level
from ..SlotObject import SlotObject
from .PollConstants import PollConstants
from .PollSelectAdapter import PollSelectAdapter
class EventLoop(object):
+ """
+ An event loop, intended to be compatible with the GLib event loop.
+ Call the iteration method in order to execute one iteration of the
+ loop. The idle_add and timeout_add methods serve as thread-safe
+ means to interact with the loop's thread.
+ """
supports_multiprocessing = True
@@ -44,6 +55,7 @@ class EventLoop(object):
@type main: bool
"""
self._use_signal = main
+ self._thread_rlock = threading.RLock()
self._poll_event_queue = []
self._poll_event_handlers = {}
self._poll_event_handler_ids = {}
@@ -89,6 +101,14 @@ class EventLoop(object):
self._sigchld_src_id = None
self._pid = os.getpid()
+ def _new_source_id(self):
+ """
+ Generate a new source id. This method is thread-safe.
+ """
+ with self._thread_rlock:
+ self._event_handler_id += 1
+ return self._event_handler_id
+
def _poll(self, timeout=None):
"""
All poll() calls pass through here. The poll events
@@ -199,14 +219,17 @@ class EventLoop(object):
return bool(events_handled)
def _get_poll_timeout(self):
- if self._child_handlers:
- if self._timeout_interval is None:
- timeout = self._sigchld_interval
+
+ with self._thread_rlock:
+ if self._child_handlers:
+ if self._timeout_interval is None:
+ timeout = self._sigchld_interval
+ else:
+ timeout = min(self._sigchld_interval,
+ self._timeout_interval)
else:
- timeout = min(self._sigchld_interval,
- self._timeout_interval)
- else:
- timeout = self._timeout_interval
+ timeout = self._timeout_interval
+
return timeout
def child_watch_add(self, pid, callback, data=None):
@@ -229,8 +252,7 @@ class EventLoop(object):
@rtype: int
@return: an integer ID
"""
- self._event_handler_id += 1
- source_id = self._event_handler_id
+ source_id = self._new_source_id()
self._child_handlers[source_id] = self._child_callback_class(
callback=callback, data=data, pid=pid, source_id=source_id)
@@ -304,20 +326,21 @@ class EventLoop(object):
"""
Like glib.idle_add(), if callback returns False it is
automatically removed from the list of event sources and will
- not be called again.
+ not be called again. This method is thread-safe.
@type callback: callable
@param callback: a function to call
@rtype: int
@return: an integer ID
"""
- self._event_handler_id += 1
- source_id = self._event_handler_id
- self._idle_callbacks[source_id] = self._idle_callback_class(
- args=args, callback=callback, source_id=source_id)
+ with self._thread_rlock:
+ source_id = self._new_source_id()
+ self._idle_callbacks[source_id] = self._idle_callback_class(
+ args=args, callback=callback, source_id=source_id)
return source_id
def _run_idle_callbacks(self):
+ # assumes caller has acquired self._thread_rlock
if not self._idle_callbacks:
return
# Iterate of our local list, since self._idle_callbacks can be
@@ -342,16 +365,18 @@ class EventLoop(object):
milliseconds between calls to your function, and your function
should return False to stop being called, or True to continue
being called. Any additional positional arguments given here
- are passed to your function when it's called.
+ are passed to your function when it's called. This method is
+ thread-safe.
"""
- self._event_handler_id += 1
- source_id = self._event_handler_id
- self._timeout_handlers[source_id] = \
- self._timeout_handler_class(
- interval=interval, function=function, args=args,
- source_id=source_id, timestamp=time.time())
- if self._timeout_interval is None or self._timeout_interval > interval:
- self._timeout_interval = interval
+ with self._thread_rlock:
+ source_id = self._new_source_id()
+ self._timeout_handlers[source_id] = \
+ self._timeout_handler_class(
+ interval=interval, function=function, args=args,
+ source_id=source_id, timestamp=time.time())
+ if self._timeout_interval is None or \
+ self._timeout_interval > interval:
+ self._timeout_interval = interval
return source_id
def _run_timeouts(self):
@@ -361,37 +386,39 @@ class EventLoop(object):
if self._poll_child_processes():
calls += 1
- self._run_idle_callbacks()
-
- if not self._timeout_handlers:
- return bool(calls)
-
- ready_timeouts = []
- current_time = time.time()
- for x in self._timeout_handlers.values():
- elapsed_seconds = current_time - x.timestamp
- # elapsed_seconds < 0 means the system clock has been adjusted
- if elapsed_seconds < 0 or \
- (x.interval - 1000 * elapsed_seconds) <= 0:
- ready_timeouts.append(x)
-
- # Iterate of our local list, since self._timeout_handlers can be
- # modified during the exection of these callbacks.
- for x in ready_timeouts:
- if x.source_id not in self._timeout_handlers:
- # it got cancelled while executing another timeout
- continue
- if x.calling:
- # don't call it recursively
- continue
- calls += 1
- x.calling = True
- try:
- x.timestamp = time.time()
- if not x.function(*x.args):
- self.source_remove(x.source_id)
- finally:
- x.calling = False
+ with self._thread_rlock:
+
+ self._run_idle_callbacks()
+
+ if not self._timeout_handlers:
+ return bool(calls)
+
+ ready_timeouts = []
+ current_time = time.time()
+ for x in self._timeout_handlers.values():
+ elapsed_seconds = current_time - x.timestamp
+ # elapsed_seconds < 0 means the system clock has been adjusted
+ if elapsed_seconds < 0 or \
+ (x.interval - 1000 * elapsed_seconds) <= 0:
+ ready_timeouts.append(x)
+
+ # Iterate of our local list, since self._timeout_handlers can be
+ # modified during the exection of these callbacks.
+ for x in ready_timeouts:
+ if x.source_id not in self._timeout_handlers:
+ # it got cancelled while executing another timeout
+ continue
+ if x.calling:
+ # don't call it recursively
+ continue
+ calls += 1
+ x.calling = True
+ try:
+ x.timestamp = time.time()
+ if not x.function(*x.args):
+ self.source_remove(x.source_id)
+ finally:
+ x.calling = False
return bool(calls)
@@ -413,8 +440,7 @@ class EventLoop(object):
"""
if f in self._poll_event_handlers:
raise AssertionError("fd %d is already registered" % f)
- self._event_handler_id += 1
- source_id = self._event_handler_id
+ source_id = self._new_source_id()
self._poll_event_handler_ids[source_id] = f
self._poll_event_handlers[f] = self._io_handler_class(
args=args, callback=callback, f=f, source_id=source_id)
@@ -434,18 +460,21 @@ class EventLoop(object):
self.source_remove(self._sigchld_src_id)
self._sigchld_src_id = None
return True
- idle_callback = self._idle_callbacks.pop(reg_id, None)
- if idle_callback is not None:
- return True
- timeout_handler = self._timeout_handlers.pop(reg_id, None)
- if timeout_handler is not None:
- if timeout_handler.interval == self._timeout_interval:
- if self._timeout_handlers:
- self._timeout_interval = \
- min(x.interval for x in self._timeout_handlers.values())
- else:
- self._timeout_interval = None
- return True
+
+ with self._thread_rlock:
+ idle_callback = self._idle_callbacks.pop(reg_id, None)
+ if idle_callback is not None:
+ return True
+ timeout_handler = self._timeout_handlers.pop(reg_id, None)
+ if timeout_handler is not None:
+ if timeout_handler.interval == self._timeout_interval:
+ if self._timeout_handlers:
+ self._timeout_interval = min(x.interval
+ for x in self._timeout_handlers.values())
+ else:
+ self._timeout_interval = None
+ return True
+
f = self._poll_event_handler_ids.pop(reg_id, None)
if f is None:
return False
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-12-28 1:36 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-12-28 1:36 UTC (permalink / raw
To: gentoo-commits
commit: d9148f4bbba31f23948f300f1d3256feed8defac
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 28 01:35:49 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Dec 28 01:35:49 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d9148f4b
EventLoop.iteration(): avoid busy waiting
In order to avoid blocking forever when may_block is True (the
default), callers must be careful to ensure that at least one of the
following conditions is met:
1) An event source or timeout is registered which is guaranteed
to trigger at least on event (a call to an idle function
only counts as an event if it returns a False value which
causes it to be stop being called)
2) Another thread is guaranteed to call one of the thread-safe
methods which notify iteration to stop waiting (such as
idle_add or timeout_add).
These rules ensure that iteration is able to block until an event
arrives, without doing any busy waiting that would waste CPU time.
This will fix busy waiting which would be triggered by
PopenPipeBlockingIOTestCase when waiting for the thread from
PipeReaderBlockingIO to call idle_add.
---
pym/portage/util/_eventloop/EventLoop.py | 54 ++++++++++++++++++++++--------
1 files changed, 40 insertions(+), 14 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index efd1f13..fea36c8 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -61,6 +61,7 @@ class EventLoop(object):
"""
self._use_signal = main and fcntl is not None
self._thread_rlock = threading.RLock()
+ self._thread_condition = threading.Condition(self._thread_rlock)
self._poll_event_queue = []
self._poll_event_handlers = {}
self._poll_event_handler_ids = {}
@@ -150,7 +151,19 @@ class EventLoop(object):
def iteration(self, *args):
"""
- Like glib.MainContext.iteration(), runs a single iteration.
+ Like glib.MainContext.iteration(), runs a single iteration. In order
+ to avoid blocking forever when may_block is True (the default),
+ callers must be careful to ensure that at least one of the following
+ conditions is met:
+ 1) An event source or timeout is registered which is guaranteed
+ to trigger at least on event (a call to an idle function
+ only counts as an event if it returns a False value which
+ causes it to be stop being called)
+ 2) Another thread is guaranteed to call one of the thread-safe
+ methods which notify iteration to stop waiting (such as
+ idle_add or timeout_add).
+ These rules ensure that iteration is able to block until an event
+ arrives, without doing any busy waiting that would waste CPU time.
@type may_block: bool
@param may_block: if True the call may block waiting for an event
(default is True).
@@ -171,19 +184,25 @@ class EventLoop(object):
events_handled = 0
if not event_handlers:
- if self._run_timeouts():
- events_handled += 1
- if not event_handlers and not events_handled and may_block:
- timeout = self._get_poll_timeout()
- if timeout is not None:
+ with self._thread_condition:
+ if self._run_timeouts():
+ events_handled += 1
+ if not event_handlers and not events_handled and may_block:
# Block so that we don't waste cpu time by looping too
# quickly. This makes EventLoop useful for code that needs
# to wait for timeout callbacks regardless of whether or
# not any IO handlers are currently registered.
- try:
- self._poll(timeout=timeout)
- except StopIteration:
- pass
+ timeout = self._get_poll_timeout()
+ if timeout is None:
+ wait_timeout = None
+ else:
+ wait_timeout = float(timeout) / 1000
+ # NOTE: In order to avoid a possible infinite wait when
+ # wait_timeout is None, the previous _run_timeouts()
+ # call must have returned False *with* _thread_condition
+ # acquired. Otherwise, we would risk going to sleep after
+ # our only notify even has already passed.
+ self._thread_condition.wait(wait_timeout)
if self._run_timeouts():
events_handled += 1
@@ -338,16 +357,18 @@ class EventLoop(object):
@rtype: int
@return: an integer ID
"""
- with self._thread_rlock:
+ with self._thread_condition:
source_id = self._new_source_id()
self._idle_callbacks[source_id] = self._idle_callback_class(
args=args, callback=callback, source_id=source_id)
+ self._thread_condition.notify()
return source_id
def _run_idle_callbacks(self):
# assumes caller has acquired self._thread_rlock
if not self._idle_callbacks:
- return
+ return False
+ state_change = 0
# Iterate of our local list, since self._idle_callbacks can be
# modified during the exection of these callbacks.
for x in list(self._idle_callbacks.values()):
@@ -360,10 +381,13 @@ class EventLoop(object):
x.calling = True
try:
if not x.callback(*x.args):
+ state_change += 1
self.source_remove(x.source_id)
finally:
x.calling = False
+ return bool(state_change)
+
def timeout_add(self, interval, function, *args):
"""
Like glib.timeout_add(), interval argument is the number of
@@ -373,7 +397,7 @@ class EventLoop(object):
are passed to your function when it's called. This method is
thread-safe.
"""
- with self._thread_rlock:
+ with self._thread_condition:
source_id = self._new_source_id()
self._timeout_handlers[source_id] = \
self._timeout_handler_class(
@@ -382,6 +406,7 @@ class EventLoop(object):
if self._timeout_interval is None or \
self._timeout_interval > interval:
self._timeout_interval = interval
+ self._thread_condition.notify()
return source_id
def _run_timeouts(self):
@@ -393,7 +418,8 @@ class EventLoop(object):
with self._thread_rlock:
- self._run_idle_callbacks()
+ if self._run_idle_callbacks():
+ calls += 1
if not self._timeout_handlers:
return bool(calls)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-12-28 1:44 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-12-28 1:44 UTC (permalink / raw
To: gentoo-commits
commit: a0f22daa7cf359aac776a45bbc60d22dcd947034
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 28 01:35:49 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Dec 28 01:44:27 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a0f22daa
EventLoop.iteration(): avoid busy waiting
In order to avoid blocking forever when may_block is True (the
default), callers must be careful to ensure that at least one of the
following conditions is met:
1) An event source or timeout is registered which is guaranteed
to trigger at least on event (a call to an idle function
only counts as an event if it returns a False value which
causes it to be stop being called)
2) Another thread is guaranteed to call one of the thread-safe
methods which notify iteration to stop waiting (such as
idle_add or timeout_add).
These rules ensure that iteration is able to block until an event
arrives, without doing any busy waiting that would waste CPU time.
This will fix busy waiting which would be triggered by
PopenPipeBlockingIOTestCase when waiting for the thread from
PipeReaderBlockingIO to call idle_add.
---
pym/portage/util/_eventloop/EventLoop.py | 54 ++++++++++++++++++++++--------
1 files changed, 40 insertions(+), 14 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index efd1f13..b77e819 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -61,6 +61,7 @@ class EventLoop(object):
"""
self._use_signal = main and fcntl is not None
self._thread_rlock = threading.RLock()
+ self._thread_condition = threading.Condition(self._thread_rlock)
self._poll_event_queue = []
self._poll_event_handlers = {}
self._poll_event_handler_ids = {}
@@ -150,7 +151,19 @@ class EventLoop(object):
def iteration(self, *args):
"""
- Like glib.MainContext.iteration(), runs a single iteration.
+ Like glib.MainContext.iteration(), runs a single iteration. In order
+ to avoid blocking forever when may_block is True (the default),
+ callers must be careful to ensure that at least one of the following
+ conditions is met:
+ 1) An event source or timeout is registered which is guaranteed
+ to trigger at least on event (a call to an idle function
+ only counts as an event if it returns a False value which
+ causes it to stop being called)
+ 2) Another thread is guaranteed to call one of the thread-safe
+ methods which notify iteration to stop waiting (such as
+ idle_add or timeout_add).
+ These rules ensure that iteration is able to block until an event
+ arrives, without doing any busy waiting that would waste CPU time.
@type may_block: bool
@param may_block: if True the call may block waiting for an event
(default is True).
@@ -171,19 +184,25 @@ class EventLoop(object):
events_handled = 0
if not event_handlers:
- if self._run_timeouts():
- events_handled += 1
- if not event_handlers and not events_handled and may_block:
- timeout = self._get_poll_timeout()
- if timeout is not None:
+ with self._thread_condition:
+ if self._run_timeouts():
+ events_handled += 1
+ if not event_handlers and not events_handled and may_block:
# Block so that we don't waste cpu time by looping too
# quickly. This makes EventLoop useful for code that needs
# to wait for timeout callbacks regardless of whether or
# not any IO handlers are currently registered.
- try:
- self._poll(timeout=timeout)
- except StopIteration:
- pass
+ timeout = self._get_poll_timeout()
+ if timeout is None:
+ wait_timeout = None
+ else:
+ wait_timeout = float(timeout) / 1000
+ # NOTE: In order to avoid a possible infinite wait when
+ # wait_timeout is None, the previous _run_timeouts()
+ # call must have returned False *with* _thread_condition
+ # acquired. Otherwise, we would risk going to sleep after
+ # our only notify event has already passed.
+ self._thread_condition.wait(wait_timeout)
if self._run_timeouts():
events_handled += 1
@@ -338,16 +357,18 @@ class EventLoop(object):
@rtype: int
@return: an integer ID
"""
- with self._thread_rlock:
+ with self._thread_condition:
source_id = self._new_source_id()
self._idle_callbacks[source_id] = self._idle_callback_class(
args=args, callback=callback, source_id=source_id)
+ self._thread_condition.notify()
return source_id
def _run_idle_callbacks(self):
# assumes caller has acquired self._thread_rlock
if not self._idle_callbacks:
- return
+ return False
+ state_change = 0
# Iterate of our local list, since self._idle_callbacks can be
# modified during the exection of these callbacks.
for x in list(self._idle_callbacks.values()):
@@ -360,10 +381,13 @@ class EventLoop(object):
x.calling = True
try:
if not x.callback(*x.args):
+ state_change += 1
self.source_remove(x.source_id)
finally:
x.calling = False
+ return bool(state_change)
+
def timeout_add(self, interval, function, *args):
"""
Like glib.timeout_add(), interval argument is the number of
@@ -373,7 +397,7 @@ class EventLoop(object):
are passed to your function when it's called. This method is
thread-safe.
"""
- with self._thread_rlock:
+ with self._thread_condition:
source_id = self._new_source_id()
self._timeout_handlers[source_id] = \
self._timeout_handler_class(
@@ -382,6 +406,7 @@ class EventLoop(object):
if self._timeout_interval is None or \
self._timeout_interval > interval:
self._timeout_interval = interval
+ self._thread_condition.notify()
return source_id
def _run_timeouts(self):
@@ -393,7 +418,8 @@ class EventLoop(object):
with self._thread_rlock:
- self._run_idle_callbacks()
+ if self._run_idle_callbacks():
+ calls += 1
if not self._timeout_handlers:
return bool(calls)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2012-12-31 3:16 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2012-12-31 3:16 UTC (permalink / raw
To: gentoo-commits
commit: 46fca763f7d08171d5c08c1c5fd1f40ce340ba56
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 31 03:16:11 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Dec 31 03:16:11 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=46fca763
EventLoop.iteration: no IO blocking if possible
Avoid blocking for IO if there are any timeout or idle callback events
available to process. This will prevent starvation of the idle
callbacks, which are much more common since commit
a3100be184ba1cac2f672f0a1cadcf01690c6d3f. We don't want these idle
callbacks to be delayed by long periods of IO blocking.
---
pym/portage/util/_eventloop/EventLoop.py | 22 ++++++++++++++++++----
1 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index b77e819..d2d4963 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -182,11 +182,13 @@ class EventLoop(object):
event_queue = self._poll_event_queue
event_handlers = self._poll_event_handlers
events_handled = 0
+ timeouts_checked = False
if not event_handlers:
with self._thread_condition:
if self._run_timeouts():
events_handled += 1
+ timeouts_checked = True
if not event_handlers and not events_handled and may_block:
# Block so that we don't waste cpu time by looping too
# quickly. This makes EventLoop useful for code that needs
@@ -205,6 +207,7 @@ class EventLoop(object):
self._thread_condition.wait(wait_timeout)
if self._run_timeouts():
events_handled += 1
+ timeouts_checked = True
# If any timeouts have executed, then return immediately,
# in order to minimize latency in termination of iteration
@@ -216,6 +219,17 @@ class EventLoop(object):
if may_block:
timeout = self._get_poll_timeout()
+
+ # Avoid blocking for IO if there are any timeout
+ # or idle callbacks available to process.
+ if timeout != 0 and not timeouts_checked:
+ if self._run_timeouts():
+ events_handled += 1
+ timeouts_checked = True
+ if events_handled:
+ # Minimize latency for loops controlled
+ # by timeout or idle callback events.
+ timeout = 0
else:
timeout = 0
@@ -235,10 +249,10 @@ class EventLoop(object):
if not x.callback(f, event, *x.args):
self.source_remove(x.source_id)
- # Run timeouts last, in order to minimize latency in
- # termination of iteration loops that they may control.
- if self._run_timeouts():
- events_handled += 1
+ if not timeouts_checked:
+ if self._run_timeouts():
+ events_handled += 1
+ timeouts_checked = True
return bool(events_handled)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2013-01-04 3:39 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2013-01-04 3:39 UTC (permalink / raw
To: gentoo-commits
commit: 65a5f14f8b76a250f5df42e57512bfc140543622
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 4 03:39:19 2013 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jan 4 03:39:19 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=65a5f14f
EventLoop: enable FD_CLOEXEC on epoll fd
---
pym/portage/util/_eventloop/EventLoop.py | 11 +++++++++++
1 files changed, 11 insertions(+), 0 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index d2d4963..6c2341b 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -84,6 +84,17 @@ class EventLoop(object):
# IOError: [Errno 38] Function not implemented
pass
else:
+
+ if fcntl is not None:
+ try:
+ fcntl.FD_CLOEXEC
+ except AttributeError:
+ pass
+ else:
+ fcntl.fcntl(epoll_obj.fileno(), fcntl.F_SETFL,
+ fcntl.fcntl(epoll_obj.fileno(),
+ fcntl.F_GETFL) | fcntl.FD_CLOEXEC)
+
self._poll_obj = _epoll_adapter(epoll_obj)
self.IO_ERR = select.EPOLLERR
self.IO_HUP = select.EPOLLHUP
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2013-02-25 23:53 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2013-02-25 23:53 UTC (permalink / raw
To: gentoo-commits
commit: 071440cc30cc5a625317d5bd47e86c2bfdd8a940
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 25 23:52:44 2013 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb 25 23:52:44 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=071440cc
EventLoop: qemu-user epoll quirk, bug #451326
The epoll implementation in qemu-user-1.2.2 appears to generate events
for file descriptors that are no longer registered. Handle it, since it
appears to be harmless (see bug #451326).
---
pym/portage/util/_eventloop/EventLoop.py | 8 +++++++-
1 files changed, 7 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index ad64406..3742055 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -256,7 +256,13 @@ class EventLoop(object):
while event_queue:
events_handled += 1
f, event = event_queue.pop()
- x = event_handlers[f]
+ try:
+ x = event_handlers[f]
+ except KeyError:
+ # This is known to be triggered by the epoll
+ # implementation in qemu-user-1.2.2, and appears
+ # to be harmless (see bug #451326).
+ continue
if not x.callback(f, event, *x.args):
self.source_remove(x.source_id)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2017-05-05 18:47 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2017-05-05 18:47 UTC (permalink / raw
To: gentoo-commits
commit: dac5089eb7908e9fd643f46c913515082077281e
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri May 5 09:07:38 2017 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri May 5 18:32:45 2017 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=dac5089e
Eventloop: fix deadlock involving idle_add/call_soon (bug 617550)
Guarantee that newly added idle_add/call_soon callbacks have an
opportunity to execute before the event loop decides to wait on
self._thread_condition without a timeout. This fixes a case where
the event loop would wait on self._thread_condition indefinitely,
even though a callback scheduled by the AsynchronousTask._async_wait
method needed to be executed first.
X-Gentoo-bug: 617550
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=617550
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>
pym/portage/util/_eventloop/EventLoop.py | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 712838e3d..cd154005f 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -108,6 +108,15 @@ class EventLoop(object):
self._poll_event_handler_ids = {}
# Increment id for each new handler.
self._event_handler_id = 0
+ # New call_soon callbacks must have an opportunity to
+ # execute before it's safe to wait on self._thread_condition
+ # without a timeout, since delaying its execution indefinitely
+ # could lead to a deadlock. The following attribute stores the
+ # event handler id of the most recently added call_soon callback.
+ # If this attribute has changed since the last time that the
+ # call_soon callbacks have been called, then it's not safe to
+ # wait on self._thread_condition without a timeout.
+ self._call_soon_id = 0
# Use OrderedDict in order to emulate the FIFO queue behavior
# of the AbstractEventLoop.call_soon method.
self._idle_callbacks = OrderedDict()
@@ -250,10 +259,15 @@ class EventLoop(object):
if not event_handlers:
with self._thread_condition:
+ prev_call_soon_id = self._call_soon_id
if self._run_timeouts():
events_handled += 1
timeouts_checked = True
- if not event_handlers and not events_handled and may_block:
+
+ call_soon = prev_call_soon_id != self._call_soon_id
+
+ if (not call_soon and not event_handlers
+ and not events_handled and may_block):
# Block so that we don't waste cpu time by looping too
# quickly. This makes EventLoop useful for code that needs
# to wait for timeout callbacks regardless of whether or
@@ -457,7 +471,7 @@ class EventLoop(object):
@return: an integer ID
"""
with self._thread_condition:
- source_id = self._new_source_id()
+ source_id = self._call_soon_id = self._new_source_id()
self._idle_callbacks[source_id] = self._idle_callback_class(
args=args, callback=callback, source_id=source_id)
self._thread_condition.notify()
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-07 20:20 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-07 20:20 UTC (permalink / raw
To: gentoo-commits
commit: db3d216676831d8b6c184fb25f94dc54f8710a11
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 7 20:19:26 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 7 20:20:15 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=db3d2166
GlibEventLoop: remove (unused and unmaintained)
pym/portage/util/_eventloop/GlibEventLoop.py | 23 -----------------------
pym/portage/util/_eventloop/global_event_loop.py | 1 -
2 files changed, 24 deletions(-)
diff --git a/pym/portage/util/_eventloop/GlibEventLoop.py b/pym/portage/util/_eventloop/GlibEventLoop.py
deleted file mode 100644
index f2f5c5e64..000000000
--- a/pym/portage/util/_eventloop/GlibEventLoop.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Copyright 2012 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-class GlibEventLoop(object):
-
- # TODO: Support multiprocessing by using a separate glib.MainContext
- # instance for each process.
- supports_multiprocessing = False
-
- def __init__(self):
- import gi.repository.GLib as glib
- self.IO_ERR = glib.IO_ERR
- self.IO_HUP = glib.IO_HUP
- self.IO_IN = glib.IO_IN
- self.IO_NVAL = glib.IO_NVAL
- self.IO_OUT = glib.IO_OUT
- self.IO_PRI = glib.IO_PRI
- self.iteration = glib.main_context_default().iteration
- self.child_watch_add = glib.child_watch_add
- self.idle_add = glib.idle_add
- self.io_add_watch = glib.io_add_watch
- self.timeout_add = glib.timeout_add
- self.source_remove = glib.source_remove
diff --git a/pym/portage/util/_eventloop/global_event_loop.py b/pym/portage/util/_eventloop/global_event_loop.py
index 502dab882..e2c7d71ea 100644
--- a/pym/portage/util/_eventloop/global_event_loop.py
+++ b/pym/portage/util/_eventloop/global_event_loop.py
@@ -6,7 +6,6 @@ import os
from .EventLoop import EventLoop
_default_constructor = EventLoop
-#from .GlibEventLoop import GlibEventLoop as _default_constructor
# If _default_constructor doesn't support multiprocessing,
# then _multiprocessing_constructor is used in subprocesses.
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-08 21:18 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-08 21:18 UTC (permalink / raw
To: gentoo-commits
commit: 13621b62e32a7c21aa08247b33f1faa0a146d0d4
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 8 21:15:39 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 8 21:15:39 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=13621b62
EventLoop: add get/set_debug methods for asyncio compat (bug 591760)
Bug: https://bugs.gentoo.org/591760
pym/portage/util/_eventloop/EventLoop.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 1574a6837..4ec67241f 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -102,6 +102,7 @@ class EventLoop(object):
@type main: bool
"""
self._use_signal = main and fcntl is not None
+ self._debug = bool(os.environ.get('PYTHONASYNCIODEBUG'))
self._thread_rlock = threading.RLock()
self._thread_condition = threading.Condition(self._thread_rlock)
self._poll_event_queue = []
@@ -763,6 +764,19 @@ class EventLoop(object):
close()
self._poll_obj = None
+ def get_debug(self):
+ """
+ Get the debug mode (bool) of the event loop.
+
+ The default value is True if the environment variable
+ PYTHONASYNCIODEBUG is set to a non-empty string, False otherwise.
+ """
+ return self._debug
+
+ def set_debug(self, enabled):
+ """Set the debug mode of the event loop."""
+ self._debug = enabled
+
_can_poll_device = None
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-08 21:37 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-08 21:37 UTC (permalink / raw
To: gentoo-commits
commit: 754010f346ec2455ea8c71a6af4796c10fd28d23
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 8 21:36:07 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 8 21:36:07 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=754010f3
EventLoop: add is_closed method for asyncio compat (bug 591760)
Bug: https://bugs.gentoo.org/591760
pym/portage/util/_eventloop/EventLoop.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 4ec67241f..1bf606354 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -747,6 +747,10 @@ class EventLoop(object):
self._default_executor = executor
return executor.submit(func, *args)
+ def is_closed(self):
+ """Returns True if the event loop was closed."""
+ return self._poll_obj is None
+
def close(self):
"""Close the event loop.
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-08 22:08 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-08 22:08 UTC (permalink / raw
To: gentoo-commits
commit: 24f861173ebe747a470deb8489887c067cd46b0f
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 2 03:46:10 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 8 22:04:37 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=24f86117
EventLoop: implement add/remove_reader/writer for asyncio compat (bug 649588)
Bug: https://bugs.gentoo.org/649588
pym/portage/util/_eventloop/EventLoop.py | 51 ++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 1bf606354..00568c997 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -93,6 +93,21 @@ class EventLoop(object):
self._callback(*self._args)
return False
+ class _repeat_callback(object):
+ """
+ Wraps an callback, and always returns True, for callbacks that
+ are supposed to run repeatedly.
+ """
+ __slots__ = ("_args", "_callback")
+
+ def __init__(self, callback, args):
+ self._callback = callback
+ self._args = args
+
+ def __call__(self, fd, event):
+ self._callback(*self._args)
+ return True
+
def __init__(self, main=True):
"""
@param main: If True then this is a singleton instance for use
@@ -569,6 +584,42 @@ class EventLoop(object):
return bool(calls)
+ def add_reader(self, fd, callback, *args):
+ """
+ Start watching the file descriptor for read availability and then
+ call the callback with specified arguments.
+
+ Use functools.partial to pass keywords to the callback.
+ """
+ self.io_add_watch(fd, self.IO_IN, self._repeat_callback(callback, args))
+
+ def remove_reader(self, fd):
+ """
+ Stop watching the file descriptor for read availability.
+ """
+ handler = self._poll_event_handlers.get(fd)
+ if fd is not None:
+ return self.source_remove(handler.source_id)
+ return False
+
+ def add_writer(self, fd, callback, *args):
+ """
+ Start watching the file descriptor for write availability and then
+ call the callback with specified arguments.
+
+ Use functools.partial to pass keywords to the callback.
+ """
+ self.io_add_watch(fd, self.IO_OUT, self._repeat_callback(callback, args))
+
+ def remove_writer(self, fd):
+ """
+ Stop watching the file descriptor for write availability.
+ """
+ handler = self._poll_event_handlers.get(fd)
+ if fd is not None:
+ return self.source_remove(handler.source_id)
+ return False
+
def io_add_watch(self, f, condition, callback, *args):
"""
Like glib.io_add_watch(), your function should return False to
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-09 1:27 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-09 1:27 UTC (permalink / raw
To: gentoo-commits
commit: 3b1c182750d82dc55b96ee111e356968ca9a9fb7
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 9 01:25:25 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 9 01:25:52 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=3b1c1827
EventLoop: add call_at method for asyncio compat (bug 591760)
Bug: https://bugs.gentoo.org/591760
pym/portage/util/_eventloop/EventLoop.py | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 00568c997..72eb407fc 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -775,6 +775,29 @@ class EventLoop(object):
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback, args)), self)
+ def call_at(self, when, callback, *args):
+ """
+ Arrange for the callback to be called at the given absolute
+ timestamp when (an int or float), using the same time reference as
+ AbstractEventLoop.time().
+
+ This method's behavior is the same as call_later().
+
+ An instance of asyncio.Handle is returned, which can be used to
+ cancel the callback.
+
+ Use functools.partial to pass keywords to the callback.
+
+ @type when: int or float
+ @param when: absolute timestamp when to call callback
+ @type callback: callable
+ @param callback: a function to call
+ @return: a handle which can be used to cancel the callback
+ @rtype: asyncio.Handle (or compatible)
+ """
+ delta = when - self.time()
+ return self.call_later(delta if delta > 0 else 0, callback, *args)
+
def run_in_executor(self, executor, func, *args):
"""
Arrange for a func to be called in the specified executor.
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-15 22:32 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-15 22:32 UTC (permalink / raw
To: gentoo-commits
commit: 18d8abb063d7730fbb86d451489dc2acf36c1327
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 15 22:25:03 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 15 22:31:07 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=18d8abb0
EventLoop: fix AttributeError in add/remove_reader
Fixes: 24f861173ebe ("EventLoop: implement add/remove_reader/writer for asyncio compat (bug 649588)")
pym/portage/util/_eventloop/EventLoop.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index a928f3138..32dc2fc9d 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -609,7 +609,7 @@ class EventLoop(object):
Stop watching the file descriptor for read availability.
"""
handler = self._poll_event_handlers.get(fd)
- if fd is not None:
+ if handler is not None:
return self.source_remove(handler.source_id)
return False
@@ -627,7 +627,7 @@ class EventLoop(object):
Stop watching the file descriptor for write availability.
"""
handler = self._poll_event_handlers.get(fd)
- if fd is not None:
+ if handler is not None:
return self.source_remove(handler.source_id)
return False
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-16 0:03 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-16 0:03 UTC (permalink / raw
To: gentoo-commits
commit: 2b6e90fadfb1adcd8ccd2f313aa009b3d19ffefe
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 15 23:42:49 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 15 23:58:05 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2b6e90fa
EventLoop: support add_reader/writer fd overlap (bug 649588)
The AbstractEventLoop add_reader and add_writer methods need to support
simultaneous registration of reader and writer callbacks for the same
fd. For example, this feature is used by the standard library's
asyncio.unix_events._UnixWritePipeTransport class, which is used to
implement AbstractEventLoop.subprocess_exec(stdin=subprocess.PIPE).
Bug: https://bugs.gentoo.org/649588
pym/portage/util/_eventloop/EventLoop.py | 83 ++++++++++++++++++++++++++++----
1 file changed, 73 insertions(+), 10 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 32dc2fc9d..35d0a35ba 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -4,6 +4,7 @@
from __future__ import division
import errno
+import functools
import logging
import os
import select
@@ -95,19 +96,20 @@ class EventLoop(object):
self._callback(*self._args)
return False
- class _repeat_callback(object):
+ class _selector_callback(object):
"""
Wraps an callback, and always returns True, for callbacks that
are supposed to run repeatedly.
"""
- __slots__ = ("_args", "_callback")
+ __slots__ = ("_args", "_callbacks")
- def __init__(self, callback, args):
- self._callback = callback
- self._args = args
+ def __init__(self, callbacks):
+ self._callbacks = callbacks
def __call__(self, fd, event):
- self._callback(*self._args)
+ for callback, mask in self._callbacks:
+ if event & mask:
+ callback()
return True
def __init__(self, main=True):
@@ -189,6 +191,9 @@ class EventLoop(object):
self.IO_OUT = PollConstants.POLLOUT
self.IO_PRI = PollConstants.POLLPRI
+ self._EVENT_READ = self.IO_IN | self.IO_HUP
+ self._EVENT_WRITE = self.IO_OUT
+
self._child_handlers = {}
self._sigchld_read = None
self._sigchld_write = None
@@ -602,7 +607,19 @@ class EventLoop(object):
Use functools.partial to pass keywords to the callback.
"""
- self.io_add_watch(fd, self.IO_IN, self._repeat_callback(callback, args))
+ handler = self._poll_event_handlers.get(fd)
+ callbacks = [(functools.partial(callback, *args), self._EVENT_READ)]
+ selector_mask = self._EVENT_READ
+ if handler is not None:
+ if not isinstance(handler.callback, self._selector_callback):
+ raise AssertionError("add_reader called with fd "
+ "registered directly via io_add_watch")
+ for item in handler.callback._callbacks:
+ callback, mask = item
+ if mask != self._EVENT_READ:
+ selector_mask |= mask
+ callbacks.append(item)
+ self.io_add_watch(fd, selector_mask, self._selector_callback(callbacks))
def remove_reader(self, fd):
"""
@@ -610,7 +627,24 @@ class EventLoop(object):
"""
handler = self._poll_event_handlers.get(fd)
if handler is not None:
- return self.source_remove(handler.source_id)
+ if not isinstance(handler.callback, self._selector_callback):
+ raise AssertionError("remove_reader called with fd "
+ "registered directly via io_add_watch")
+ callbacks = []
+ selector_mask = 0
+ removed = False
+ for item in handler.callback._callbacks:
+ callback, mask = item
+ if mask == self._EVENT_READ:
+ removed = True
+ else:
+ selector_mask |= mask
+ callbacks.append(item)
+ self.source_remove(handler.source_id)
+ if callbacks:
+ self.io_add_watch(fd, selector_mask,
+ self._selector_callback(callbacks))
+ return removed
return False
def add_writer(self, fd, callback, *args):
@@ -620,7 +654,19 @@ class EventLoop(object):
Use functools.partial to pass keywords to the callback.
"""
- self.io_add_watch(fd, self.IO_OUT, self._repeat_callback(callback, args))
+ handler = self._poll_event_handlers.get(fd)
+ callbacks = [(functools.partial(callback, *args), self._EVENT_WRITE)]
+ selector_mask = self._EVENT_WRITE
+ if handler is not None:
+ if not isinstance(handler.callback, self._selector_callback):
+ raise AssertionError("add_reader called with fd "
+ "registered directly via io_add_watch")
+ for item in handler.callback._callbacks:
+ callback, mask = item
+ if mask != self._EVENT_WRITE:
+ selector_mask |= mask
+ callbacks.append(item)
+ self.io_add_watch(fd, selector_mask, self._selector_callback(callbacks))
def remove_writer(self, fd):
"""
@@ -628,7 +674,24 @@ class EventLoop(object):
"""
handler = self._poll_event_handlers.get(fd)
if handler is not None:
- return self.source_remove(handler.source_id)
+ if not isinstance(handler.callback, self._selector_callback):
+ raise AssertionError("remove_reader called with fd "
+ "registered directly via io_add_watch")
+ callbacks = []
+ selector_mask = 0
+ removed = False
+ for item in handler.callback._callbacks:
+ callback, mask = item
+ if mask == self._EVENT_WRITE:
+ removed = True
+ else:
+ selector_mask |= mask
+ callbacks.append(item)
+ self.source_remove(handler.source_id)
+ if callbacks:
+ self.io_add_watch(fd, selector_mask,
+ self._selector_callback(callbacks))
+ return removed
return False
def io_add_watch(self, f, condition, callback, *args):
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-16 6:43 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-16 6:43 UTC (permalink / raw
To: gentoo-commits
commit: dc7faa37c3b5b9cbb9736d4b1669510db1b953d6
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 16 06:40:16 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 16 06:40:16 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=dc7faa37
EventLoop: fix add_reader/writer to call source_remove
The previous instance of self._selector_callback has to be removed
before replacing it with a new instance.
Fixes: 2b6e90fadfb1 ("EventLoop: support add_reader/writer fd overlap (bug 649588)")
pym/portage/util/_eventloop/EventLoop.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 0024d162c..38e735999 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -620,6 +620,7 @@ class EventLoop(object):
if mask != self._EVENT_READ:
selector_mask |= mask
callbacks.append(item)
+ self.source_remove(handler.source_id)
self.io_add_watch(fd, selector_mask, self._selector_callback(callbacks))
def remove_reader(self, fd):
@@ -667,6 +668,7 @@ class EventLoop(object):
if mask != self._EVENT_WRITE:
selector_mask |= mask
callbacks.append(item)
+ self.source_remove(handler.source_id)
self.io_add_watch(fd, selector_mask, self._selector_callback(callbacks))
def remove_writer(self, fd):
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-16 8:48 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-16 8:48 UTC (permalink / raw
To: gentoo-commits
commit: 1ee8971ba1cb34e6b3cd3d5fda23066b24630e3a
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 16 08:29:36 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 16 08:46:21 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=1ee8971b
EventLoop: eliminate thread safety from call_soon
The call_soon method is used heavily by asyncio.Task to execute
coroutine steps, so it's important to eliminate the overhead
associated with thread safety.
pym/portage/util/_eventloop/EventLoop.py | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 38e735999..4ef600a5b 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -506,12 +506,17 @@ class EventLoop(object):
@return: an integer ID
"""
with self._thread_condition:
- source_id = self._call_soon_id = self._new_source_id()
- self._idle_callbacks[source_id] = self._idle_callback_class(
- args=args, callback=callback, source_id=source_id)
+ source_id = self._idle_add(callback, *args)
self._thread_condition.notify()
return source_id
+ def _idle_add(self, callback, *args):
+ """Like idle_add(), but without thread safety."""
+ source_id = self._call_soon_id = self._new_source_id()
+ self._idle_callbacks[source_id] = self._idle_callback_class(
+ args=args, callback=callback, source_id=source_id)
+ return source_id
+
def _run_idle_callbacks(self):
# assumes caller has acquired self._thread_rlock
if not self._idle_callbacks:
@@ -810,11 +815,14 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
- return self._handle(self.idle_add(
+ return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
- # The call_soon method inherits thread safety from the idle_add method.
- call_soon_threadsafe = call_soon
+ def call_soon_threadsafe(self, callback, *args):
+ """Like call_soon(), but thread safe."""
+ # idle_add provides thread safety
+ return self._handle(self.idle_add(
+ self._call_soon_callback(callback, args)), self)
def time(self):
"""Return the time according to the event loop's clock.
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-17 0:53 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-17 0:53 UTC (permalink / raw
To: gentoo-commits
commit: 9772f8f2a58a858a80ad1542d1ce46193616be67
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 16 23:55:14 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 17 00:49:09 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=9772f8f2
EventLoop._idle_add: use thread-safe deque append
This fixes previously unsafe usage of self._idle_callbacks when it was
a dictionary. The deque append is thread-safe, but it does *not* notify
the loop's thread, so the caller must notify if appropriate.
Fixes: 1ee8971ba1cb ("EventLoop: eliminate thread safety from call_soon")
pym/portage/util/_eventloop/EventLoop.py | 90 ++++++++++++++++++++------------
1 file changed, 58 insertions(+), 32 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index ae5a0a70a..d4f20c6ed 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -3,6 +3,7 @@
from __future__ import division
+import collections
import errno
import functools
import logging
@@ -30,7 +31,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.futures.unix_events:_PortageEventLoop,_PortageChildWatcher',
)
-from portage import OrderedDict
from portage.util import writemsg_level
from portage.util.monotonic import monotonic
from ..SlotObject import SlotObject
@@ -55,7 +55,7 @@ class EventLoop(object):
__slots__ = ("callback", "data", "pid", "source_id")
class _idle_callback_class(SlotObject):
- __slots__ = ("args", "callback", "calling", "source_id")
+ __slots__ = ("_args", "_callback", "_calling", "_cancelled")
class _io_handler_class(SlotObject):
__slots__ = ("args", "callback", "f", "source_id")
@@ -141,10 +141,10 @@ class EventLoop(object):
# If this attribute has changed since the last time that the
# call_soon callbacks have been called, then it's not safe to
# wait on self._thread_condition without a timeout.
- self._call_soon_id = 0
- # Use OrderedDict in order to emulate the FIFO queue behavior
- # of the AbstractEventLoop.call_soon method.
- self._idle_callbacks = OrderedDict()
+ self._call_soon_id = None
+ # Use deque, with thread-safe append, in order to emulate the FIFO
+ # queue behavior of the AbstractEventLoop.call_soon method.
+ self._idle_callbacks = collections.deque()
self._timeout_handlers = {}
self._timeout_interval = None
self._default_executor = None
@@ -298,7 +298,10 @@ class EventLoop(object):
events_handled += 1
timeouts_checked = True
- call_soon = prev_call_soon_id != self._call_soon_id
+ call_soon = prev_call_soon_id is not self._call_soon_id
+ if self._call_soon_id is not None and self._call_soon_id._cancelled:
+ # Allow garbage collection of cancelled callback.
+ self._call_soon_id = None
if (not call_soon and not event_handlers
and not events_handled and may_block):
@@ -501,8 +504,9 @@ class EventLoop(object):
@type callback: callable
@param callback: a function to call
- @rtype: int
- @return: an integer ID
+ @return: a handle which can be used to cancel the callback
+ via the source_remove method
+ @rtype: object
"""
with self._thread_condition:
source_id = self._idle_add(callback, *args)
@@ -511,32 +515,51 @@ class EventLoop(object):
def _idle_add(self, callback, *args):
"""Like idle_add(), but without thread safety."""
- source_id = self._call_soon_id = self._new_source_id()
- self._idle_callbacks[source_id] = self._idle_callback_class(
- args=args, callback=callback, source_id=source_id)
- return source_id
+ # Hold self._thread_condition when assigning self._call_soon_id,
+ # since it might be modified via a thread-safe method.
+ with self._thread_condition:
+ handle = self._call_soon_id = self._idle_callback_class(
+ _args=args, _callback=callback)
+ # This deque append is thread-safe, but it does *not* notify the
+ # loop's thread, so the caller must notify if appropriate.
+ self._idle_callbacks.append(handle)
+ return handle
def _run_idle_callbacks(self):
# assumes caller has acquired self._thread_rlock
if not self._idle_callbacks:
return False
state_change = 0
- # Iterate of our local list, since self._idle_callbacks can be
- # modified during the exection of these callbacks.
- for x in list(self._idle_callbacks.values()):
- if x.source_id not in self._idle_callbacks:
- # it got cancelled while executing another callback
- continue
- if x.calling:
- # don't call it recursively
- continue
- x.calling = True
- try:
- if not x.callback(*x.args):
- state_change += 1
- self.source_remove(x.source_id)
- finally:
- x.calling = False
+ reschedule = []
+ # Use remaining count to avoid calling any newly scheduled callbacks,
+ # since self._idle_callbacks can be modified during the exection of
+ # these callbacks.
+ remaining = len(self._idle_callbacks)
+ try:
+ while remaining:
+ remaining -= 1
+ try:
+ x = self._idle_callbacks.popleft() # thread-safe
+ except IndexError:
+ break
+ if x._cancelled:
+ # it got cancelled while executing another callback
+ continue
+ if x._calling:
+ # don't call it recursively
+ continue
+ x._calling = True
+ try:
+ if x._callback(*x._args):
+ reschedule.append(x)
+ else:
+ x._cancelled = True
+ state_change += 1
+ finally:
+ x._calling = False
+ finally:
+ # Reschedule those that were not cancelled.
+ self._idle_callbacks.extend(reschedule)
return bool(state_change)
@@ -732,6 +755,12 @@ class EventLoop(object):
is found and removed, and False if the reg_id is invalid or has
already been removed.
"""
+ if isinstance(reg_id, self._idle_callback_class):
+ if not reg_id._cancelled:
+ reg_id._cancelled = True
+ return True
+ return False
+
x = self._child_handlers.pop(reg_id, None)
if x is not None:
if not self._child_handlers and self._use_signal:
@@ -741,9 +770,6 @@ class EventLoop(object):
return True
with self._thread_rlock:
- idle_callback = self._idle_callbacks.pop(reg_id, None)
- if idle_callback is not None:
- return True
timeout_handler = self._timeout_handlers.pop(reg_id, None)
if timeout_handler is not None:
if timeout_handler.interval == self._timeout_interval:
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-17 6:24 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-17 6:24 UTC (permalink / raw
To: gentoo-commits
commit: 0f8e3cd3cc695e721a8b1f7cfc56c53aca19fe4d
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 17 06:20:45 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 17 06:20:45 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=0f8e3cd3
EventLoop._run_idle_callbacks: make recursive callbacks fatal
The caller should use call_soon in order to prevent recursion
here. Raise an error because recursive calls would make the
remaining count for this loop meaningless.
Fixes: 9772f8f2a58a (EventLoop._idle_add: use thread-safe deque append)
pym/portage/util/_eventloop/EventLoop.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index d4f20c6ed..c38a4defd 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -546,8 +546,11 @@ class EventLoop(object):
# it got cancelled while executing another callback
continue
if x._calling:
- # don't call it recursively
- continue
+ # The caller should use call_soon in order to prevent
+ # recursion here. Raise an error because recursive
+ # calls would make the remaining count for this loop
+ # meaningless.
+ raise AssertionError('recursive idle callback')
x._calling = True
try:
if x._callback(*x._args):
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-17 18:02 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-17 18:02 UTC (permalink / raw
To: gentoo-commits
commit: 0f7c9a73a805af5ec70da587b3c7d7f59dabe5ce
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 17 17:57:04 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 17 17:59:10 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=0f7c9a73
EventLoop._run_idle_callbacks: remove recursion check
This recursion check does not really work because callbacks are
removed from self._idle_callbacks before recursion would occur.
Fixes: 9772f8f2a58a (EventLoop._idle_add: use thread-safe deque append)
pym/portage/util/_eventloop/EventLoop.py | 22 ++++++----------------
1 file changed, 6 insertions(+), 16 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index c38a4defd..a61a3d74a 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -55,7 +55,7 @@ class EventLoop(object):
__slots__ = ("callback", "data", "pid", "source_id")
class _idle_callback_class(SlotObject):
- __slots__ = ("_args", "_callback", "_calling", "_cancelled")
+ __slots__ = ("_args", "_callback", "_cancelled")
class _io_handler_class(SlotObject):
__slots__ = ("args", "callback", "f", "source_id")
@@ -545,21 +545,11 @@ class EventLoop(object):
if x._cancelled:
# it got cancelled while executing another callback
continue
- if x._calling:
- # The caller should use call_soon in order to prevent
- # recursion here. Raise an error because recursive
- # calls would make the remaining count for this loop
- # meaningless.
- raise AssertionError('recursive idle callback')
- x._calling = True
- try:
- if x._callback(*x._args):
- reschedule.append(x)
- else:
- x._cancelled = True
- state_change += 1
- finally:
- x._calling = False
+ if x._callback(*x._args):
+ reschedule.append(x)
+ else:
+ x._cancelled = True
+ state_change += 1
finally:
# Reschedule those that were not cancelled.
self._idle_callbacks.extend(reschedule)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-17 18:24 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-17 18:24 UTC (permalink / raw
To: gentoo-commits
commit: d36f8b2c9c43311f4c1333afe7ce1cc7a147b836
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 17 18:19:02 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 17 18:19:02 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=d36f8b2c
EventLoop._run_idle_callbacks: make recursive calls fatal
The caller should use call_soon in order to prevent recursion
here. Raise an error because _run_idle_callbacks has an internal
remaining count that recursion would render meaningless.
Fixes: 9772f8f2a58a (EventLoop._idle_add: use thread-safe deque append)
pym/portage/util/_eventloop/EventLoop.py | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index a61a3d74a..895303699 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -145,6 +145,7 @@ class EventLoop(object):
# Use deque, with thread-safe append, in order to emulate the FIFO
# queue behavior of the AbstractEventLoop.call_soon method.
self._idle_callbacks = collections.deque()
+ self._idle_callbacks_running = False
self._timeout_handlers = {}
self._timeout_interval = None
self._default_executor = None
@@ -586,8 +587,19 @@ class EventLoop(object):
with self._thread_rlock:
- if self._run_idle_callbacks():
- calls += 1
+ if self._idle_callbacks_running:
+ # The caller should use call_soon in order to
+ # prevent recursion here. Raise an error because
+ # _run_idle_callbacks has an internal remaining
+ # count that recursion would render meaningless.
+ raise AssertionError('idle callback recursion')
+
+ self._idle_callbacks_running = True
+ try:
+ if self._run_idle_callbacks():
+ calls += 1
+ finally:
+ self._idle_callbacks_running = False
if not self._timeout_handlers:
return bool(calls)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-18 21:52 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-18 21:52 UTC (permalink / raw
To: gentoo-commits
commit: b443b87c5397867c287f3bc4c5f1f4fa5e234d0a
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 18 21:38:33 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Apr 18 21:49:42 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=b443b87c
EventLoop._run_idle_callbacks: support recursive calls (bug 653508)
Since recursive calls must be supported until all consumers of the
AsynchronousLock.unlock() method have been migrated to use the
async_unlock() method (bug 614108.), support recursive calls by using
a self._idle_callbacks_remaining loop control variable that is reset
by each recursive call.
Fixes: 9772f8f2a58a (EventLoop._idle_add: use thread-safe deque append)
Bug: https://bugs.gentoo.org/653508
pym/portage/util/_eventloop/EventLoop.py | 60 ++++++++++++++------------------
1 file changed, 26 insertions(+), 34 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 895303699..7208c3aa1 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -145,7 +145,7 @@ class EventLoop(object):
# Use deque, with thread-safe append, in order to emulate the FIFO
# queue behavior of the AbstractEventLoop.call_soon method.
self._idle_callbacks = collections.deque()
- self._idle_callbacks_running = False
+ self._idle_callbacks_remaining = 0
self._timeout_handlers = {}
self._timeout_interval = None
self._default_executor = None
@@ -534,26 +534,29 @@ class EventLoop(object):
reschedule = []
# Use remaining count to avoid calling any newly scheduled callbacks,
# since self._idle_callbacks can be modified during the exection of
- # these callbacks.
- remaining = len(self._idle_callbacks)
- try:
- while remaining:
- remaining -= 1
- try:
- x = self._idle_callbacks.popleft() # thread-safe
- except IndexError:
- break
- if x._cancelled:
- # it got cancelled while executing another callback
- continue
- if x._callback(*x._args):
- reschedule.append(x)
- else:
- x._cancelled = True
- state_change += 1
- finally:
- # Reschedule those that were not cancelled.
- self._idle_callbacks.extend(reschedule)
+ # these callbacks. The remaining count can be reset by recursive
+ # calls to this method. Recursion must remain supported until all
+ # consumers of AsynchronousLock.unlock() have been migrated to the
+ # async_unlock() method, see bug 614108.
+ self._idle_callbacks_remaining = len(self._idle_callbacks)
+
+ while self._idle_callbacks_remaining:
+ self._idle_callbacks_remaining -= 1
+ try:
+ x = self._idle_callbacks.popleft() # thread-safe
+ except IndexError:
+ break
+ if x._cancelled:
+ # it got cancelled while executing another callback
+ continue
+ if x._callback(*x._args):
+ # Reschedule, but not until after it's called, since
+ # we don't want it to call itself in a recursive call
+ # to this method.
+ self._idle_callbacks.append(x)
+ else:
+ x._cancelled = True
+ state_change += 1
return bool(state_change)
@@ -587,19 +590,8 @@ class EventLoop(object):
with self._thread_rlock:
- if self._idle_callbacks_running:
- # The caller should use call_soon in order to
- # prevent recursion here. Raise an error because
- # _run_idle_callbacks has an internal remaining
- # count that recursion would render meaningless.
- raise AssertionError('idle callback recursion')
-
- self._idle_callbacks_running = True
- try:
- if self._run_idle_callbacks():
- calls += 1
- finally:
- self._idle_callbacks_running = False
+ if self._run_idle_callbacks():
+ calls += 1
if not self._timeout_handlers:
return bool(calls)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-19 6:16 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-19 6:16 UTC (permalink / raw
To: gentoo-commits
commit: 25245d7eb86ed197b7d7cfead0dbe4ce8ad4bc5b
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 19 06:13:33 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Apr 19 06:13:33 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=25245d7e
EventLoop.run_until_complete: wait for done callbacks
Since done callbacks are executed via call_soon, it's desirable to
continue iterating until those callbacks have executed, which is easily
achieved by registering a done callback and waiting for it to execute.
pym/portage/util/_eventloop/EventLoop.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 7208c3aa1..33fae26f2 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -812,7 +812,14 @@ class EventLoop(object):
@raise: the Future's exception
"""
future = asyncio.ensure_future(future, loop=self._asyncio_wrapper)
- while not future.done():
+
+ # Since done callbacks are executed via call_soon, it's desirable
+ # to continue iterating until those callbacks have executed, which
+ # is easily achieved by registering a done callback and waiting for
+ # it to execute.
+ waiter = self.create_future()
+ future.add_done_callback(functools.partial(waiter.set_result))
+ while not waiter.done():
self.iteration()
return future.result()
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-04-19 6:41 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-04-19 6:41 UTC (permalink / raw
To: gentoo-commits
commit: da5efb5e677d12c850ff02140e24a095a8efdafb
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 19 06:36:32 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Apr 19 06:36:32 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=da5efb5e
EventLoop.run_until_complete: remove unneeded partial
Fixes: 25245d7eb86e ("EventLoop.run_until_complete: wait for done callbacks")
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 33fae26f2..6a8b906ed 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -818,7 +818,7 @@ class EventLoop(object):
# is easily achieved by registering a done callback and waiting for
# it to execute.
waiter = self.create_future()
- future.add_done_callback(functools.partial(waiter.set_result))
+ future.add_done_callback(waiter.set_result)
while not waiter.done():
self.iteration()
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-05-07 0:27 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-05-07 0:27 UTC (permalink / raw
To: gentoo-commits
commit: a1360e017b7ee8156ad4ad850e2f8ea40228ca1a
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 7 00:06:07 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 7 00:19:54 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=a1360e01
EventLoop.run_in_executor: use asyncio.wrap_future
Since executors from the concurrent.futures package return
concurrent.futures.Future, it's necessary to wrap them.
pym/portage/util/_eventloop/EventLoop.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index fc7380b03..de0795224 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -13,6 +13,11 @@ import signal
import sys
import traceback
+try:
+ import asyncio as _real_asyncio
+except ImportError:
+ _real_asyncio = None
+
try:
import fcntl
except ImportError:
@@ -937,7 +942,11 @@ class EventLoop(object):
if executor is None:
executor = ForkExecutor(loop=self)
self._default_executor = executor
- return executor.submit(func, *args)
+ future = executor.submit(func, *args)
+ if _real_asyncio is not None:
+ future = _real_asyncio.wrap_future(future,
+ loop=self._asyncio_wrapper)
+ return future
def is_running(self):
"""Return whether the event loop is currently running."""
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-05-13 16:58 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-05-13 16:58 UTC (permalink / raw
To: gentoo-commits
commit: 65379f436759dfbc4d56e52f1a145950779ebb60
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 13 16:45:27 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 13 16:57:38 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=65379f43
AsyncioEventLoop: suppress BlockingIOError warning during loop close (bug 655656)
Disable the asyncio event loop's SIGCHLD handler before attempting
to close it, in order to suppress a harmless BlockingIOError warning
during loop close.
Closes: https://bugs.gentoo.org/655656
Reported-by: Helmut Jarausch <jarausch <AT> igpm.rwth-aachen.de>
pym/portage/util/_eventloop/asyncio_event_loop.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py
index b365939b0..bf5937de8 100644
--- a/pym/portage/util/_eventloop/asyncio_event_loop.py
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -1,6 +1,8 @@
# Copyright 2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+import signal
+
try:
import asyncio as _real_asyncio
from asyncio.events import AbstractEventLoop as _AbstractEventLoop
@@ -31,7 +33,6 @@ class AsyncioEventLoop(_AbstractEventLoop):
self.call_at = loop.call_at
self.is_running = loop.is_running
self.is_closed = loop.is_closed
- self.close = loop.close
self.create_future = (loop.create_future
if hasattr(loop, 'create_future') else self._create_future)
self.create_task = loop.create_task
@@ -75,3 +76,10 @@ class AsyncioEventLoop(_AbstractEventLoop):
@return: the internal event loop's AbstractEventLoop interface
"""
return self
+
+ def close(self):
+ # Suppress spurious error messages like the following for bug 655656:
+ # Exception ignored when trying to write to the signal wakeup fd:
+ # BlockingIOError: [Errno 11] Resource temporarily unavailable
+ self._loop.remove_signal_handler(signal.SIGCHLD)
+ self._loop.close()
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-05-25 3:15 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-05-25 3:15 UTC (permalink / raw
To: gentoo-commits
commit: 8a8f527c7587bf388645ad703e0305797a26c3b4
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri May 25 03:06:08 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri May 25 03:06:32 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a8f527c
AsyncioEventLoop: remove redundant set_wakeup_fd call
pym/portage/util/_eventloop/asyncio_event_loop.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py
index 65b354544..c07b71103 100644
--- a/pym/portage/util/_eventloop/asyncio_event_loop.py
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -104,5 +104,3 @@ class AsyncioEventLoop(_AbstractEventLoop):
return self._loop.run_until_complete(future)
finally:
self._wakeup_fd = signal.set_wakeup_fd(-1)
- if self._wakeup_fd != -1:
- signal.set_wakeup_fd(-1)
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-05-26 20:22 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-05-26 20:22 UTC (permalink / raw
To: gentoo-commits
commit: 4fb5ef2ce2cb27ae155a25bfa5a4666597afb6ac
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 26 20:20:06 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 26 20:20:06 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=4fb5ef2c
EventLoop.close: fix 'AttributeError: close' for python2
For python2 without epoll, fix handling of missing
'close' attribute on self._poll_obj.
Fixes: 4095be74985c ("Add ForkExecutor (bug 649588)")
Reported-by: Brian Evans <grknight <AT> gentoo.org>
pym/portage/util/_eventloop/EventLoop.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index de0795224..df76374d9 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -968,7 +968,7 @@ class EventLoop(object):
executor.shutdown(wait=True)
if self._poll_obj is not None:
- close = getattr(self._poll_obj, 'close')
+ close = getattr(self._poll_obj, 'close', None)
if close is not None:
close()
self._poll_obj = None
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-06-27 3:05 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-06-27 3:05 UTC (permalink / raw
To: gentoo-commits
commit: 6b50ba69f5a8e311fcddfb2e5c203631bd292c71
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 21 21:03:38 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jun 27 03:04:38 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=6b50ba69
AsyncioEventLoop: exit after unhandled exception (bug 658684)
Fix portage commands to exit immediately for any unhandled
exceptions that are raised while the asyncio event loop is running
without a tty. If we have a tty then start the debugger,
since in might aid in diagnosis of the problem.
In order to avoid potential interference with API consumers, do not
call set_exception_handler unless portage._internal_caller is True.
Bug: https://bugs.gentoo.org/658684
pym/portage/util/_eventloop/asyncio_event_loop.py | 31 +++++++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py
index c07b71103..ea0e03b23 100644
--- a/pym/portage/util/_eventloop/asyncio_event_loop.py
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py
@@ -2,7 +2,9 @@
# Distributed under the terms of the GNU General Public License v2
import os
+import pdb
import signal
+import sys
try:
import asyncio as _real_asyncio
@@ -53,6 +55,35 @@ class AsyncioEventLoop(_AbstractEventLoop):
self.get_debug = loop.get_debug
self._wakeup_fd = -1
+ if portage._internal_caller:
+ loop.set_exception_handler(self._internal_caller_exception_handler)
+
+ @staticmethod
+ def _internal_caller_exception_handler(loop, context):
+ """
+ An exception handler which drops to a pdb shell if std* streams
+ refer to a tty, and otherwise kills the process with SIGTERM.
+
+ In order to avoid potential interference with API consumers, this
+ implementation is only used when portage._internal_caller is True.
+ """
+ loop.default_exception_handler(context)
+ if 'exception' in context:
+ # If we have a tty then start the debugger, since in might
+ # aid in diagnosis of the problem. If there's no tty, then
+ # exit immediately.
+ if all(s.isatty() for s in (sys.stdout, sys.stderr, sys.stdin)):
+ pdb.set_trace()
+ else:
+ # Normally emerge will wait for all coroutines to complete
+ # after SIGTERM has been received. However, an unhandled
+ # exception will prevent the interrupted coroutine from
+ # completing, therefore use the default SIGTERM handler
+ # in order to ensure that emerge exits immediately (though
+ # uncleanly).
+ signal.signal(signal.SIGTERM, signal.SIG_DFL)
+ os.kill(os.getpid(), signal.SIGTERM)
+
def _create_future(self):
"""
Provide AbstractEventLoop.create_future() for python3.4.
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-07-17 19:07 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-07-17 19:07 UTC (permalink / raw
To: gentoo-commits
commit: ae8cc32ccd812661650647feffa1b10fc3ab5837
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 17 19:04:28 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jul 17 19:05:38 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=ae8cc32c
EventLoop: add call_* context arg for python3.7 compat
The context argument currently does nothing, but exists for minimal
interoperability with Future instances that require it for PEP 567.
pym/portage/util/_eventloop/EventLoop.py | 26 ++++++++++++++++++++++----
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index df76374d9..69ccbac2c 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -832,7 +832,7 @@ class EventLoop(object):
return future.result()
- def call_soon(self, callback, *args):
+ def call_soon(self, callback, *args, context=None):
"""
Arrange for a callback to be called as soon as possible. The callback
is called after call_soon() returns, when control returns to the event
@@ -844,18 +844,25 @@ class EventLoop(object):
Any positional arguments after the callback will be passed to the
callback when it is called.
+ The context argument currently does nothing, but exists for minimal
+ interoperability with Future instances that require it for PEP 567.
+
An object compatible with asyncio.Handle is returned, which can
be used to cancel the callback.
@type callback: callable
@param callback: a function to call
+ @type context: contextvars.Context
+ @param context: An optional keyword-only context argument allows
+ specifying a custom contextvars.Context for the callback to run
+ in. The current context is used when no context is provided.
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
- def call_soon_threadsafe(self, callback, *args):
+ def call_soon_threadsafe(self, callback, *args, context=None):
"""Like call_soon(), but thread safe."""
# idle_add provides thread safety
return self._handle(self.idle_add(
@@ -870,7 +877,7 @@ class EventLoop(object):
"""
return monotonic()
- def call_later(self, delay, callback, *args):
+ def call_later(self, delay, callback, *args, context=None):
"""
Arrange for the callback to be called after the given delay seconds
(either an int or float).
@@ -886,19 +893,26 @@ class EventLoop(object):
it is called. If you want the callback to be called with some named
arguments, use a closure or functools.partial().
+ The context argument currently does nothing, but exists for minimal
+ interoperability with Future instances that require it for PEP 567.
+
Use functools.partial to pass keywords to the callback.
@type delay: int or float
@param delay: delay seconds
@type callback: callable
@param callback: a function to call
+ @type context: contextvars.Context
+ @param context: An optional keyword-only context argument allows
+ specifying a custom contextvars.Context for the callback to run
+ in. The current context is used when no context is provided.
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback, args)), self)
- def call_at(self, when, callback, *args):
+ def call_at(self, when, callback, *args, context=None):
"""
Arrange for the callback to be called at the given absolute
timestamp when (an int or float), using the same time reference as
@@ -915,6 +929,10 @@ class EventLoop(object):
@param when: absolute timestamp when to call callback
@type callback: callable
@param callback: a function to call
+ @type context: contextvars.Context
+ @param context: An optional keyword-only context argument allows
+ specifying a custom contextvars.Context for the callback to run
+ in. The current context is used when no context is provided.
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-07-17 19:27 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-07-17 19:27 UTC (permalink / raw
To: gentoo-commits
commit: e46dd735cd4dde58cf3f8ef3cd2b8b29561f5b3e
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 17 19:27:28 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jul 17 19:27:28 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=e46dd735
EventLoop: use python2.7 compatible **kwargs for call_* context arg
Since python2.7 does not allow positional default arguments after
*args, use **kwargs instead.
Fixes: ae8cc32ccd81 ("EventLoop: add call_* context arg for python3.7 compat")
pym/portage/util/_eventloop/EventLoop.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 69ccbac2c..084ff0c18 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -832,7 +832,7 @@ class EventLoop(object):
return future.result()
- def call_soon(self, callback, *args, context=None):
+ def call_soon(self, callback, *args, **kwargs):
"""
Arrange for a callback to be called as soon as possible. The callback
is called after call_soon() returns, when control returns to the event
@@ -862,7 +862,7 @@ class EventLoop(object):
return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
- def call_soon_threadsafe(self, callback, *args, context=None):
+ def call_soon_threadsafe(self, callback, *args, **kwargs):
"""Like call_soon(), but thread safe."""
# idle_add provides thread safety
return self._handle(self.idle_add(
@@ -877,7 +877,7 @@ class EventLoop(object):
"""
return monotonic()
- def call_later(self, delay, callback, *args, context=None):
+ def call_later(self, delay, callback, *args, **kwargs):
"""
Arrange for the callback to be called after the given delay seconds
(either an int or float).
@@ -912,7 +912,7 @@ class EventLoop(object):
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback, args)), self)
- def call_at(self, when, callback, *args, context=None):
+ def call_at(self, when, callback, *args, **kwargs):
"""
Arrange for the callback to be called at the given absolute
timestamp when (an int or float), using the same time reference as
^ permalink raw reply related [flat|nested] 61+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/
@ 2018-07-18 3:41 Zac Medico
0 siblings, 0 replies; 61+ messages in thread
From: Zac Medico @ 2018-07-18 3:41 UTC (permalink / raw
To: gentoo-commits
commit: a7c7af98d755f34e84d1f0f847e2c0d5cc5b7e2f
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 18 03:36:59 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jul 18 03:40:05 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7c7af98
EventLoop: raise TypeError for unexpected call_* keyword args
pym/portage/util/_eventloop/EventLoop.py | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py
index 084ff0c18..ffd12cff9 100644
--- a/pym/portage/util/_eventloop/EventLoop.py
+++ b/pym/portage/util/_eventloop/EventLoop.py
@@ -859,11 +859,23 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+ try:
+ unexpected = next(key for key in kwargs if key != 'context')
+ except StopIteration:
+ pass
+ else:
+ raise TypeError("call_soon() got an unexpected keyword argument '%s'" % unexpected)
return self._handle(self._idle_add(
self._call_soon_callback(callback, args)), self)
def call_soon_threadsafe(self, callback, *args, **kwargs):
"""Like call_soon(), but thread safe."""
+ try:
+ unexpected = next(key for key in kwargs if key != 'context')
+ except StopIteration:
+ pass
+ else:
+ raise TypeError("call_soon_threadsafe() got an unexpected keyword argument '%s'" % unexpected)
# idle_add provides thread safety
return self._handle(self.idle_add(
self._call_soon_callback(callback, args)), self)
@@ -909,6 +921,12 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+ try:
+ unexpected = next(key for key in kwargs if key != 'context')
+ except StopIteration:
+ pass
+ else:
+ raise TypeError("call_later() got an unexpected keyword argument '%s'" % unexpected)
return self._handle(self.timeout_add(
delay * 1000, self._call_soon_callback(callback, args)), self)
@@ -936,6 +954,12 @@ class EventLoop(object):
@return: a handle which can be used to cancel the callback
@rtype: asyncio.Handle (or compatible)
"""
+ try:
+ unexpected = next(key for key in kwargs if key != 'context')
+ except StopIteration:
+ pass
+ else:
+ raise TypeError("call_at() got an unexpected keyword argument '%s'" % unexpected)
delta = when - self.time()
return self.call_later(delta if delta > 0 else 0, callback, *args)
^ permalink raw reply related [flat|nested] 61+ messages in thread
end of thread, other threads:[~2018-07-18 3:41 UTC | newest]
Thread overview: 61+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-04-19 6:16 [gentoo-commits] proj/portage:master commit in: pym/portage/util/_eventloop/ Zac Medico
-- strict thread matches above, loose matches on Subject: below --
2018-07-18 3:41 Zac Medico
2018-07-17 19:27 Zac Medico
2018-07-17 19:07 Zac Medico
2018-06-27 3:05 Zac Medico
2018-05-26 20:22 Zac Medico
2018-05-25 3:15 Zac Medico
2018-05-13 16:58 Zac Medico
2018-05-07 0:27 Zac Medico
2018-04-19 6:41 Zac Medico
2018-04-18 21:52 Zac Medico
2018-04-17 18:24 Zac Medico
2018-04-17 18:02 Zac Medico
2018-04-17 6:24 Zac Medico
2018-04-17 0:53 Zac Medico
2018-04-16 8:48 Zac Medico
2018-04-16 6:43 Zac Medico
2018-04-16 0:03 Zac Medico
2018-04-15 22:32 Zac Medico
2018-04-09 1:27 Zac Medico
2018-04-08 22:08 Zac Medico
2018-04-08 21:37 Zac Medico
2018-04-08 21:18 Zac Medico
2018-04-07 20:20 Zac Medico
2017-05-05 18:47 Zac Medico
2013-02-25 23:53 Zac Medico
2013-01-04 3:39 Zac Medico
2012-12-31 3:16 Zac Medico
2012-12-28 1:44 Zac Medico
2012-12-28 1:36 Zac Medico
2012-12-27 2:39 Zac Medico
2012-12-27 2:31 Zac Medico
2012-11-22 12:23 Zac Medico
2012-09-26 2:26 Zac Medico
2012-08-22 16:23 Zac Medico
2012-08-22 5:39 Zac Medico
2012-02-18 4:04 Zac Medico
2012-02-17 23:23 Zac Medico
2012-02-17 22:17 Zac Medico
2012-02-17 10:31 Zac Medico
2012-02-17 10:20 Zac Medico
2012-02-17 9:29 Zac Medico
2012-02-17 6:29 Zac Medico
2012-02-17 6:04 Zac Medico
2012-02-17 3:08 Zac Medico
2012-02-16 21:43 Zac Medico
2012-02-16 21:35 Zac Medico
2012-02-16 4:58 Zac Medico
2012-02-16 4:41 Zac Medico
2012-02-14 16:27 Zac Medico
2012-02-14 3:06 Zac Medico
2012-02-14 1:23 Zac Medico
2012-02-11 23:56 Zac Medico
2012-02-11 21:41 Zac Medico
2012-02-11 21:11 Zac Medico
2012-02-11 19:35 Zac Medico
2012-02-11 2:59 Zac Medico
2012-02-10 0:42 Zac Medico
2012-02-10 0:20 Zac Medico
2012-02-10 0:17 Zac Medico
2012-02-09 7:25 Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox