* [gentoo-commits] proj/R_overlay:depres_wip commit in: roverlay/ebuild/, roverlay/depres/simpledeprule/, roverlay/depres/, roverlay/, ...
@ 2012-07-12 18:10 André Erdmann
0 siblings, 0 replies; only message in thread
From: André Erdmann @ 2012-07-12 18:10 UTC (permalink / raw
To: gentoo-commits
commit: a8a7be81e617a81e9e7740184b1b6368cb656f75
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 12 17:48:59 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 12 18:09:44 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a8a7be81
introducing dependency types
The dependency resolver can now differentiate between dependency types
The actual deptype is a bit field that sets flags like 'is mandatory',
'is an external dep', 'try other dep types, too'.
Functionality for this has been added to most modules/classes, particularly
to EbuildJobChannel and ebuild/depres, but not to the rule reader,
which means that there's currently no way to _use_ the dep types!
Also added an ErrorQueue implementation that saves some "if <has err_queue>"
checks. It's also able to unblock waiting queues, which helps to end
threads when an exception has been raised.
---
roverlay/depres/channels.py | 116 +++++++++++++---------------
roverlay/depres/depenv.py | 89 ++++++++++++----------
roverlay/depres/depresolver.py | 89 +++++++++++----------
roverlay/depres/deprule.py | 17 +++--
roverlay/depres/deptype.py | 19 +++++
roverlay/depres/simpledeprule/console.py | 15 +++-
roverlay/depres/simpledeprule/pool.py | 24 ++----
roverlay/depres/simpledeprule/rulemaker.py | 4 +
roverlay/ebuild/depres.py | 54 +++++++++----
roverlay/errorqueue.py | 115 +++++++++++++++++++++++++++
roverlay/overlay/creator.py | 42 +++-------
roverlay/overlay/worker.py | 21 ++----
roverlay/recipe/easyresolver.py | 8 +-
13 files changed, 379 insertions(+), 234 deletions(-)
diff --git a/roverlay/depres/channels.py b/roverlay/depres/channels.py
index 72f2b87..f15a6d0 100644
--- a/roverlay/depres/channels.py
+++ b/roverlay/depres/channels.py
@@ -4,12 +4,13 @@
import logging
+from roverlay.depres import deptype
from roverlay.depres.depenv import DepEnv
from roverlay.depres.communication import DependencyResolverChannel
COMLINK = logging.getLogger ( "depres.com" )
-class EbuildJobChannel ( DependencyResolverChannel ):
+class _EbuildJobChannelBase ( DependencyResolverChannel ):
"""The EbuildJobChannel is an interface to the dependency resolver used
in EbuildJobs.
It can be used to insert dependency strings, trigger dep resolution,
@@ -26,10 +27,10 @@ class EbuildJobChannel ( DependencyResolverChannel ):
* name -- name of this channel, optional
* logger --
"""
- super ( EbuildJobChannel, self ) . __init__ ( main_resolver=None )
+ super ( _EbuildJobChannelBase, self ) . __init__ ( main_resolver=None )
- # this is the number of resolved deps so far, should only be modified
- # in the join()-method
+ # this is the number of resolved deps so far,
+ # should only be modified in the join()-method
self._depdone = 0
self.err_queue = err_queue
@@ -41,11 +42,10 @@ class EbuildJobChannel ( DependencyResolverChannel ):
# used to communicate with the resolver
self._depres_queue = None
- # the list of dependency lookups assigned to this channel
- ## todo: could remove this list
- self.dep_env_list = list ()
+ # the count of dep strings ever assigned to this channel
+ self._depcount = 0
- _logger = logger if hasattr ( logger, 'log' ) else COMLINK
+ _logger = logger if logger is not None else COMLINK
if name:
self.name = name
self.logger = _logger.getChild ( 'channel.' + name )
@@ -58,8 +58,9 @@ class EbuildJobChannel ( DependencyResolverChannel ):
if self._depdone >= 0:
# else already closed
- super ( EbuildJobChannel, self ) . close()
- del self.dep_env_list, self._collected_deps, self._depres_queue
+ super ( _EbuildJobChannelBase, self ).close()
+ self.err_queue.remove_queue ( self._depres_queue )
+ del self._collected_deps, self._depres_queue
del self.logger
self._depdone = -1
@@ -77,11 +78,12 @@ class EbuildJobChannel ( DependencyResolverChannel ):
if self._depres_master is None:
self._depres_master = resolver
self._depres_queue = channel_queue
+ self.err_queue.attach_queue ( self._depres_queue, None )
else:
raise Exception ( "channel already bound to a resolver." )
# --- end of set_resolver (...) ---
- def add_dependency ( self, dep_str ):
+ def add_dependency ( self, dep_str, deptype_mask ):
"""Adds a dependency string that should be looked up.
This channel will create a "dependency environment" for it and enqueues
that in the dep resolver.
@@ -98,13 +100,13 @@ class EbuildJobChannel ( DependencyResolverChannel ):
"This channel is 'done', it doesn't accept new dependencies."
)
else:
- dep_env = DepEnv ( dep_str )
- self.dep_env_list.append ( dep_env )
+ dep_env = DepEnv ( dep_str=dep_str, deptype_mask=deptype_mask )
+ self._depcount += 1
self._depres_master.enqueue ( dep_env, self.ident )
# --- end of add_dependency (...) ---
- def add_dependencies ( self, dep_list ):
+ def add_dependencies ( self, dep_list, deptype_mask ):
"""Adds dependency strings to this channel. See add_dependency (...)
for details.
@@ -115,30 +117,10 @@ class EbuildJobChannel ( DependencyResolverChannel ):
returns: None (implicit)
"""
- for dep_str in dep_list: self.add_dependency ( dep_str )
+ for dep_str in dep_list:
+ self.add_dependency ( dep_str=dep_str, deptype_mask=deptype_mask )
# --- end of add_dependencies (...) ---
- def lookup ( self, dep_str ):
- """Looks up a specific dep str. Use the (faster) collect_dependencies()
- method for getting all dependencies if order doesn't matter.
-
- ! It assumes that dep_str is unique in this channel.
-
- arguments:
- * dep_str -- to be looked up
-
- raises: Exception if dep_str not in the dep env list.
- """
- # it's no requirement that this channel is done when calling this method
- for de in self.dep_env_list:
- if de.dep_str == dep_str:
- return de.get_result() [1]
-
- raise Exception (
- "bad usage: %s not in channel's dep env list!" % dep_str
- )
- # --- end of lookup (...) ---
-
def collect_dependencies ( self ):
"""Returns a list that contains all resolved deps,
including ignored deps that resolve to None.
@@ -155,12 +137,19 @@ class EbuildJobChannel ( DependencyResolverChannel ):
raise Exception ( "cannot do that" )
# --- end of collect_dependencies (...) ---
+ def satisfy_request ( self, *args, **kw ):
+ raise Exception ( "method stub" )
+
+
+class EbuildJobChannel ( _EbuildJobChannelBase ):
+
+
def satisfy_request ( self,
close_if_unresolvable=True, preserve_order=False
):
"""Tells to the dependency resolver to run.
It blocks until this channel is done, which means that either all
- deps are resolved or one is unresolvable.
+ deps are resolved or a mandatory one is unresolvable.
arguments:
* close_if_unresolvable -- close the channel if one dep is unresolvable
@@ -173,54 +162,57 @@ class EbuildJobChannel ( DependencyResolverChannel ):
Returns the list of resolved dependencies if all could be resolved,
else None.
"""
-
- # using a set allows easy difference() operations between
- # DEPEND/RDEPEND/.. later, seewave requires sci-libs/fftw
- # in both DEPEND and RDEPEND for example
dep_collected = list()
- satisfiable = True \
- if self.err_queue is None \
- else self.err_queue.empty()
-
+ resolved = dep_collected.append
def handle_queue_item ( dep_env ):
self._depdone += 1
+ ret = False
if dep_env is None:
- # could used to unblock the queue
- if self.err_queue is None:
- return False
- else:
- return self.err_queue.empty()
+ # queue unblocked -> on_error mode, return False
+ #ret = False
+ pass
elif dep_env.is_resolved():
- ### and dep_env in self.dep_env_list
# successfully resolved
- dep_collected.append ( dep_env.get_result() [1] )
- self._depres_queue.task_done()
- return True
- else:
- # failed
- self._depres_queue.task_done()
- return False
+ resolved ( dep_env.get_resolved() )
+ ret = True
+ elif deptype.mandatory & ~dep_env.deptype_mask:
+ # not resolved, but deptype has no mandatory bit set
+ # => dep is not required, resolve as None
+ resolved ( None )
+ ret = True
+ # else failed
+
+ self._depres_queue.task_done()
+ return ret
# --- end of handle_queue_item (...) ---
+ satisfiable = True
# loop until
- # (a) at least one dependency could not be resolved or
+ # (a) at least one required dependency could not be resolved or
# (b) all deps processed or
# (c) error queue not empty
- while self._depdone < len ( self.dep_env_list ) and satisfiable:
+ while self._depdone < self._depcount and \
+ satisfiable and self.err_queue.empty \
+ :
# tell the resolver to start
self._depres_master.start()
+ # ^, race condition, running resolver vs waiting queue (FIXME)
+
# wait for one result at least
satisfiable = handle_queue_item ( self._depres_queue.get() )
# and process all available results
- while not self._depres_queue.empty() and satisfiable:
+ while satisfiable and not self._depres_queue.empty():
satisfiable = handle_queue_item ( self._depres_queue.get_nowait() )
# --- end while
- if satisfiable:
+ if satisfiable and self.err_queue.empty:
+ # using a set allows easy difference() operations between
+ # DEPEND/RDEPEND/.. later, seewave requires sci-libs/fftw
+ # in both DEPEND and RDEPEND for example
self._collected_deps = frozenset ( dep_collected )
if preserve_order:
return tuple ( dep_collected )
diff --git a/roverlay/depres/depenv.py b/roverlay/depres/depenv.py
index eba03a8..7cef1c9 100644
--- a/roverlay/depres/depenv.py
+++ b/roverlay/depres/depenv.py
@@ -3,58 +3,61 @@
# Distributed under the terms of the GNU General Public License v2
import re
-
-# excluding A-Z since dep_str_low will be used to find a match
-_NAME = '(?P<name>[a-z0-9_\-/]+)'
-_VER = '(?P<ver>[0-9._\-]+)'
-# { <, >, <=, >=, =, != } (TODO !=)
-_VERMOD = '(?P<vmod>[<>]|[<>!]?[=])'
-
-# this lists ... <fixme>
-V_REGEX_STR = frozenset ( (
- # 'R >= 2.15', 'R >=2.15' etc. (but not 'R>=2.15'!)
- '^%s\s+%s?\s*%s\s*$' % ( _NAME, _VERMOD, _VER ),
-
- # TODO: merge these regexes: ([{ )]}]
-
- # 'R (>= 2.15)', 'R(>=2.15)' etc.
- '^%s\s*\(%s?\s*%s\s*\)$' % ( _NAME, _VERMOD, _VER ),
-
- # 'R [>= 2.15]', 'R[>=2.15]' etc.
- '^%s\s*\[%s?\s*%s\s*\]$' % ( _NAME, _VERMOD, _VER ),
-
-
- # 'R {>= 2.15}', 'R{>=2.15}' etc.
- '^%s\s*\{%s?\s*%s\s*\}$' % ( _NAME, _VERMOD, _VER ),
-) )
-
-VERSION_REGEX = frozenset ( re.compile ( regex ) for regex in V_REGEX_STR )
-
-FIXVERSION_REGEX = re.compile ( '[_\-]' )
-
-TRY_ALL_REGEXES = False
-
-
class DepEnv ( object ):
+ # excluding A-Z since dep_str_low will be used to find a match
+ _NAME = '(?P<name>[a-z0-9_\-/]+)'
+ _VER = '(?P<ver>[0-9._\-]+)'
+ # { <, >, <=, >=, =, != } (TODO !=)
+ _VERMOD = '(?P<vmod>[<>]|[<>!]?[=])'
+
+ V_REGEX_STR = frozenset ( (
+ # 'R >= 2.15', 'R >=2.15' etc. (but not 'R>=2.15'!)
+ '^{name}\s+{vermod}?\s*{ver}\s*$'.format (
+ name=_NAME, vermod=_VERMOD, ver=_VER
+ ),
+ # TODO: merge these regexes: () [] {} (but not (],{), ...)
+ # 'R (>= 2.15)', 'R(>=2.15)' etc.
+ '^{name}\s*\(\s*{vermod}?\s*{ver}\s*\)$'.format (
+ name=_NAME, vermod=_VERMOD, ver=_VER
+ ),
+ # 'R [>= 2.15]', 'R[>=2.15]' etc.
+ '^{name}\s*\[\s*{vermod}?\s*{ver}\s*\]$'.format (
+ name=_NAME, vermod=_VERMOD, ver=_VER
+ ),
+
+ # 'R {>= 2.15}', 'R{>=2.15}' etc.
+ '^{name}\s*\{{\s*{vermod}?\s*{ver}\s*\}}$'.format (
+ name=_NAME, vermod=_VERMOD, ver=_VER
+ ),
+ ) )
+
+ VERSION_REGEX = frozenset (
+ re.compile ( regex ) for regex in V_REGEX_STR
+ )
+ FIXVERSION_REGEX = re.compile ( '[_\-]' )
+ TRY_ALL_REGEXES = False
+
STATUS_UNDONE = 1
STATUS_RESOLVED = 2
STATUS_UNRESOLVABLE = 4
- def __init__ ( self, dep_str ):
+ def __init__ ( self, dep_str, deptype_mask ):
"""Initializes a dependency environment that represents the dependency
resolution of one entry in the description data of an R package.
+ Precalculating most (if not all) data since this object will be passed
+ through many dep rules.
arguments:
* dep_str -- dependency string at it appears in the description data.
"""
- self.ident = id ( self )
- self.dep_str = dep_str
- self.dep_str_low = dep_str.lower()
- self.status = DepEnv.STATUS_UNDONE
- self.resolved_by = None
+ self.deptype_mask = deptype_mask
+ self.dep_str = dep_str
+ self.dep_str_low = dep_str.lower()
+ self.status = DepEnv.STATUS_UNDONE
+ self.resolved_by = None
- self.try_all_regexes = TRY_ALL_REGEXES
+ self.try_all_regexes = self.__class__.TRY_ALL_REGEXES
self._depsplit()
@@ -66,14 +69,14 @@ class DepEnv ( object ):
def _depsplit ( self ):
result = list()
- for r in VERSION_REGEX:
+ for r in self.__class__.VERSION_REGEX:
m = r.match ( self.dep_str_low )
if m is not None:
result.append ( dict (
name = m.group ( 'name' ),
version_modifier = m.group ( 'vmod' ),
- version = FIXVERSION_REGEX.sub (
+ version = self.__class__.FIXVERSION_REGEX.sub (
'.', m.group ( 'ver' )
)
) )
@@ -143,3 +146,7 @@ class DepEnv ( object ):
return ( self.dep_str, self.resolved_by )
# --- end of get_result (...) ---
+
+ def get_resolved ( self ):
+ return self.resolved_by
+ # --- end of get_resolved (...) ---
diff --git a/roverlay/depres/depresolver.py b/roverlay/depres/depresolver.py
index d0383dd..bbe58b4 100644
--- a/roverlay/depres/depresolver.py
+++ b/roverlay/depres/depresolver.py
@@ -13,7 +13,8 @@ except ImportError:
from roverlay import config
-from roverlay.depres import communication, events
+from roverlay.depres import communication, deptype, events
+#from roverlay.depres import simpledeprule
#from roverlay.depres.depenv import DepEnv (implicit)
@@ -31,7 +32,7 @@ class DependencyResolver ( object ):
NUMTHREADS = config.get ( "DEPRES.jobcount", 0 )
- def __init__ ( self ):
+ def __init__ ( self, err_queue ):
"""Initializes a DependencyResolver."""
self.logger = logging.getLogger ( self.__class__.__name__ )
@@ -51,7 +52,7 @@ class DependencyResolver ( object ):
# the dep res worker threads
self._threads = None
- self.err_queue = None
+ self.err_queue = err_queue
# the list of registered listener modules
@@ -80,7 +81,6 @@ class DependencyResolver ( object ):
# list of rule pools that have been created from reading files
self.static_rule_pools = list ()
-
if SAFE_CHANNEL_IDS:
# this lock is used in register_channel
self._chanlock = threading.Lock()
@@ -88,10 +88,6 @@ class DependencyResolver ( object ):
self.all_channel_ids = set()
# --- end of __init__ (...) ---
- def set_exception_queue ( self, equeue ):
- self.err_queue = equeue
- # --- end of set_exception_queue (...) ---
-
def _sort ( self ):
"""Sorts the rule pools of this resolver."""
for pool in self.static_rule_pools: pool.sort()
@@ -104,6 +100,18 @@ class DependencyResolver ( object ):
self._dep_unresolvable.clear()
# --- end of _reset_unresolvable (...) ---
+ def _new_rulepools_added ( self ):
+ """Called after adding new rool pools."""
+ self._reset_unresolvable()
+ self._sort()
+ # --- end of _new_rulepools_added (...) ---
+
+ #def get_reader ( self ):
+ # return simpledeprule.reader.SimpleDependencyRuleReader (
+ # pool_add=self.static_rule_pools.append,
+ # when_done=self._new_rulepools_added
+ # )
+
def add_rulepool ( self, rulepool, pool_type=None ):
"""Adds a (static) rule pool to this resolver.
Calls self.sort() afterwards.
@@ -113,14 +121,12 @@ class DependencyResolver ( object ):
* pool_type -- ignored.
"""
self.static_rule_pools.append ( rulepool )
- self._reset_unresolvable()
- self._sort()
+ self._new_rulepools_added()
# --- end of add_rulepool (...) ---
def _report_event ( self, event, dep_env=None, pkg_env=None, msg=None ):
"""Reports an event to the log and listeners.
-
arguments:
* event -- name of the event (RESOLVED etc., use capslock!)
* dep_env -- dependency env
@@ -308,23 +314,6 @@ class DependencyResolver ( object ):
# self.runlock is released when _thread_run_main is done
# --- end of start (...) ---
- def unblock_channels ( self ):
- # unblock all channels by processing all remaining deps as
- # unresolved
- ## other option: select channel, but this may interfere with
- ## channel_closed()
- channel_gone = set()
- while not self._depqueue.empty():
- chan, dep_env = self._depqueue.get_nowait()
- dep_env.set_unresolvable()
-
- if chan not in channel_gone:
- try:
- self._depqueue_done [chan].put_nowait ( dep_env )
- except KeyError:
- channel_gone.add ( chan )
- # --- end of unblock_channels (...) ---
-
def _thread_run_main ( self ):
"""Tells the resolver to run."""
jobcount = self.__class__.NUMTHREADS
@@ -362,7 +351,7 @@ class DependencyResolver ( object ):
# iterate over _depqueue_failed and report unresolved
## todo can thread this
- while not self._depqueue_failed.empty():
+ while not self._depqueue_failed.empty() and self.err_queue.empty:
try:
channel_id, dep_env = self._depqueue_failed.get_nowait()
@@ -383,14 +372,12 @@ class DependencyResolver ( object ):
# channel has been closed before calling put, ignore this
pass
- if self.err_queue and not self.err_queue.empty():
- self.unblock_channels()
+ if not self.err_queue.really_empty():
+ self.err_queue.unblock_queues()
except ( Exception, KeyboardInterrupt ) as e:
- self.unblock_channels()
-
- if jobcount > 0 and self.err_queue:
- self.err_queue.put_nowait ( id ( self ), e )
+ if jobcount > 0:
+ self.err_queue.push ( id ( self ), e )
return
else:
raise e
@@ -407,9 +394,7 @@ class DependencyResolver ( object ):
returns: None (implicit)
"""
try:
- while ( not self.err_queue or self.err_queue.empty() ) \
- and not self._depqueue.empty() \
- :
+ while self.err_queue.empty and not self._depqueue.empty():
try:
to_resolve = self._depqueue.get_nowait()
except queue.Empty:
@@ -446,14 +431,34 @@ class DependencyResolver ( object ):
is_resolved = 1
if is_resolved == 0:
- # search for a match in the rule pools
- for rulepool in self.static_rule_pools:
+ # search for a match in the rule pools that accept
+ # the dep type
+ for rulepool in ( p for p in self.static_rule_pools \
+ if p.deptype_mask & dep_env.deptype_mask
+ ):
result = rulepool.matches ( dep_env )
if not result is None and result [0] > 0:
resolved = result [1]
is_resolved = 2
break
+ if is_resolved == 0 and \
+ dep_env.deptype_mask & deptype.try_other \
+ :
+ # search for a match in the rule pools that (normally)
+ # don't accept the dep type
+ for rulepool in ( p for p in self.static_rule_pools \
+ if p.deptype_mask & ~dep_env.deptype_mask
+ ):
+ result = rulepool.matches ( dep_env )
+ if not result is None and result [0] > 0:
+ resolved = result [1]
+ is_resolved = 2
+ break
+
+
+
+
if is_resolved == 2:
dep_env.set_resolved ( resolved, append=False )
@@ -485,8 +490,8 @@ class DependencyResolver ( object ):
# --- end while
except ( Exception, KeyboardInterrupt ) as e:
- if self.__class__.NUMTHREADS > 0 and self.err_queue:
- self.err_queue.put_nowait ( id ( self ), e )
+ if self.__class__.NUMTHREADS > 0:
+ self.err_queue.push ( id ( self ), e )
return
else:
raise e
diff --git a/roverlay/depres/deprule.py b/roverlay/depres/deprule.py
index 27e0f47..32c73e1 100644
--- a/roverlay/depres/deprule.py
+++ b/roverlay/depres/deprule.py
@@ -2,6 +2,8 @@
# Copyright 2006-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from roverlay.depres import deptype
+
class DependencyRule ( object ):
"""Prototype of a dependency rule. Not meant for instantiation."""
@@ -26,7 +28,7 @@ class DependencyRule ( object ):
class DependencyRulePool ( object ):
- def __init__ ( self, name, priority ):
+ def __init__ ( self, name, priority, deptype_mask ):
"""Initializes an DependencyRulePool, which basically is a set of
dependency rules with methods like "search for x in all rules."
@@ -34,13 +36,16 @@ class DependencyRulePool ( object ):
* name -- name of this rule pool
* priority -- priority of this pool (lower is better)
"""
- self.rules = list()
- self.name = name
- self.priority = priority
+ self.rules = list()
+ self._rule_add = self.rules.append
+ self.name = name
+ self.priority = priority
+ # filter out deptype flags like "mandatory"
+ self.deptype_mask = deptype_mask & deptype.RESOLVE_ALL
# the "rule weight" is the sum of the rules' priorities
# it's used to compare/sort dependency pools with
# the same priority (lesser weight is better)
- self.rule_weight = 0
+ self.rule_weight = 0
# --- end of __init__ (...) ---
def sort ( self ):
@@ -64,7 +69,7 @@ class DependencyRulePool ( object ):
* rule --
"""
if issubclass ( rule, DependencyRule ):
- self.rules.append ( rule )
+ self._rule_add ( rule )
else:
raise Exception ( "bad usage (dependency rule expected)." )
diff --git a/roverlay/depres/deptype.py b/roverlay/depres/deptype.py
new file mode 100644
index 0000000..73d6265
--- /dev/null
+++ b/roverlay/depres/deptype.py
@@ -0,0 +1,19 @@
+# not using a single bool for these two types
+# * some dependencies could be resolved as sys and internal ("just resolve it")
+# * allows to add other types
+
+# <deptype> ::= 2**k | k in {0,1,2,...}
+# try_call indicates that the dep can be checked world-wide (in non-accepting
+# rule pools) after unsuccessful resolution
+try_other = 1
+mandatory = 2
+external = 4
+internal = 8
+
+ALL = external | internal | mandatory
+RESOLVE_ALL = external | internal
+
+SYS = internal | mandatory
+PKG = external | mandatory
+
+MANDATORY_TRY = try_other | mandatory
diff --git a/roverlay/depres/simpledeprule/console.py b/roverlay/depres/simpledeprule/console.py
index e3db541..5d84010 100644
--- a/roverlay/depres/simpledeprule/console.py
+++ b/roverlay/depres/simpledeprule/console.py
@@ -5,6 +5,8 @@ import shlex
import logging
from roverlay import config
+from roverlay.errorqueue import NopErrorQueue
+from roverlay.depres import deptype
from roverlay.depres.depresolver import DependencyResolver
from roverlay.depres.channels import EbuildJobChannel
from roverlay.depres.simpledeprule.pool import SimpleDependencyRulePool
@@ -63,7 +65,9 @@ class DepResConsole ( object ):
whitespace = re.compile ( "\s+" )
def __init__ ( self ):
- self.resolver = DependencyResolver()
+ self.err_queue = NopErrorQueue()
+
+ self.resolver = DependencyResolver ( err_queue=self.err_queue )
# log everything
self.resolver.set_logmask ( -1 )
# disable passing events to listeners
@@ -122,7 +126,10 @@ class DepResConsole ( object ):
def _getpool ( self, new=False ):
if new or not self.poolstack:
- pool = SimpleDependencyRulePool ( "pool" + str ( self.pool_id ) )
+ pool = SimpleDependencyRulePool (
+ "pool" + str ( self.pool_id ),
+ deptype_mask=deptype.RESOLVE_ALL
+ )
self.pool_id += 1
self.poolstack.append ( pool )
self.resolver._reset_unresolvable()
@@ -360,7 +367,9 @@ class DepResConsole ( object ):
if not dep_list:
self.stdout ( "Resolve what?\n" )
else:
- channel = EbuildJobChannel ( err_queue=None, name="channel" )
+ channel = EbuildJobChannel (
+ err_queue=self.err_queue, name="channel"
+ )
self.resolver.register_channel ( channel )
self.stdout ( "Trying to resolve {!r}.\n".format ( dep_list ) )
diff --git a/roverlay/depres/simpledeprule/pool.py b/roverlay/depres/simpledeprule/pool.py
index 8a2b7b2..4d7905b 100644
--- a/roverlay/depres/simpledeprule/pool.py
+++ b/roverlay/depres/simpledeprule/pool.py
@@ -2,20 +2,15 @@
# Copyright 2006-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-#import re
import logging
-#from roverlay import config
from roverlay.depres import deprule
from roverlay.depres.simpledeprule.reader import SimpleDependencyRuleReader
-#from roverlay.depres.simpledeprule.rules import *
from roverlay.depres.simpledeprule.abstractrules import SimpleRule
-TMP_LOGGER = logging.getLogger ('simpledeps')
-
class SimpleDependencyRulePool ( deprule.DependencyRulePool ):
- def __init__ ( self, name, priority=70, filepath=None ):
+ def __init__ ( self, name, priority=70, **kw ):
"""Initializes a SimpleDependencyRulePool, which is a DependencyRulePool
specialized in simple dependency rules;
it offers loading rules from files.
@@ -23,13 +18,10 @@ class SimpleDependencyRulePool ( deprule.DependencyRulePool ):
arguments:
* name -- string identifier for this pool
* priority -- of this pool
- * filepath -- if set and not None: load a rule file directly
"""
- super ( SimpleDependencyRulePool, self ) . __init__ ( name, priority )
-
- if not filepath is None:
- self.load_rule_file ( filepath )
-
+ super ( SimpleDependencyRulePool, self ) . __init__ (
+ name, priority, **kw
+ )
# --- end of __init__ (...) ---
def add ( self, rule ):
@@ -39,15 +31,17 @@ class SimpleDependencyRulePool ( deprule.DependencyRulePool ):
arguments:
* rule --
"""
+ # trust in proper usage
if isinstance ( rule, SimpleRule ):
- self.rules.append ( rule )
+ self._rule_add ( rule )
else:
raise Exception ( "bad usage (simple dependency rule expected)." )
-
# --- end of add (...) ---
def get_reader ( self ):
- return SimpleDependencyRuleReader ( rule_add=self.add )
+ # trusting in SimpleDependencyRuleReader's correctness,
+ # let it add rules directly without is-instance checks
+ return SimpleDependencyRuleReader ( rule_add=self._rule_add )
# --- end of get_reader (...) ---
def load_rule_file ( self, filepath ):
diff --git a/roverlay/depres/simpledeprule/rulemaker.py b/roverlay/depres/simpledeprule/rulemaker.py
index 2435787..e1ebd6d 100644
--- a/roverlay/depres/simpledeprule/rulemaker.py
+++ b/roverlay/depres/simpledeprule/rulemaker.py
@@ -7,11 +7,14 @@ import logging
from roverlay import config
+#from roverlay.depres import deptype
from roverlay.depres.simpledeprule import rules
from roverlay.depres.simpledeprule.abstractrules import *
class SimpleRuleMaker ( object ):
+# class RuleTypes ( object ): pass
+
class RuleKeywords ( object ):
def __init__ ( self ):
self._default_rule, self._rule_map = rules.get_rule_map()
@@ -49,6 +52,7 @@ class SimpleRuleMaker ( object ):
self._kw = self.__class__.RuleKeywords()
self._next = None
self._rules = list()
+ #self._rules = list() :: ( deptype, rule )
# --- end of __init__ (...) ---
def zap ( self ):
diff --git a/roverlay/ebuild/depres.py b/roverlay/ebuild/depres.py
index 1c19a00..5a85bb4 100644
--- a/roverlay/ebuild/depres.py
+++ b/roverlay/ebuild/depres.py
@@ -2,15 +2,38 @@
# Copyright 2006-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from roverlay.depres import deptype
from roverlay.ebuild import evars
# TODO/FIXME/IGNORE move this to const / config
-FIELDS = {
- 'R_SUGGESTS' : [ 'Suggests' ],
- 'DEPENDS' : [ 'Depends', 'Imports' ],
- 'RDEPENDS' : [ 'LinkingTo', 'SystemRequirements' ]
+FIELDS_TO_EVAR = {
+ 'R_SUGGESTS' : ( 'Suggests', ),
+ 'DEPENDS' : ( 'Depends', 'Imports' ),
+ 'RDEPENDS' : ( 'LinkingTo', 'SystemRequirements' ),
+ # ? : ( 'Enhances', )
}
+# setting per-field dep types here, in accordance with
+# http://cran.r-project.org/doc/manuals/R-exts.html#The-DESCRIPTION-file
+FIELDS = {
+ # "The Depends field gives a comma-separated
+ # list of >>package names<< which this package depends on."
+ 'Depends' : deptype.SYS,
+ # "Other dependencies (>>external to the R system<<)
+ # should be listed in the SystemRequirements field"
+ 'SystemRequirements' : deptype.PKG,
+ # "The Imports field lists >>packages<< whose namespaces
+ # are imported from (as specified in the NAMESPACE file)
+ # but which do not need to be attached."
+ 'Imports' : deptype.PKG,
+ # "The Suggests field uses the same syntax as Depends
+ # and lists >>packages<< that are >>not necessarily needed<<."
+ 'Suggests' : deptype.external,
+ # "A package that wishes to make use of header files
+ # in other >>packages<< needs to declare them as
+ # a comma-separated list in the field LinkingTo in the DESCRIPTION file."
+ 'LinkingTo' : deptype.PKG,
+}
EBUILDVARS = {
'R_SUGGESTS' : evars.R_SUGGESTS,
'DEPENDS' : evars.DEPEND,
@@ -54,10 +77,10 @@ class EbuildDepRes ( object ):
# --- end of __init__ (...) ---
- def done ( self ) : return self.status < 1
- def busy ( self ) : return self.status > 0
+ #def done ( self ) : return self.status < 1
+ #def busy ( self ) : return self.status > 0
def success ( self ) : return self.status == 0
- def fail ( self ) : return self.status < 0
+ #def fail ( self ) : return self.status < 0
def get_result ( self ):
"""Returns the result of dependency resolution,
@@ -93,7 +116,7 @@ class EbuildDepRes ( object ):
self._channels [dependency_type] = self.request_resolver (
name=dependency_type,
logger=self.logger,
- err_queue=self.err_queue
+ err_queue=self.err_queue,
)
return self._channels [dependency_type]
# --- end of get_channel (...) ---
@@ -115,21 +138,18 @@ class EbuildDepRes ( object ):
dep_type = desc_field = None
- for dep_type in FIELDS:
+ for dep_type in FIELDS_TO_EVAR:
resolver = None
- for desc_field in FIELDS [dep_type]:
+ for desc_field in FIELDS_TO_EVAR [dep_type]:
if desc_field in desc:
if not resolver:
resolver = self._get_channel ( dep_type )
- if isinstance ( desc [desc_field], str ):
- resolver.add_dependency ( desc [desc_field] )
- elif hasattr ( desc [desc_field], '__iter__' ):
- resolver.add_dependencies ( desc [desc_field] )
- else:
- logger.warning (
- "Cannot add dependency '%s'." % desc [desc_field]
+ # make sure that DescriptionReader reads all dep fields as list
+ resolver.add_dependencies (
+ dep_list = desc [desc_field],
+ deptype_mask = FIELDS [desc_field]
)
# -- for dep_type
diff --git a/roverlay/errorqueue.py b/roverlay/errorqueue.py
new file mode 100644
index 0000000..d0aa042
--- /dev/null
+++ b/roverlay/errorqueue.py
@@ -0,0 +1,115 @@
+import sys
+import threading
+
+class _EQueue ( object ) : pass
+
+class NopErrorQueue ( _EQueue ):
+ """This can be used as error queue in single-threaded execution."""
+ def __init__ ( self ):
+ self.empty = True
+
+ def push ( self, context, error ):
+ self.empty = False
+ sys.stderr.write ( "Exception from {!r}:\n".format ( context ) )
+ raise error
+
+ def really_empty ( self ): return self.empty
+ def attach_queue ( self, q, unblock_item ): pass
+ def remove_queue ( self, q ): pass
+ def unblock_queues ( self ): pass
+ def peek ( self ): return ( None, None )
+ def get_all ( self ): return ( ( None, None ), )
+ def get_exceptions ( self ): return ()
+
+
+#class ErrorQueue ( NopErrorQueue ):
+class ErrorQueue ( _EQueue ):
+ """This is the error queue for threaded execution."""
+ # (it's not a queue)
+
+ def __init__ ( self ):
+ self.empty = True
+ self._exceptions = list()
+ # this error queue is able to unblock waiting queues (in future; TODO)
+ # id -> queue [, unblocking_item:=None]
+ self._queues_to_unblock = dict()
+
+ self._lock = threading.Lock()
+
+ def really_empty ( self ):
+ """Returns true if no exception stored. Uses a lock to ensure
+ correctness of the result.
+ """
+ with self._lock:
+ self.empty = len ( self._exceptions ) == 0
+ return self.empty
+
+ def _unblock_queues ( self ):
+ """Sends an unblock item to all attached queues."""
+ for q, v in self._queues_to_unblock.values():
+ try:
+ q.put_nowait ( v )
+ except:
+ pass
+
+ def push ( self, context, error ):
+ """Pushes an exception. This also triggers on-error mode, which
+ unblock all attached queues.
+
+ arguments:
+ * context -- the origin of the exception
+ * error -- the exception
+ """
+ with self._lock:
+ self._exceptions.append ( ( context, error ) )
+ self.empty = False
+ self._unblock_queues()
+
+ def unblock_queues ( self ):
+ """Unblocks all attached queues."""
+ with self._lock:
+ self._unblock_queues()
+
+ def attach_queue ( self, q, unblock_item ):
+ """Attaches a queue. Nothing will be done with it, unless an exception
+ is pushed to this ErrorQueue, in which case all attached queues will
+ be unblocked which allows queue-waiting threads to end.
+
+ arguments:
+ * q -- queue
+ * unblock_item -- item that is used for unblocking, e.g. 'None'
+ """
+ with self._lock:
+ self._queues_to_unblock [id (q)] = ( q, unblock_item )
+
+ def remove_queue ( self, q ):
+ """Removes a queue. It will no longer receive an unblock item if
+ on error mode.
+
+ arguments:
+ * q -- queue to remove
+ """
+ self._lock.acquire()
+ try:
+ del self._queues_to_unblock [id (q)]
+ except KeyError:
+ pass
+ finally:
+ self._lock.release()
+
+ def peek ( self ):
+ """Returns the latest pushed exception."""
+ return self._exceptions [-1]
+
+ def get_all ( self ):
+ """Returns all pushed exceptions."""
+ # not copying, caller shouldn't modify the exception list
+ return self._exceptions
+
+ def get_exceptions ( self ):
+ """Similar to get_all, but a generator that filters out no-op exception
+ pushes that are used to trigger on-error mode without a valid exception.
+ """
+ for e in self._exceptions:
+ if isinstance ( e [1], ( Exception, KeyboardInterrupt ) ):
+ yield e
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index bb2391b..78be3c3 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -18,7 +18,7 @@ except ImportError:
import Queue as queue
-from roverlay import config
+from roverlay import config, errorqueue
from roverlay.overlay import Overlay
from roverlay.overlay.worker import OverlayWorker
from roverlay.packageinfo import PackageInfo
@@ -83,35 +83,29 @@ class OverlayCreator ( object ):
else:
self.logger = logger.getChild ( 'OverlayCreator' )
+ # this queue is used to propagate exceptions from threads
+ self._err_queue = errorqueue.ErrorQueue()
+
# init overlay using config values
self.overlay = Overlay ( logger=self.logger )
- self.depresolver = easyresolver.setup()
+ self.depresolver = easyresolver.setup ( self._err_queue )
self.NUMTHREADS = config.get ( 'EBUILD.jobcount', 0 )
- # --
self._pkg_queue = queue.Queue()
-
- # this queue is used to propagate exceptions from threads
- # it's
- self._err_queue = queue.Queue()
+ self._err_queue.attach_queue ( self._pkg_queue, None )
#self._time_start_run = list()
#self._time_stop_run = list()
-
self._workers = None
self._runlock = threading.RLock()
self.can_write_overlay = OVERLAY_WRITE_ALLOWED
-
- self.depresolver.set_exception_queue ( self._err_queue )
-
self.closed = False
-
# queued packages counter,
# package_added != (create_success + create_fail) if a thread hangs
# or did not call _pkg_done
@@ -137,9 +131,6 @@ class OverlayCreator ( object ):
processed = pkg_created + pkg_failed
failed = pkg_failed + ov_failed
-
- # namedtuple? TODO
-
return (
pkg_added, pkg_created, pkg_failed,
ov_added, ov_failed,
@@ -272,7 +263,7 @@ class OverlayCreator ( object ):
self.join()
self._make_workers()
except:
- self._err_queue.put_nowait ( ( -1, None ) )
+ self._err_queue.push ( context=-1, error=None )
raise
finally:
self._runlock.release()
@@ -324,7 +315,7 @@ class OverlayCreator ( object ):
if self.NUMTHREADS > 0: start = time.time()
if do_close:
- self._err_queue.put_nowait ( ( -1, None ) )
+ self._err_queue.push ( context=-1, error=None )
# fixme: remove enabled?
for w in self._workers: w.enabled = False
else:
@@ -336,14 +327,9 @@ class OverlayCreator ( object ):
del self._workers
self._workers = None
- while not self._err_queue.empty():
- e = self._err_queue.get_nowait()
- self._err_queue.put_nowait ( ( -2, None ) )
- if isinstance ( e [1], ( Exception, KeyboardInterrupt ) ):
- self._err_queue.put ( e )
- self.logger.warning ( "Reraising thread exception." )
- raise e [1]
-
+ for e in self._err_queue.get_exceptions():
+ self.logger.warning ( "Reraising thread exception." )
+ raise e [1]
except ( Exception, KeyboardInterrupt ) as err:
@@ -355,11 +341,9 @@ class OverlayCreator ( object ):
# )
try:
- self._err_queue.put_nowait ( ( -1, None ) )
- self.depresolver.unblock_channels()
- while hasattr ( self, '_workers' ) and self._workers is not None:
-
+ self._err_queue.push ( context=-1, error=None )
+ while hasattr ( self, '_workers' ) and self._workers is not None:
if True in ( w.active() for w in self._workers ):
self._pkg_queue.put_nowait ( None )
else:
diff --git a/roverlay/overlay/worker.py b/roverlay/overlay/worker.py
index 541de7f..880835d 100644
--- a/roverlay/overlay/worker.py
+++ b/roverlay/overlay/worker.py
@@ -106,27 +106,18 @@ class OverlayWorker ( object ):
try:
self.running = True
self.halting = False
- while self.enabled or (
- self.halting and not self.pkg_queue.empty()
+ while ( self.enabled or (
+ self.halting and not self.pkg_queue.empty()
+ ) and \
+ self.err_queue.empty
):
- if not self.err_queue.empty():
- # other workers died (or exit request sent)
- debug ( "STOPPING #1" )
- break
-
debug ( "WAITING" )
p = self.pkg_queue.get()
debug ( "RECEIVED A TASK, " + str ( p ) )
- if not self.err_queue.empty():
- debug ( "STOPPING #2" )
- break
-
# drop empty requests that are used to unblock get()
- if p is not None:
+ if p is not None and self.err_queue.empty:
debug ( "ENTER PROC" )
- if self.err_queue.empty():
- debug ( "__ empty exception/error queue!" )
self._process ( p )
elif self.halting:
# receiving an empty request while halting means 'stop now',
@@ -138,7 +129,7 @@ class OverlayWorker ( object ):
debug ( "STOPPING - DONE" )
except ( Exception, KeyboardInterrupt ) as e:
self.logger.exception ( e )
- self.err_queue.put_nowait ( ( id ( self ), e ) )
+ self.err_queue.push ( id ( self ), e )
self.running = False
diff --git a/roverlay/recipe/easyresolver.py b/roverlay/recipe/easyresolver.py
index 749396b..1ed75fa 100644
--- a/roverlay/recipe/easyresolver.py
+++ b/roverlay/recipe/easyresolver.py
@@ -1,19 +1,19 @@
from roverlay import config
-from roverlay.depres import listeners
+from roverlay.depres import listeners, deptype
from roverlay.depres.depresolver import DependencyResolver
from roverlay.depres.simpledeprule import SimpleDependencyRulePool
-def setup():
- res = DependencyResolver()
+def setup ( err_queue ):
+ res = DependencyResolver ( err_queue=err_queue )
# log everything
res.set_logmask ( -1 )
srule_files = config.get ( 'DEPRES.simple_rules.files', None )
if srule_files:
- srule_pool = SimpleDependencyRulePool ( 'default pool', priority=45 )
+ srule_pool = SimpleDependencyRulePool ( 'default pool', priority=45, deptype_mask=deptype.RESOLVE_ALL, )
srule_pool.get_reader().read ( srule_files )
res.add_rulepool ( srule_pool )
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2012-07-12 18:10 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2012-07-12 18:10 [gentoo-commits] proj/R_overlay:depres_wip commit in: roverlay/ebuild/, roverlay/depres/simpledeprule/, roverlay/depres/, roverlay/, André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox