* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-06-21 16:55 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-06-21 16:55 UTC (permalink / raw
To: gentoo-commits
commit: 35da2e8aaccd1b1163016c861aa06fd53f4875a8
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jun 21 16:45:02 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jun 21 16:45:02 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=35da2e8a
fixes in overlay/
modified: roverlay/overlay/__init__.py
modified: roverlay/overlay/package.py
---
roverlay/overlay/__init__.py | 10 +++++++++-
roverlay/overlay/package.py | 2 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py
index 7e4bbea..dadf998 100644
--- a/roverlay/overlay/__init__.py
+++ b/roverlay/overlay/__init__.py
@@ -51,7 +51,15 @@ class Overlay ( object ):
else:
self.default_category = default_category
- self.eclass_files = eclass_files
+
+ if eclass_files is None:
+ eclass_files = config.get ( 'OVERLAY.eclass_files', None )
+
+ if isinstance ( eclass_files, str ):
+ self.eclass_files = frozenset ( eclass_files )
+ else:
+ self.eclass_files = eclass_files
+
#
self._profiles_dir = os.path.join ( self.physical_location, 'profiles' )
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 3138779..b571ad9 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -218,7 +218,7 @@ class PackageDir ( object ):
self.name, shortver
)
if SUPPRESS_EXCEPTIONS:
- logger.warning ( msg )
+ self.logger.warning ( msg )
else:
raise Exception ( msg )
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-06-22 18:13 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-06-22 18:13 UTC (permalink / raw
To: gentoo-commits
commit: 9646d809ea212ce68a04904a2d3f2d10d3fed718
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jun 22 18:04:31 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jun 22 18:04:31 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9646d809
fix wait() in overlay creation
* workers now run until the queue is empty after receiving stop_when_empty()
instead of stopping after the next package (which is what close() does)
* added some comments in creator,worker
modified: overlay/creator.py
modified: overlay/worker.py
---
roverlay/overlay/creator.py | 71 ++++++++++++++++++++++++++++++++++++++-----
roverlay/overlay/worker.py | 71 ++++++++++++++++++++++++++++++++++--------
2 files changed, 120 insertions(+), 22 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 0936ca9..bbaaa8e 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -26,6 +26,7 @@ OVERLAY_WRITE_ALLOWED = False
class OverlayCreator ( object ):
+ """This is a 'R packages -> Overlay' interface."""
def __init__ ( self, logger=None ):
@@ -51,7 +52,13 @@ class OverlayCreator ( object ):
# --- end of __init__ (...) ---
def _timestamp ( self, description, start, stop=None ):
+ """Logs a timestamp, used for testing.
+ arguments:
+ * description -- timestamp text
+ * start -- when measuring for this timestamp has been started
+ * stop -- stop time; defaults to now (time.time()) if unset
+ """
_stop = time.time() if stop is None else stop
delta = _stop - start
@@ -87,6 +94,11 @@ class OverlayCreator ( object ):
# --- end of add_distdir (...) ---
def write_overlay ( self, incremental=False ):
+ """Writes the overlay.
+
+ arguments:
+ * incremental -- (TODO)
+ """
if self.can_write_overlay:
start = time.time()
if incremental:
@@ -101,10 +113,12 @@ class OverlayCreator ( object ):
# --- end of write_overlay (...) ---
def show_overlay ( self ):
+ """Prints the overlay to the console. Does not create Manifest files."""
self.overlay.show()
# --- end of show_overlay (...) ---
def run ( self ):
+ """Starts ebuild creation and waits until done."""
self._runlock.acquire()
self.start()
self.join()
@@ -112,25 +126,30 @@ class OverlayCreator ( object ):
# --- end of run (...) ---
def start ( self ):
+ """Starts ebuild creation."""
self._runlock.acquire()
self.join()
self._make_workers()
self._runlock.release()
# --- end of start (...) ---
- def wait ( self ):
- self._close_workers()
+ def join ( self ):
+ """Waits until ebuild creation is done."""
+ self._join_workers()
# --- end of wait (...) ---
- join = wait
-
def close ( self, write=False ):
+ """Closes this OverlayCreator."""
self._close_workers()
self._close_resolver()
if write: self.write_overlay()
# --- end of close (...) ---
def _close_resolver ( self ):
+ """Tells the dependency resolver to close.
+ This is useful 'cause certain depres listener modules will write files
+ when told to exit.
+ """
if self.depresolver is None: return
self._runlock.acquire()
@@ -143,7 +162,12 @@ class OverlayCreator ( object ):
self._runlock.release()
# --- end of _close_resolver (...) ---
- def _close_workers ( self ):
+ def _waitfor_workers ( self, do_close ):
+ """Waits until the workers are done.
+
+ arguments:
+ * do_close -- close (exit) workers if True, else wait until done.
+ """
if self._workers is None: return
self._runlock.acquire()
@@ -151,7 +175,10 @@ class OverlayCreator ( object ):
if self.NUMTHREADS > 0: start = time.time()
- for w in self._workers: w.enabled = False
+ if do_close:
+ for w in self._workers: w.enabled = False
+ else:
+ for w in self._workers: w.stop_when_empty()
while True in ( w.active() for w in self._workers ):
self._pkg_queue.put ( None )
@@ -159,13 +186,34 @@ class OverlayCreator ( object ):
del self._workers
self._workers = None
- if self.NUMTHREADS > 0:
+ self._runlock.release()
+ if self.NUMTHREADS > 0: return start
+ # --- end of _waitfor_workers (...) ---
+
+ def _join_workers ( self ):
+ """Waits until all workers are done."""
+ start = self._waitfor_workers ( False )
+ if start is not None:
self._timestamp ( "worker threads are done", start )
+ # --- end of _join_workers (...) ---
- self._runlock.release()
+ def _close_workers ( self ):
+ """Tells the workers to exit.
+ This is done by disabling them and inserting empty requests (None as
+ PackageInfo) to unblock them.
+ """
+ start = self._waitfor_workers ( True )
+ if start is not None:
+ self._timestamp ( "worker threads are closed", start )
# --- end of _close_workers (...) ---
def _pkg_done ( self, package_info ):
+ """This is an event method used by worker threads when they have
+ processed a package info.
+
+ arguments:
+ * package_info --
+ """
# ... TODO
# * increase the number of successful/failed packages,
# * request an incremental write to save memory etc.
@@ -176,6 +224,12 @@ class OverlayCreator ( object ):
# --- end of _add_to_overlay (...) ---
def _get_worker ( self, start_now=False, use_threads=True ):
+ """Creates and returns a worker.
+
+ arguments:
+ * start_now -- if set and True: start the worker immediately
+ * use_threads -- if set and False: disable threads
+ """
w = OverlayWorker (
self._pkg_queue, self.depresolver, self.logger, self._pkg_done,
use_threads=use_threads
@@ -185,6 +239,7 @@ class OverlayCreator ( object ):
# --- end of _get_worker (...) ---
def _make_workers ( self ):
+ """Creates and starts workers."""
self._close_workers()
if self.NUMTHREADS > 0:
diff --git a/roverlay/overlay/worker.py b/roverlay/overlay/worker.py
index 4f03fe2..471cb94 100644
--- a/roverlay/overlay/worker.py
+++ b/roverlay/overlay/worker.py
@@ -1,34 +1,44 @@
+# R Overlay -- R package -> overlay interface, PackageInfo worker
+# Copyright 2006-2012 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
-import time
+#import time
import threading
-try:
- import queue
-except ImportError:
- # python2
- import Queue as queue
-
-
from roverlay.depres.channels import EbuildJobChannel
from roverlay.ebuild.creation import EbuildCreation
class OverlayWorker ( object ):
-
- def __init__ ( self, pkg_queue, depresolver, logger, pkg_done, use_threads ):
+ """Overlay package queue worker."""
+
+ def __init__ ( self,
+ pkg_queue, depresolver, logger, pkg_done, use_threads
+ ):
+ """Initializes a worker.
+
+ arguments:
+ * pkg_queue -- queue with PackageInfos
+ * depresolver -- dependency resolver to use
+ * logger -- logger to use
+ * pkg_done -- method to call when a PackageInfo has been
+ processed
+ * use_threads -- whether to run this worker as a thread or not
+ """
self.logger = logger
self.pkg_queue = pkg_queue
self.pkg_done = pkg_done
self.depresolver = depresolver
-
- self.enabled = True
- self.running = False
self._use_thread = use_threads
self._thread = None
+ self.enabled = True
+ self.running = False
+ self.halting = False
# --- end of __init__ (...) ---
def start ( self ):
+ """Starts the worker."""
if self._thread is not None:
self._thread.join()
del self._thread
@@ -41,17 +51,34 @@ class OverlayWorker ( object ):
self._run_nothread()
# --- end of start (...) ---
+ def stop_when_empty ( self ):
+ """Tells the worker thread to exit when the queue is empty."""
+ self.enabled = False
+ self.halting = True
+ # --- end of stop_when_empty (...) ---
+
def active ( self ):
+ """Returns True if this worker is active (running or enabled)."""
return self.running or self.enabled
# --- end of active (...) ---
def _get_resolver_channel ( self, **channel_kw ):
+ """Returns a resolver channel.
+
+ arguments:
+ * **channel_kw -- keywords for EbuildJobChannel.__init__
+ """
return self.depresolver.register_channel (
EbuildJobChannel ( **channel_kw )
)
# --- end of _get_resolver_channel (...) ---
def _process ( self, package_info ):
+ """Processes a PackageInfo taken from the queue.
+
+ arguments:
+ * package_info --
+ """
job = EbuildCreation (
package_info,
depres_channel_spawner=self._get_resolver_channel
@@ -61,21 +88,37 @@ class OverlayWorker ( object ):
# --- end of _process (...) ---
def _run ( self ):
+ """Runs the worker (thread mode)."""
self.running = True
- while self.enabled:
+ self.halting = False
+ while self.enabled or (
+ self.halting and not self.pkg_queue.empty()
+ ):
+ if not self.running:
+ # exit now
+ break
p = self.pkg_queue.get()
# drop empty requests that are used to unblock get()
if p is not None:
self._process ( p )
+ elif self.halting:
+ # receiving an empty request while halting means 'stop now',
+ self.enabled = False
+ self.halting = False
self.pkg_queue.task_done()
self.running = False
# --- end of run (...) ---
def _run_nothread ( self ):
+ """Runs the worker (no-thread mode)."""
self.running = True
while self.enabled and not self.pkg_queue.empty():
+ if not self.running:
+ # exit now
+ break
+
p = self.pkg_queue.get()
# drop empty requests that are used to unblock get()
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-06-26 15:42 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-06-26 15:42 UTC (permalink / raw
To: gentoo-commits
commit: 544b6f2f6668a3bb254209a7afe498f9925fe2cb
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jun 26 15:39:51 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jun 26 15:39:51 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=544b6f2f
overlay creation: unused method stubs removed
modified: roverlay/overlay/creator.py
---
roverlay/overlay/creator.py | 16 ++--------------
1 files changed, 2 insertions(+), 14 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index bbaaa8e..6d97d48 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -47,6 +47,8 @@ class OverlayCreator ( object ):
self._workers = None
self._runlock = threading.RLock()
+ self.package_add = self._pkg_queue.put
+
self.can_write_overlay = OVERLAY_WRITE_ALLOWED
# --- end of __init__ (...) ---
@@ -68,11 +70,6 @@ class OverlayCreator ( object ):
return _stop
# --- end of _timestamp (...) ---
- def add ( self, *to_add ):
- """Add a directory/package files/distdir to the package queue."""
- raise Exception ( "method stub" )
- # --- end of add (...) ---
-
def add_package_file ( self, package_file ):
"""Adds a single R package."""
self._pkg_queue.put ( PackageInfo ( filepath=package_file ) )
@@ -83,15 +80,6 @@ class OverlayCreator ( object ):
for p in package_files: self.add_package_file ( p )
# --- end of add_packages (...) ---
- def add_directory ( self, directory ):
- """Adds all packages from a directory to the package queue."""
- raise Exception ( "method stub" )
- # --- end of add_directory (...) ---
-
- def add_distdir ( self, repo_name ):
- """Adds a distfiles directory to the package queue."""
- self.add_directory ( config.get_or_fail ( [ 'DISTFILES', repo_name ] ) )
- # --- end of add_distdir (...) ---
def write_overlay ( self, incremental=False ):
"""Writes the overlay.
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-06-27 14:46 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-06-27 14:46 UTC (permalink / raw
To: gentoo-commits
commit: d145bb5b3a9f0fc8f93d52cfe8c3002cc1bb416d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun 27 14:40:43 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun 27 14:40:43 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d145bb5b
typo
---
roverlay/overlay/creator.py | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 6d97d48..608ba77 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -47,10 +47,11 @@ class OverlayCreator ( object ):
self._workers = None
self._runlock = threading.RLock()
- self.package_add = self._pkg_queue.put
-
self.can_write_overlay = OVERLAY_WRITE_ALLOWED
+ # this is a method that adds PackageInfo objects to the pkg queue
+ self.add_package = self._pkg_queue.put
+
# --- end of __init__ (...) ---
def _timestamp ( self, description, start, stop=None ):
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-07-03 17:48 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-07-03 17:48 UTC (permalink / raw
To: gentoo-commits
commit: 396f148a3912c82357e68533eb6858a252ca7307
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 3 17:41:06 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 3 17:41:06 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=396f148a
reduce default numthreads to 0
---
roverlay/overlay/creator.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index aa1a3a8..79f2da3 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -43,7 +43,7 @@ class OverlayCreator ( object ):
self.depresolver = easyresolver.setup()
- self.NUMTHREADS = config.get ( 'EBUILD.jobcount', 10 )
+ self.NUMTHREADS = config.get ( 'EBUILD.jobcount', 0 )
# --
self._pkg_queue = queue.Queue()
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-07-04 18:21 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-07-04 18:21 UTC (permalink / raw
To: gentoo-commits
commit: 6d1ec878f71025e6681255f8b63cd404f9ce74f3
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 4 18:10:14 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 4 18:10:14 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6d1ec878
overlay: collect some stats
modified: roverlay/overlay/__init__.py
modified: roverlay/overlay/category.py
modified: roverlay/overlay/creator.py
modified: roverlay/overlay/package.py
---
roverlay/overlay/__init__.py | 4 +-
roverlay/overlay/category.py | 4 +-
roverlay/overlay/creator.py | 167 +++++++++++++++++++++++++++++++++++++++---
roverlay/overlay/package.py | 22 +++---
4 files changed, 171 insertions(+), 26 deletions(-)
diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py
index a8a939d..13b6615 100644
--- a/roverlay/overlay/__init__.py
+++ b/roverlay/overlay/__init__.py
@@ -99,9 +99,9 @@ class Overlay ( object ):
* category -- category where the pkg should be put in, defaults to
self.default_category
- returns: None (implicit)
+ returns: True if successfully added else False
"""
- self._get_category (
+ return self._get_category (
self.default_category if category is None else category
) . add ( package_info )
# --- end of add (...) ---
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index e938dbf..271b524 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -46,7 +46,7 @@ class Category ( object ):
arguments:
* package_info --
- returns: None (implicit)
+ returns: success
"""
pkg_name = package_info ['name']
@@ -63,7 +63,7 @@ class Category ( object ):
finally:
self._lock.release()
- self._subdirs [pkg_name].add ( package_info )
+ return self._subdirs [pkg_name].add ( package_info )
# --- end of add (...) ---
def generate_metadata ( self, **metadata_kw ):
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 79f2da3..bb2391b 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -5,10 +5,12 @@
import time
import logging
import threading
-import signal
-import traceback
+#import signal
+#import traceback
import sys
+from collections import deque
+
try:
import queue
except ImportError:
@@ -23,18 +25,61 @@ from roverlay.packageinfo import PackageInfo
from roverlay.recipe import easyresolver
-LOGGER = logging.getLogger ( 'OverlayCreator' )
-
+# this is used as the default value for can_write_overlay in OverlayCreator
+# instances
OVERLAY_WRITE_ALLOWED = False
+class PseudoAtomicCounter ( object ):
+
+ def __init__ ( self, start=0, long_int=False ):
+ if long_int and sys.version_info < ( 3, 0 ):
+ self._value = long ( start )
+ else:
+ self._value = int ( start )
+ self._lock = threading.Lock()
+
+ def _get_and_inc ( self, step ):
+ ret = None
+ self._lock.acquire()
+ try:
+ old_val = self._value
+ if step > 0:
+ self._value += step
+ ret = ( self._value, old_val )
+ else:
+ ret = old_val
+ finally:
+ self._lock.release()
+
+ return ret
+ # --- end of _get_and_inc (...) ---
+
+ def inc ( self, step=1 ):
+ self._get_and_inc ( step )
+ # --- end of inc (...) ---
+
+ def get ( self ):
+ return self._get_and_inc ( 0 )
+ # --- end of get (...) ---
+
+ def get_nowait ( self ):
+ return self._value
+ # --- end of get_nowait (...) ---
+
+ def __str__ ( self ):
+ return str ( self._value )
+ # --- end of __str__ (...) ---
+
class OverlayCreator ( object ):
"""This is a 'R packages -> Overlay' interface."""
+ LOGGER = logging.getLogger ( 'OverlayCreator' )
+
def __init__ ( self, logger=None ):
if logger is None:
- self.logger = LOGGER
+ self.logger = self.__class__.LOGGER
else:
self.logger = logger.getChild ( 'OverlayCreator' )
@@ -52,19 +97,97 @@ class OverlayCreator ( object ):
# it's
self._err_queue = queue.Queue()
+ #self._time_start_run = list()
+ #self._time_stop_run = list()
+
self._workers = None
self._runlock = threading.RLock()
self.can_write_overlay = OVERLAY_WRITE_ALLOWED
- # this is a method that adds PackageInfo objects to the pkg queue
- self.add_package = self._pkg_queue.put
self.depresolver.set_exception_queue ( self._err_queue )
+ self.closed = False
+
+
+ # queued packages counter,
+ # package_added != (create_success + create_fail) if a thread hangs
+ # or did not call _pkg_done
+ self.package_added = PseudoAtomicCounter()
+
+ # counts packages that passed ebuild creation
+ self.create_success = PseudoAtomicCounter()
+
+ # counts packages that failed ebuild creation
+ self.create_fail = PseudoAtomicCounter()
+
+ # counts packages that passed adding to overlay
+ self.overlay_added = PseudoAtomicCounter()
+
# --- end of __init__ (...) ---
+ def get_stats ( self ):
+ pkg_added = self.package_added.get_nowait()
+ pkg_created = self.create_success.get_nowait()
+ pkg_failed = self.create_fail.get_nowait()
+ ov_added = self.overlay_added.get_nowait()
+ ov_failed = pkg_created - ov_added
+ processed = pkg_created + pkg_failed
+ failed = pkg_failed + ov_failed
+
+
+ # namedtuple? TODO
+
+ return (
+ pkg_added, pkg_created, pkg_failed,
+ ov_added, ov_failed,
+ processed, failed
+ )
+ # --- end of get_stats (...) ---
+
+ def stats_str ( self, enclose=True ):
+ """Returns a string with some overlay creation stats."""
+ def stats_gen():
+ """Yields stats strings."""
+ stats = self.get_stats()
+
+ # the length of the highest number in stats (^=digit count)
+ # max_number_len := { 1,...,5 }
+ max_number_len = min ( 5, len ( str ( max ( stats ) ) ) )
+ num_fmt = '%(num)-' + str ( max_number_len ) + 's '
+
+ for i, s in enumerate ((
+ 'packages added to the ebuild creation queue',
+ 'packages passed ebuild creation',
+ 'packages failed ebuild creation',
+ 'ebuilds could be added to the overlay',
+ 'ebuilds couldn\'t be added to the overlay',
+ 'packages processed in total',
+ 'packages failed in total',
+ )):
+ yield num_fmt % { 'num' : stats [i] } + s
+ # --- end of stats_gen (...) ---
+
+ if enclose:
+ stats_str = deque ( stats_gen() )
+ # maxlen := { 2,...,80 }
+ maxlen = 2 + min ( 78,
+ len ( max ( stats_str, key=lambda s : len( s ) ) )
+ )
+
+ stats_str.appendleft (
+ " Overlay creation stats ".center ( maxlen, '-' )
+ )
+ stats_str.append ( '-' * maxlen )
+
+ return '\n'.join ( stats_str )
+
+ else:
+ return '\n'.join ( stats_gen() )
+ # --- end of stats_str (...) ---
+
def _timestamp ( self, description, start, stop=None ):
"""Logs a timestamp, used for testing.
@@ -82,14 +205,25 @@ class OverlayCreator ( object ):
return _stop
# --- end of _timestamp (...) ---
+ def add_package ( self, package_info ):
+ """Adds a PackageInfo to the package queue.
+
+ arguments:
+ * package_info --
+ """
+ self._pkg_queue.put ( package_info )
+ self.package_added.inc()
+
def add_package_file ( self, package_file ):
"""Adds a single R package."""
self._pkg_queue.put ( PackageInfo ( filepath=package_file ) )
+ self.package_added.inc()
# --- end of add_package (...) ---
def add_package_files ( self, *package_files ):
"""Adds multiple R packages."""
for p in package_files: self.add_package_file ( p )
+ self.package_added.inc()
# --- end of add_packages (...) ---
@@ -117,14 +251,18 @@ class OverlayCreator ( object ):
self.overlay.show()
# --- end of show_overlay (...) ---
- def run ( self ):
+ def run ( self, close_when_done=False ):
"""Starts ebuild creation and waits until done."""
self._runlock.acquire()
+ #self._time_start_run.append ( time.time() )
try:
self.start()
self.join()
finally:
+ #self._time_stop_run.append ( time.time() )
self._runlock.release()
+ if close_when_done:
+ self.close()
# --- end of run (...) ---
def start ( self ):
@@ -145,11 +283,11 @@ class OverlayCreator ( object ):
self._join_workers()
# --- end of wait (...) ---
- def close ( self, write=False ):
+ def close ( self ):
"""Closes this OverlayCreator."""
self._close_workers()
self._close_resolver()
- if write: self.write_overlay()
+ self.closed = True
# --- end of close (...) ---
def _close_resolver ( self ):
@@ -266,8 +404,13 @@ class OverlayCreator ( object ):
# * request an incremental write to save memory etc.
# if <>:
- if package_info ['ebuild'] is not None:
- self.overlay.add ( package_info )
+ if package_info ['ebuild'] is None:
+ self.create_fail.inc()
+ else:
+ self.create_success.inc()
+ if self.overlay.add ( package_info ):
+ self.overlay_added.inc()
+
# --- end of _add_to_overlay (...) ---
def _get_worker ( self, start_now=False, use_threads=True ):
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 77fcbee..429200f 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -203,11 +203,9 @@ class PackageDir ( object ):
"""
shortver = package_info ['ebuild_verstr']
- def already_exists ( release=False ):
+ def already_exists ():
if shortver in self._packages:
- if release: self._lock.release()
-
msg = "'%s-%s.ebuild' already exists, cannot add it!" % (
self.name, shortver
)
@@ -221,14 +219,18 @@ class PackageDir ( object ):
return False
# --- end of already_exists (...) ---
- if already_exists ( release=False ): return False
- self._lock.acquire()
- if already_exists ( release=True ): return False
-
- self._packages [shortver] = package_info
+ _success = False
- self._lock.release()
- return True
+ if not already_exists():
+ try:
+ self._lock.acquire()
+ if not already_exists():
+ self._packages [shortver] = package_info
+ _success = True
+ finally:
+ self._lock.release()
+
+ return _success
# --- end of add (...) ---
def _regen_metadata ( self ):
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-07-05 16:00 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-07-05 16:00 UTC (permalink / raw
To: gentoo-commits
commit: 9d2a243a6e6eadc71e3962726b29c84d1251d732
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 5 15:59:40 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 5 15:59:40 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9d2a243a
overlay: calculate inherit statement for ebuilds
modified: roverlay/overlay/__init__.py
---
roverlay/overlay/__init__.py | 83 +++++++++++++++++++++++++++++++++---------
1 files changed, 66 insertions(+), 17 deletions(-)
diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py
index 13b6615..b6a65a1 100644
--- a/roverlay/overlay/__init__.py
+++ b/roverlay/overlay/__init__.py
@@ -60,6 +60,7 @@ class Overlay ( object ):
else:
self.eclass_files = eclass_files
+ self.eclass_names = None
#
self._profiles_dir = os.path.join ( self.physical_location, 'profiles' )
@@ -115,7 +116,7 @@ class Overlay ( object ):
returns: None (implicit)
"""
for cat in self._categories.values():
- cat.show ( default_header=self._default_header )
+ cat.show ( default_header=self._get_header() )
# --- end of show (...) ---
def write ( self, **write_kw ):
@@ -138,7 +139,7 @@ class Overlay ( object ):
for cat in self._categories.values():
if cat.physical_location and not cat.empty():
util.dodir ( cat.physical_location )
- cat.write ( default_header=self._default_header )
+ cat.write ( default_header=self._get_header() )
self._write_categories ( only_active=True )
# --- end of write (...) ---
@@ -253,29 +254,58 @@ class Overlay ( object ):
self._write_profiles_file ( 'use.desc', use_desc + '\n' )
# --- end of _write_usedesc (...) ---
+ def _get_eclass_import_info ( self, only_eclass_names=False ):
+ """Yields eclass import information (eclass names and files).
+
+ arguments:
+ * only_eclass_names -- if True: yield eclass dest names only,
+ else : yield (eclass name, eclass src file)
+ Defaults to False.
+
+ raises: AssertionError if a file does not end with '.eclass'.
+ """
+ if self.eclass_files:
+
+ for eclass in self.eclass_files:
+ dest = os.path.splitext ( os.path.basename ( eclass ) )
+
+ if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ):
+ if only_eclass_names:
+ yield dest[0]
+ else:
+ yield ( dest[0], eclass )
+ else:
+ raise AssertionError (
+ "{!r} does not end with '.eclass'!".format ( eclass )
+ )
+ # --- end of _get_eclass_import_info (...) ---
+
def _import_eclass ( self, reimport_eclass ):
+ """Imports eclass files to the overlay. Also sets ebuild_names.
+
+ arguments:
+ * reimport_eclass -- whether to import existing eclass files (again)
+
+ raises:
+ * AssertionError, passed from _get_eclass_import_info()
+ * Exception if copying fails
+ """
if self.eclass_files:
# import eclass files
eclass_dir = os.path.join ( self.physical_location, 'eclass' )
try:
+ eclass_names = list()
util.dodir ( eclass_dir )
- for eclass in self.eclass_files:
- src = eclass
- dest = None
- if isinstance ( eclass, str ):
- dest = os.path.basename ( eclass )
- else:
- # list-like specification ( src, destname )
- src = eclass [0]
- dest = eclass [1]
-
- dest = os.path.join ( eclass_dir, dest )
-
+ for destname, eclass in self._get_eclass_import_info ( False ):
+ dest = os.path.join ( eclass_dir, destname + '.eclass' )
if reimport_eclass or not os.path.isfile ( dest ):
- shutil.copyfile ( src, dest )
+ shutil.copyfile ( eclass, dest )
+ eclass_names.append ( destname )
+
+ self.eclass_names = frozenset ( eclass_names )
except Exception as e:
self.logger.critical ( "Cannot import eclass files!" )
@@ -311,10 +341,29 @@ class Overlay ( object ):
self.logger.exception ( e )
self.logger.critical ( "^failed to init overlay" )
raise
+ # --- end of _init_overlay (...) ---
+
+ def _get_header ( self ):
+ """Returns the ebuild header (including inherit <eclasses>)."""
+ if self.eclass_names is None:
+ # writing is possibly disabled since eclass files have not been
+ # imported (or show() used before write())
+ inherit = ' '.join ( self._get_eclass_import_info ( True ) )
+ else:
+ inherit = ' '.join ( self.eclass_names )
+ inherit = "inherit " + inherit if inherit else None
+ # header and inherit is expected and therefore the first condition here
+ if inherit and self._default_header:
+ return '\n'.join (( self._default_header, '', inherit ))
+ elif inherit:
+ return inherit
+ elif self._default_header:
+ return self._default_header
-
-
+ else:
+ return None
+ # --- end of _get_header (...) ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/
@ 2012-07-24 16:59 André Erdmann
2012-07-30 8:52 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
0 siblings, 1 reply; 42+ messages in thread
From: André Erdmann @ 2012-07-24 16:59 UTC (permalink / raw
To: gentoo-commits
commit: daed3bde53ce1fec42ad806813bb2798321b76d4
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:21:07 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:21:07 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=daed3bde
overlay: split incremental/runtime_incremental
Also added a _writeable bool to Overlay which controls
whether writing is allowed or not.
geändert: roverlay/overlay/category.py
geändert: roverlay/overlay/creator.py
geändert: roverlay/overlay/package.py
geändert: roverlay/overlay/root.py
---
roverlay/overlay/category.py | 36 +++++++++-----
roverlay/overlay/creator.py | 20 +++----
roverlay/overlay/package.py | 23 ++++++--
roverlay/overlay/root.py | 113 +++++++++++++++++++++++++++++------------
4 files changed, 129 insertions(+), 63 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index fab6373..a0b1607 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -16,25 +16,35 @@ class Category ( object ):
WRITE_JOBCOUNT = 3
- def __init__ ( self, name, logger, directory, get_header, incremental ):
+ def __init__ ( self,
+ name, logger, directory, get_header, runtime_incremental
+ ):
"""Initializes a overlay/portage category (such as 'app-text', 'sci-R').
arguments:
- * name -- name of the category
- * logger -- parent logger
- * directory -- filesystem location
- * get_header -- function that returns an ebuild header
+ * name -- name of the category
+ * logger -- parent logger
+ * directory -- filesystem location
+ * get_header -- function that returns an ebuild header
+ * runtime_incremental -- enable/disable runtime incremental writing
+ for this category (and all created PackageDirs)
"""
- self.logger = logger.getChild ( name )
- self.name = name
- self._lock = threading.RLock()
- self._subdirs = dict()
- self.physical_location = directory
- self.get_header = get_header
- self.incremental = incremental
+ self.logger = logger.getChild ( name )
+ self.name = name
+ self._lock = threading.RLock()
+ self._subdirs = dict()
+ self.physical_location = directory
+ self.get_header = get_header
+ self.runtime_incremental = runtime_incremental
# --- end of __init__ (...) ---
def _get_package_dir ( self, pkg_name ):
+ """Returns a PackageDir object for pkg_name.
+ (so that <new object>.name == pkg_name and pkg_name in self._subdirs)
+
+ arguments:
+ * pkg_name --
+ """
if not pkg_name in self._subdirs:
self._lock.acquire()
try:
@@ -44,7 +54,7 @@ class Category ( object ):
logger = self.logger,
directory = self.physical_location + os.sep + pkg_name,
get_header = self.get_header,
- incremental = self.incremental
+ runtime_incremental = self.runtime_incremental
)
self._subdirs [pkg_name] = newpkg
finally:
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index d12e0a5..60e9517 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -82,18 +82,16 @@ class OverlayCreator ( object ):
# this queue is used to propagate exceptions from threads
self._err_queue = errorqueue.ErrorQueue()
- self.can_write_overlay = allow_write
- self.write_incremental = allow_write and USE_INCREMENTAL_WRITE
-
# init overlay using config values
self.overlay = Overlay (
- name=config.get_or_fail ( 'OVERLAY.name' ),
- logger=self.logger,
- directory=config.get_or_fail ( 'OVERLAY.dir' ),
- default_category= config.get_or_fail ( 'OVERLAY.category' ),
- eclass_files=config.get ( 'OVERLAY.eclass_files', None ),
- ebuild_header=config.get ( 'EBUILD.default_header', None ),
- incremental=self.write_incremental
+ name = config.get_or_fail ( 'OVERLAY.name' ),
+ logger = self.logger,
+ directory = config.get_or_fail ( 'OVERLAY.dir' ),
+ default_category = config.get_or_fail ( 'OVERLAY.category' ),
+ eclass_files = config.get ( 'OVERLAY.eclass_files', None ),
+ ebuild_header = config.get ( 'EBUILD.default_header', None ),
+ incremental = USE_INCREMENTAL_WRITE,
+ write_allowed = allow_write
)
self.depresolver = easyresolver.setup ( self._err_queue )
@@ -219,7 +217,7 @@ class OverlayCreator ( object ):
arguments:
"""
- if self.can_write_overlay:
+ if self.overlay.writeable():
start = time.time()
self.overlay.write()
self._timestamp ( "overlay written", start )
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index d2662bd..bedbce3 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -13,15 +13,25 @@ SUPPRESS_EXCEPTIONS = True
class PackageDir ( object ):
EBUILD_SUFFIX = '.ebuild'
- def __init__ ( self, name, logger, directory, get_header, incremental ):
+ def __init__ ( self,
+ name, logger, directory, get_header, runtime_incremental
+ ):
"""Initializes a PackageDir which contains ebuilds, metadata and
a Manifest file.
arguments:
- * name -- name of the directory (${PN} in ebuilds)
- * logger -- parent logger
- * directory -- filesystem location of this PackageDir
- * get_header -- function that returns an ebuild header
+ * name -- name of the directory (${PN} in ebuilds)
+ * logger -- parent logger
+ * directory -- filesystem location of this PackageDir
+ * get_header -- function that returns an ebuild header
+ * runtime_incremental -- enable/disable runtime incremental ebuild
+ writing. This trades speed (disabled) for
+ memory consumption (enabled) 'cause it will
+ write _all_ successfully created ebuilds
+ directly after they've been created.
+ Writing all ebuilds at once is generally faster
+ (+threading), but all PackageInfos must be
+ kept in memory for that.
"""
self.logger = logger.getChild ( name )
self.name = name
@@ -30,7 +40,7 @@ class PackageDir ( object ):
self._packages = dict()
self.physical_location = directory
self.get_header = get_header
- self.runtime_incremental = incremental
+ self.runtime_incremental = runtime_incremental
self._metadata = MetadataJob (
filepath = self.physical_location + os.sep + 'metadata.xml',
@@ -250,6 +260,7 @@ class PackageDir ( object ):
self._need_manifest = True
self._need_metadata = True
self.modified = True
+ # FIXME: delete PackageInfo if ebuild creation did not succeed
if self.runtime_incremental:
with self._lock:
return self.write_ebuilds ( overwrite=False )
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 219fd70..ae3267f 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -22,23 +22,49 @@ class Overlay ( object ):
def __init__ (
self,
- name, logger, directory,
- default_category, eclass_files,
+ name,
+ logger,
+ directory,
+ default_category,
+ eclass_files,
ebuild_header,
- incremental
+ write_allowed,
+ incremental,
+ runtime_incremental=True
):
- self.name = name
- self.logger = logger.getChild ( 'overlay' )
- self.physical_location = directory
- self.default_category = default_category
- self.eclass_files = eclass_files
+ """Initializes an overlay.
- self.ignore_existing_ebuilds = False
-
- self._profiles_dir = self.physical_location + os.sep + 'profiles'
- self._catlock = threading.Lock()
- self._categories = dict()
- self._header = EbuildHeader ( ebuild_header )
+ arguments:
+ * name -- name of this overlay
+ * logger -- parent logger to use
+ * directory -- filesystem location of this overlay
+ * default_category -- category of packages being added without a
+ specific category
+ * eclass_files -- eclass files to import and
+ inherit in all ebuilds
+ * ebuild_header -- the header text included in all created ebuilds
+ * write_allowed -- whether writing is allowed
+ * incremental -- enable/disable incremental writing:
+ use already existing ebuilds (don't recreate
+ them)
+ * runtime_incremental -- see package.py:PackageDir.__init__ (...),
+ Defaults to ?FIXME?
+ """
+ self.name = name
+ self.logger = logger.getChild ( 'overlay' )
+ self.physical_location = directory
+ self.default_category = default_category
+
+ self._eclass_files = eclass_files
+ #self._incremental = incremental
+ # disable runtime_incremental if writing not allowed
+ self._runtime_incremental = write_allowed and runtime_incremental
+ self._writeable = write_allowed
+
+ self._profiles_dir = self.physical_location + os.sep + 'profiles'
+ self._catlock = threading.Lock()
+ self._categories = dict()
+ self._header = EbuildHeader ( ebuild_header )
# fixme or ignore: calculating eclass names twice,
# once here and another time when calling _init_overlay
@@ -46,13 +72,12 @@ class Overlay ( object ):
self._get_eclass_import_info ( only_eclass_names=True )
) )
- self.incremental = incremental
- if self.incremental:
+ #if self._incremental:
+ if incremental:
# this is multiple-run incremental writing (in contrast to runtime
# incremental writing, which writes ebuilds as soon as they're
- # ready) FIXME: split incremental <-> runtime_incremental
+ # ready)
self.scan()
-
# --- end of __init__ (...) ---
def _get_category ( self, category ):
@@ -70,7 +95,7 @@ class Overlay ( object ):
self.logger,
self.physical_location + os.sep + category,
get_header=self._header.get,
- incremental=self.incremental
+ runtime_incremental=self._runtime_incremental
)
self._categories [category] = newcat
finally:
@@ -89,9 +114,9 @@ class Overlay ( object ):
raises: AssertionError if a file does not end with '.eclass'.
"""
- if self.eclass_files:
+ if self._eclass_files:
- for eclass in self.eclass_files:
+ for eclass in self._eclass_files:
dest = os.path.splitext ( os.path.basename ( eclass ) )
if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ):
@@ -116,7 +141,7 @@ class Overlay ( object ):
* Exception if copying fails
"""
- if self.eclass_files:
+ if self._eclass_files:
# import eclass files
eclass_dir = self.physical_location + os.sep + 'eclass'
try:
@@ -240,6 +265,10 @@ class Overlay ( object ):
yield kwargs
# --- end of list_rule_kwargs (...) ---
+ def readonly ( self ):
+ return not self._writeable
+ # --- end of readonly (...) ---
+
def scan ( self, **kw ):
def scan_categories():
for x in os.listdir ( self.physical_location ):
@@ -271,6 +300,10 @@ class Overlay ( object ):
cat.show ( **show_kw )
# --- end of show (...) ---
+ def writeable ( self ):
+ return self._writeable
+ # --- end of writeable (...) ---
+
def write ( self ):
"""Writes the overlay to its physical location (filesystem), including
metadata and Manifest files as well as cleanup actions.
@@ -285,14 +318,21 @@ class Overlay ( object ):
! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
when ebuild creation is done.
"""
- self._init_overlay ( reimport_eclass=True )
-
- for cat in self._categories.values():
- cat.write (
- overwrite_ebuilds=False,
- keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
- cautious=True
- )
+ if self._writeable:
+ self._init_overlay ( reimport_eclass=True )
+
+ for cat in self._categories.values():
+ cat.write (
+ overwrite_ebuilds=False,
+ keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
+ cautious=True
+ )
+ else:
+ # FIXME debug print
+ print (
+ "Dropped write request for readonly overlay {}!".format (
+ self.name
+ ) )
# --- end of write (...) ---
def write_manifest ( self, **manifest_kw ):
@@ -305,7 +345,14 @@ class Overlay ( object ):
returns: None (implicit)
"""
- # FIXME: it would be good to ensure that profiles/categories exist
- for cat in self._categories.values():
- cat.write_manifest ( **manifest_kw )
+ if self._writeable:
+ # FIXME: it would be good to ensure that profiles/categories exist
+ for cat in self._categories.values():
+ cat.write_manifest ( **manifest_kw )
+ else:
+ # FIXME debug print
+ print (
+ "Dropped write_manifest request for readonly overlay {}!".format (
+ self.name
+ ) )
# --- end of write_manifest (...) ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
2012-07-24 16:59 [gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/ André Erdmann
@ 2012-07-30 8:52 ` André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-07-30 8:52 UTC (permalink / raw
To: gentoo-commits
commit: daed3bde53ce1fec42ad806813bb2798321b76d4
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:21:07 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:21:07 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=daed3bde
overlay: split incremental/runtime_incremental
Also added a _writeable bool to Overlay which controls
whether writing is allowed or not.
geändert: roverlay/overlay/category.py
geändert: roverlay/overlay/creator.py
geändert: roverlay/overlay/package.py
geändert: roverlay/overlay/root.py
---
roverlay/overlay/category.py | 36 +++++++++-----
roverlay/overlay/creator.py | 20 +++----
roverlay/overlay/package.py | 23 ++++++--
roverlay/overlay/root.py | 113 +++++++++++++++++++++++++++++------------
4 files changed, 129 insertions(+), 63 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index fab6373..a0b1607 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -16,25 +16,35 @@ class Category ( object ):
WRITE_JOBCOUNT = 3
- def __init__ ( self, name, logger, directory, get_header, incremental ):
+ def __init__ ( self,
+ name, logger, directory, get_header, runtime_incremental
+ ):
"""Initializes a overlay/portage category (such as 'app-text', 'sci-R').
arguments:
- * name -- name of the category
- * logger -- parent logger
- * directory -- filesystem location
- * get_header -- function that returns an ebuild header
+ * name -- name of the category
+ * logger -- parent logger
+ * directory -- filesystem location
+ * get_header -- function that returns an ebuild header
+ * runtime_incremental -- enable/disable runtime incremental writing
+ for this category (and all created PackageDirs)
"""
- self.logger = logger.getChild ( name )
- self.name = name
- self._lock = threading.RLock()
- self._subdirs = dict()
- self.physical_location = directory
- self.get_header = get_header
- self.incremental = incremental
+ self.logger = logger.getChild ( name )
+ self.name = name
+ self._lock = threading.RLock()
+ self._subdirs = dict()
+ self.physical_location = directory
+ self.get_header = get_header
+ self.runtime_incremental = runtime_incremental
# --- end of __init__ (...) ---
def _get_package_dir ( self, pkg_name ):
+ """Returns a PackageDir object for pkg_name.
+ (so that <new object>.name == pkg_name and pkg_name in self._subdirs)
+
+ arguments:
+ * pkg_name --
+ """
if not pkg_name in self._subdirs:
self._lock.acquire()
try:
@@ -44,7 +54,7 @@ class Category ( object ):
logger = self.logger,
directory = self.physical_location + os.sep + pkg_name,
get_header = self.get_header,
- incremental = self.incremental
+ runtime_incremental = self.runtime_incremental
)
self._subdirs [pkg_name] = newpkg
finally:
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index d12e0a5..60e9517 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -82,18 +82,16 @@ class OverlayCreator ( object ):
# this queue is used to propagate exceptions from threads
self._err_queue = errorqueue.ErrorQueue()
- self.can_write_overlay = allow_write
- self.write_incremental = allow_write and USE_INCREMENTAL_WRITE
-
# init overlay using config values
self.overlay = Overlay (
- name=config.get_or_fail ( 'OVERLAY.name' ),
- logger=self.logger,
- directory=config.get_or_fail ( 'OVERLAY.dir' ),
- default_category= config.get_or_fail ( 'OVERLAY.category' ),
- eclass_files=config.get ( 'OVERLAY.eclass_files', None ),
- ebuild_header=config.get ( 'EBUILD.default_header', None ),
- incremental=self.write_incremental
+ name = config.get_or_fail ( 'OVERLAY.name' ),
+ logger = self.logger,
+ directory = config.get_or_fail ( 'OVERLAY.dir' ),
+ default_category = config.get_or_fail ( 'OVERLAY.category' ),
+ eclass_files = config.get ( 'OVERLAY.eclass_files', None ),
+ ebuild_header = config.get ( 'EBUILD.default_header', None ),
+ incremental = USE_INCREMENTAL_WRITE,
+ write_allowed = allow_write
)
self.depresolver = easyresolver.setup ( self._err_queue )
@@ -219,7 +217,7 @@ class OverlayCreator ( object ):
arguments:
"""
- if self.can_write_overlay:
+ if self.overlay.writeable():
start = time.time()
self.overlay.write()
self._timestamp ( "overlay written", start )
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index d2662bd..bedbce3 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -13,15 +13,25 @@ SUPPRESS_EXCEPTIONS = True
class PackageDir ( object ):
EBUILD_SUFFIX = '.ebuild'
- def __init__ ( self, name, logger, directory, get_header, incremental ):
+ def __init__ ( self,
+ name, logger, directory, get_header, runtime_incremental
+ ):
"""Initializes a PackageDir which contains ebuilds, metadata and
a Manifest file.
arguments:
- * name -- name of the directory (${PN} in ebuilds)
- * logger -- parent logger
- * directory -- filesystem location of this PackageDir
- * get_header -- function that returns an ebuild header
+ * name -- name of the directory (${PN} in ebuilds)
+ * logger -- parent logger
+ * directory -- filesystem location of this PackageDir
+ * get_header -- function that returns an ebuild header
+ * runtime_incremental -- enable/disable runtime incremental ebuild
+ writing. This trades speed (disabled) for
+ memory consumption (enabled) 'cause it will
+ write _all_ successfully created ebuilds
+ directly after they've been created.
+ Writing all ebuilds at once is generally faster
+ (+threading), but all PackageInfos must be
+ kept in memory for that.
"""
self.logger = logger.getChild ( name )
self.name = name
@@ -30,7 +40,7 @@ class PackageDir ( object ):
self._packages = dict()
self.physical_location = directory
self.get_header = get_header
- self.runtime_incremental = incremental
+ self.runtime_incremental = runtime_incremental
self._metadata = MetadataJob (
filepath = self.physical_location + os.sep + 'metadata.xml',
@@ -250,6 +260,7 @@ class PackageDir ( object ):
self._need_manifest = True
self._need_metadata = True
self.modified = True
+ # FIXME: delete PackageInfo if ebuild creation did not succeed
if self.runtime_incremental:
with self._lock:
return self.write_ebuilds ( overwrite=False )
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 219fd70..ae3267f 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -22,23 +22,49 @@ class Overlay ( object ):
def __init__ (
self,
- name, logger, directory,
- default_category, eclass_files,
+ name,
+ logger,
+ directory,
+ default_category,
+ eclass_files,
ebuild_header,
- incremental
+ write_allowed,
+ incremental,
+ runtime_incremental=True
):
- self.name = name
- self.logger = logger.getChild ( 'overlay' )
- self.physical_location = directory
- self.default_category = default_category
- self.eclass_files = eclass_files
+ """Initializes an overlay.
- self.ignore_existing_ebuilds = False
-
- self._profiles_dir = self.physical_location + os.sep + 'profiles'
- self._catlock = threading.Lock()
- self._categories = dict()
- self._header = EbuildHeader ( ebuild_header )
+ arguments:
+ * name -- name of this overlay
+ * logger -- parent logger to use
+ * directory -- filesystem location of this overlay
+ * default_category -- category of packages being added without a
+ specific category
+ * eclass_files -- eclass files to import and
+ inherit in all ebuilds
+ * ebuild_header -- the header text included in all created ebuilds
+ * write_allowed -- whether writing is allowed
+ * incremental -- enable/disable incremental writing:
+ use already existing ebuilds (don't recreate
+ them)
+ * runtime_incremental -- see package.py:PackageDir.__init__ (...),
+ Defaults to ?FIXME?
+ """
+ self.name = name
+ self.logger = logger.getChild ( 'overlay' )
+ self.physical_location = directory
+ self.default_category = default_category
+
+ self._eclass_files = eclass_files
+ #self._incremental = incremental
+ # disable runtime_incremental if writing not allowed
+ self._runtime_incremental = write_allowed and runtime_incremental
+ self._writeable = write_allowed
+
+ self._profiles_dir = self.physical_location + os.sep + 'profiles'
+ self._catlock = threading.Lock()
+ self._categories = dict()
+ self._header = EbuildHeader ( ebuild_header )
# fixme or ignore: calculating eclass names twice,
# once here and another time when calling _init_overlay
@@ -46,13 +72,12 @@ class Overlay ( object ):
self._get_eclass_import_info ( only_eclass_names=True )
) )
- self.incremental = incremental
- if self.incremental:
+ #if self._incremental:
+ if incremental:
# this is multiple-run incremental writing (in contrast to runtime
# incremental writing, which writes ebuilds as soon as they're
- # ready) FIXME: split incremental <-> runtime_incremental
+ # ready)
self.scan()
-
# --- end of __init__ (...) ---
def _get_category ( self, category ):
@@ -70,7 +95,7 @@ class Overlay ( object ):
self.logger,
self.physical_location + os.sep + category,
get_header=self._header.get,
- incremental=self.incremental
+ runtime_incremental=self._runtime_incremental
)
self._categories [category] = newcat
finally:
@@ -89,9 +114,9 @@ class Overlay ( object ):
raises: AssertionError if a file does not end with '.eclass'.
"""
- if self.eclass_files:
+ if self._eclass_files:
- for eclass in self.eclass_files:
+ for eclass in self._eclass_files:
dest = os.path.splitext ( os.path.basename ( eclass ) )
if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ):
@@ -116,7 +141,7 @@ class Overlay ( object ):
* Exception if copying fails
"""
- if self.eclass_files:
+ if self._eclass_files:
# import eclass files
eclass_dir = self.physical_location + os.sep + 'eclass'
try:
@@ -240,6 +265,10 @@ class Overlay ( object ):
yield kwargs
# --- end of list_rule_kwargs (...) ---
+ def readonly ( self ):
+ return not self._writeable
+ # --- end of readonly (...) ---
+
def scan ( self, **kw ):
def scan_categories():
for x in os.listdir ( self.physical_location ):
@@ -271,6 +300,10 @@ class Overlay ( object ):
cat.show ( **show_kw )
# --- end of show (...) ---
+ def writeable ( self ):
+ return self._writeable
+ # --- end of writeable (...) ---
+
def write ( self ):
"""Writes the overlay to its physical location (filesystem), including
metadata and Manifest files as well as cleanup actions.
@@ -285,14 +318,21 @@ class Overlay ( object ):
! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
when ebuild creation is done.
"""
- self._init_overlay ( reimport_eclass=True )
-
- for cat in self._categories.values():
- cat.write (
- overwrite_ebuilds=False,
- keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
- cautious=True
- )
+ if self._writeable:
+ self._init_overlay ( reimport_eclass=True )
+
+ for cat in self._categories.values():
+ cat.write (
+ overwrite_ebuilds=False,
+ keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
+ cautious=True
+ )
+ else:
+ # FIXME debug print
+ print (
+ "Dropped write request for readonly overlay {}!".format (
+ self.name
+ ) )
# --- end of write (...) ---
def write_manifest ( self, **manifest_kw ):
@@ -305,7 +345,14 @@ class Overlay ( object ):
returns: None (implicit)
"""
- # FIXME: it would be good to ensure that profiles/categories exist
- for cat in self._categories.values():
- cat.write_manifest ( **manifest_kw )
+ if self._writeable:
+ # FIXME: it would be good to ensure that profiles/categories exist
+ for cat in self._categories.values():
+ cat.write_manifest ( **manifest_kw )
+ else:
+ # FIXME debug print
+ print (
+ "Dropped write_manifest request for readonly overlay {}!".format (
+ self.name
+ ) )
# --- end of write_manifest (...) ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-07-30 8:52 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-07-30 8:52 UTC (permalink / raw
To: gentoo-commits
commit: c456433d70dae2e58fb0297ab1b82822b2405b2d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:58:40 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:58:40 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=c456433d
add "ebuild uncreatable" event
Immediately removes the PackageInfo object from the overlay
structure to save some memory.
geändert: roverlay/overlay/creator.py
geändert: roverlay/overlay/package.py
---
roverlay/overlay/creator.py | 1 +
roverlay/overlay/package.py | 22 ++++++++++++++++++++--
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 60e9517..5b0a973 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -382,6 +382,7 @@ class OverlayCreator ( object ):
if package_info.overlay_package_ref.new_ebuild():
self.overlay_added.inc()
else:
+ package_info.overlay_package_ref.ebuild_uncreateable ( package_info )
self.create_fail.inc()
# --- end of _add_to_overlay (...) ---
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index bedbce3..cb5cf72 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -145,6 +145,25 @@ class PackageDir ( object ):
return False
# --- end of check_empty (...) ---
+ def ebuild_uncreateable ( self, package_info ):
+ """Called when ebuild creation (finally) failed for a PackageInfo
+ object of this PackageDir.
+
+ arguments:
+ * package_info --
+ """
+ try:
+ self._lock.acquire()
+ pvr = package_info ['ebuild_verstr']
+ # FIXME debug print
+ print ( "removing {PVR} from {PN}".format ( PVR=pvr, PN=self.name ) )
+ del self._packages [pvr]
+ except KeyError:
+ pass
+ finally:
+ self._lock.release()
+ # --- end of uncreateable_ebuild (...) ---
+
def empty ( self ):
"""Returns True if no ebuilds stored, else False.
Note that "not empty" doesn't mean "has ebuilds to write" or "has
@@ -255,12 +274,11 @@ class PackageDir ( object ):
return self._packages.keys()
# --- end of list_versions (...) ---
- def new_ebuild ( self, write=False ):
+ def new_ebuild ( self ):
"""Called when a new ebuild has been created for this PackageDir."""
self._need_manifest = True
self._need_metadata = True
self.modified = True
- # FIXME: delete PackageInfo if ebuild creation did not succeed
if self.runtime_incremental:
with self._lock:
return self.write_ebuilds ( overwrite=False )
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-07-30 8:52 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-07-30 8:52 UTC (permalink / raw
To: gentoo-commits
commit: b568e1b218a2ed83dc0ae36bb4a3326b19078034
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:58:09 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:58:09 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b568e1b2
stop thread when queue is empty
---
roverlay/overlay/category.py | 25 ++++++++++++-------------
1 files changed, 12 insertions(+), 13 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index a0b1607..2d66246 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -145,19 +145,18 @@ class Category ( object ):
* q -- queue
* write_kw -- keywords for write(...)
"""
- try:
- while not q.empty():
- try:
- pkg = q.get_nowait()
- # remove manifest writing from threaded writing since it's
- # single-threaded
- pkg.write ( write_manifest=False, **write_kw )
- #except ( Exception, KeyboardInterrupt ) as e:
- except Exception as e:
- # FIXME: reintroduce RERAISE
- self.logger.exception ( e )
- except queue.Empty:
- pass
+ while not q.empty():
+ try:
+ pkg = q.get_nowait()
+ # remove manifest writing from threaded writing since it's
+ # single-threaded
+ pkg.write ( write_manifest=False, **write_kw )
+ except queue.Empty:
+ break
+ #except ( Exception, KeyboardInterrupt ) as e:
+ except Exception as e:
+ # FIXME: reintroduce RERAISE
+ self.logger.exception ( e )
# --- end of run_write_queue (...) ---
if len ( self._subdirs ) == 0: return
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-07-30 15:53 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-07-30 15:53 UTC (permalink / raw
To: gentoo-commits
commit: 2d4af7c3ebe183400bcb0b732518e03d189cf35b
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 30 15:50:05 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 30 15:53:31 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2d4af7c3
overlay: skip_manifest
---
roverlay/overlay/category.py | 7 ++++++-
roverlay/overlay/creator.py | 5 +++--
roverlay/overlay/root.py | 10 +++++++++-
3 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 2d66246..6623446 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -133,7 +133,12 @@ class Category ( object ):
package.show ( **show_kw )
# --- end of show (...) ---
- def write ( self, overwrite_ebuilds, keep_n_ebuilds, cautious ):
+ def write ( self,
+ overwrite_ebuilds,
+ keep_n_ebuilds,
+ cautious,
+ skip_manifest=False
+ ):
"""Writes this category to its filesystem location.
returns: None (implicit)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 5b0a973..5f346fe 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -72,7 +72,7 @@ class OverlayCreator ( object ):
LOGGER = logging.getLogger ( 'OverlayCreator' )
- def __init__ ( self, logger=None, allow_write=True ):
+ def __init__ ( self, skip_manifest, logger=None, allow_write=True ):
if logger is None:
self.logger = self.__class__.LOGGER
@@ -91,7 +91,8 @@ class OverlayCreator ( object ):
eclass_files = config.get ( 'OVERLAY.eclass_files', None ),
ebuild_header = config.get ( 'EBUILD.default_header', None ),
incremental = USE_INCREMENTAL_WRITE,
- write_allowed = allow_write
+ write_allowed = allow_write,
+ skip_manifest = skip_manifest
)
self.depresolver = easyresolver.setup ( self._err_queue )
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index ae3267f..53babdf 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -30,6 +30,7 @@ class Overlay ( object ):
ebuild_header,
write_allowed,
incremental,
+ skip_manifest,
runtime_incremental=True
):
"""Initializes an overlay.
@@ -47,8 +48,12 @@ class Overlay ( object ):
* incremental -- enable/disable incremental writing:
use already existing ebuilds (don't recreate
them)
+ * skip_manifest -- skip Manifest generation to save time
+ !!! The created overlay cannot be used with
+ emerge/portage
* runtime_incremental -- see package.py:PackageDir.__init__ (...),
Defaults to ?FIXME?
+
"""
self.name = name
self.logger = logger.getChild ( 'overlay' )
@@ -66,6 +71,8 @@ class Overlay ( object ):
self._categories = dict()
self._header = EbuildHeader ( ebuild_header )
+ self.skip_manifest = skip_manifest
+
# fixme or ignore: calculating eclass names twice,
# once here and another time when calling _init_overlay
self._header.set_eclasses ( frozenset (
@@ -325,7 +332,8 @@ class Overlay ( object ):
cat.write (
overwrite_ebuilds=False,
keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
- cautious=True
+ cautious=True,
+ skip_manifest=self.skip_manifest
)
else:
# FIXME debug print
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-08-01 21:10 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-08-01 21:10 UTC (permalink / raw
To: gentoo-commits
commit: 91736d8f9c07b1d64c42b6ddb0ab6b7da80ceb99
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Aug 1 12:17:02 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Aug 1 12:17:02 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=91736d8f
actually skip manifest creation with --no-manifest
---
roverlay/overlay/category.py | 13 +++++++------
roverlay/overlay/root.py | 12 ++++++------
2 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 6623446..3551b81 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -137,7 +137,7 @@ class Category ( object ):
overwrite_ebuilds,
keep_n_ebuilds,
cautious,
- skip_manifest=False
+ write_manifest
):
"""Writes this category to its filesystem location.
@@ -205,14 +205,15 @@ class Category ( object ):
# write manifest files
# fixme: debug print
- #self.logger.info ( "Writing Manifest files for {}".format ( name ) )
- print ( "Writing Manifest files ..." )
- for package in self._subdirs.values():
- package.write_manifest ( ignore_empty=True )
+ if write_manifest:
+ #self.logger.info ( "Writing Manifest files for {}".format ( name ) )
+ print ( "Writing Manifest files ..." )
+ for package in self._subdirs.values():
+ package.write_manifest ( ignore_empty=True )
else:
for package in self._subdirs.values():
- package.write ( **write_kwargs )
+ package.write ( write_manifest=write_manifest, **write_kwargs )
self.remove_empty()
# --- end of write (...) ---
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 53babdf..3efdf66 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -330,10 +330,10 @@ class Overlay ( object ):
for cat in self._categories.values():
cat.write (
- overwrite_ebuilds=False,
- keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
- cautious=True,
- skip_manifest=self.skip_manifest
+ overwrite_ebuilds = False,
+ keep_n_ebuilds = config.get ( 'OVERLAY.keep_nth_latest', None ),
+ cautious = True,
+ write_manifest = not self.skip_manifest
)
else:
# FIXME debug print
@@ -353,11 +353,11 @@ class Overlay ( object ):
returns: None (implicit)
"""
- if self._writeable:
+ if self._writeable and not self.skip_manifest:
# FIXME: it would be good to ensure that profiles/categories exist
for cat in self._categories.values():
cat.write_manifest ( **manifest_kw )
- else:
+ elif not self.skip_manifest:
# FIXME debug print
print (
"Dropped write_manifest request for readonly overlay {}!".format (
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-08-03 13:38 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-08-03 13:38 UTC (permalink / raw
To: gentoo-commits
commit: 63afbb8871376b2e08168115d12f703e4b35afc8
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 3 09:38:34 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 3 09:38:43 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=63afbb88
package.py: filter takes no keyword arguments
---
roverlay/overlay/package.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 722be8b..18b1141 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -240,8 +240,8 @@ class PackageDir ( object ):
# sort them by version in reverse order
packages = reversed ( sorted (
filter (
- function=is_ebuild if not cautious else is_ebuild_cautious,
- iterable=self._packages.items()
+ is_ebuild if not cautious else is_ebuild_cautious,
+ self._packages.items()
),
key=lambda p : p [1] ['version']
) )
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2012-08-17 17:26 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2012-08-17 17:26 UTC (permalink / raw
To: gentoo-commits
commit: 56fcb45535c4457d9dff7844c54f8a0c49a8500a
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 17 17:26:15 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 17 17:26:15 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=56fcb455
code comment for overlay root
---
roverlay/overlay/root.py | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 8c22ce2..99c0850 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -10,6 +10,9 @@ This module provides the Overlay class that acts as an interface between
PackageInfo instances (in memory) and the R overlay (directory in filesystem).
Most requests are redirected to its subdirectories (Category), which, in turn,
pass them to their subdirectories (PackageDir).
+
+Caution: Never deep-copy an overlay object. This leads to infinite recursion
+due do double-linkage between PackageInfo and PackageDir.
"""
__all__ = [ 'Overlay', ]
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-01-30 20:16 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-01-30 20:16 UTC (permalink / raw
To: gentoo-commits
commit: b1425ef200dd984ffd6387baa4eebc6bb0f6fb68
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jan 30 20:15:18 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jan 30 20:15:18 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b1425ef2
overlay/creator: use package rules
Currently, this is a no-op as package rule loading is not implemented (also
previous commit).
---
roverlay/overlay/creator.py | 15 +++++++++++----
1 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 1e1c3cb..6674e24 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -27,12 +27,15 @@ except ImportError:
from roverlay import config, errorqueue
+
from roverlay.overlay import Overlay
from roverlay.overlay.worker import OverlayWorker
from roverlay.packageinfo import PackageInfo
+from roverlay.packagerules.rules import PackageRules
from roverlay.recipe import easyresolver
+
class PseudoAtomicCounter ( object ):
def __init__ ( self, start=0, long_int=False ):
@@ -105,6 +108,8 @@ class OverlayCreator ( object ):
self.depresolver = easyresolver.setup ( self._err_queue )
self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
+ self.package_rules = PackageRules.get_configured()
+
self.NUMTHREADS = config.get ( 'EBUILD.jobcount', 0 )
self._pkg_queue = queue.Queue()
@@ -214,10 +219,12 @@ class OverlayCreator ( object ):
arguments:
* package_info --
"""
- if self.overlay.add ( package_info ):
- self._pkg_queue.put ( package_info )
- # FIXME package_added is now the # of packages queued for creation
- self.package_added.inc()
+ if self.package_rules.apply_actions ( package_info ):
+ if self.overlay.add ( package_info ):
+ self._pkg_queue.put ( package_info )
+ # FIXME package_added is now the # of packages queued for creation
+ self.package_added.inc()
+ # else filtered out
# --- end of add_package (...) ---
def write_overlay ( self ):
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-02-09 20:45 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-02-09 20:45 UTC (permalink / raw
To: gentoo-commits
commit: 0e093591866ba15abd66b692ec190602ee9d9673
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb 9 19:16:45 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb 9 20:08:14 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=0e093591
overlay/creator: timestats
added (vague) time measurement for get_stats().
---
roverlay/overlay/creator.py | 102 ++++++++++++++++++++++++++++++++----------
1 files changed, 77 insertions(+), 25 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 6674e24..6aa5af9 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -12,13 +12,12 @@ main script).
"""
__all__ = [ 'OverlayCreator', ]
+import collections
import time
import logging
import threading
import sys
-from collections import deque
-
try:
import queue
except ImportError:
@@ -137,8 +136,22 @@ class OverlayCreator ( object ):
# counts packages that passed adding to overlay
self.overlay_added = PseudoAtomicCounter()
+ self._timestats = collections.OrderedDict()
+
+ for k in (
+ 'sync_packages',
+ 'add_packages',
+ 'ebuild_creation',
+ 'overlay_write',
+ ):
+ self._timestats [k] = -1
+
# --- end of __init__ (...) ---
+ def set_timestats ( self, name, seconds ):
+ self._timestats [name] = seconds
+ # --- end of set_timestats (...) ---
+
def get_stats ( self ):
pkg_added = self.package_added.get_nowait()
pkg_created = self.create_success.get_nowait()
@@ -164,33 +177,68 @@ class OverlayCreator ( object ):
# the length of the highest number in stats (^=digit count)
# max_number_len := { 1,...,5 }
max_number_len = min ( 5, len ( str ( max ( stats ) ) ) )
- num_fmt = '%(num)-' + str ( max_number_len ) + 's '
-
- for i, s in enumerate ((
- 'packages added to the ebuild creation queue',
- 'packages passed ebuild creation',
- 'packages failed ebuild creation',
- 'ebuilds could be added to the overlay',
- 'ebuilds couldn\'t be added to the overlay',
- 'packages processed in total',
- 'packages failed in total',
- )):
- yield num_fmt % { 'num' : stats [i] } + s
+
+ for stats_tuple in zip (
+ stats,
+ (
+ 'packages added to the ebuild creation queue',
+ 'packages passed ebuild creation',
+ 'packages failed ebuild creation',
+ 'ebuilds could be added to the overlay',
+ 'ebuilds couldn\'t be added to the overlay',
+ 'packages processed in total',
+ 'packages failed in total',
+ ),
+ ):
+ yield "{num:<{l}} {s}".format (
+ num = stats_tuple [0],
+ s = stats_tuple [1],
+ l = max_number_len,
+ )
+
+ yield ""
+
+ k_len = min (
+ 39,
+ max ( len ( k ) for k in self._timestats.keys() )
+ )
+
+ for k, v in self._timestats.items():
+ if v < 0:
+ yield "time for {:<{l}} : <unknown>".format ( k, l=k_len, )
+
+ elif v < 1:
+ yield "time for {:<{l}} : {} ms".format (
+ k,
+ round ( v * 1000, 2 ),
+ l = k_len,
+ )
+
+ elif v > 300:
+ yield "time for {:<{l}} : {} minutes".format (
+ k,
+ round ( v / 60., 2 ),
+ l = k_len,
+ )
+
+ else:
+ yield "time for {}: {} seconds".format ( k, round ( v, 2 ) )
# --- end of stats_gen (...) ---
if enclose:
- stats_str = deque ( stats_gen() )
+ stats = list ( stats_gen() )
+
# maxlen := { 2,...,80 }
maxlen = 2 + min ( 78,
- len ( max ( stats_str, key=lambda s : len( s ) ) )
+ len ( max ( stats, key=lambda s : len( s ) ) )
)
- stats_str.appendleft (
- " Overlay creation stats ".center ( maxlen, '-' )
+ return (
+ "{0:-^{1}}\n".format ( " Overlay creation stats ", maxlen )
+ + '\n'.join ( stats )
+ #+ '\n{0:-^{1}}'.format ( '', maxlen )
+ + '\n' + ( maxlen * '-' )
)
- stats_str.append ( '-' * maxlen )
-
- return '\n'.join ( stats_str )
else:
return '\n'.join ( stats_gen() )
@@ -210,7 +258,7 @@ class OverlayCreator ( object ):
self.logger.debug (
"timestamp: {} (after {} seconds)".format ( description, delta )
)
- return _stop
+ return delta
# --- end of _timestamp (...) ---
def add_package ( self, package_info ):
@@ -232,7 +280,9 @@ class OverlayCreator ( object ):
if self.overlay.writeable():
start = time.time()
self.overlay.write()
- self._timestamp ( "overlay written", start )
+ self._timestats ['overlay_write'] = (
+ self._timestamp ( "overlay written", start )
+ )
else:
self.logger.warning ( "Not allowed to write overlay!" )
# --- end of write_overlay (...) ---
@@ -245,12 +295,14 @@ class OverlayCreator ( object ):
def run ( self, close_when_done=False ):
"""Starts ebuild creation and waits until done."""
self._runlock.acquire()
- #self._time_start_run.append ( time.time() )
+ t_start = time.time()
try:
self.start()
self.join()
finally:
- #self._time_stop_run.append ( time.time() )
+ self._timestats ['ebuild_creation'] = (
+ self._timestamp ( "run() done", t_start )
+ )
self._runlock.release()
if close_when_done:
self.close()
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-02-09 20:45 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-02-09 20:45 UTC (permalink / raw
To: gentoo-commits
commit: d21225c2eba72f010b9ca824db192f450895bc03
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb 9 19:34:29 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb 9 20:08:15 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d21225c2
overlay/category: RERAISE with correct traceback
self.RERAISE contains the exception info (as returned by sys.exc_info()) now
---
roverlay/overlay/category.py | 11 +++++------
1 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 3a58367..4a03cce 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -16,6 +16,7 @@ __all__ = [ 'Category', ]
import threading
import os
+import sys
try:
import queue
@@ -206,9 +207,8 @@ class Category ( object ):
pkg.write ( **write_kw )
except queue.Empty:
break
- except Exception as e:
- #self.logger.exception ( e )
- self.RERAISE = e
+ except:
+ self.RERAISE = sys.exc_info()
# --- end of run_write_queue (...) ---
if len ( self._subdirs ) == 0: return
@@ -257,15 +257,14 @@ class Category ( object ):
for w in workers: w.start()
for w in workers: w.join()
- if hasattr ( self, 'RERAISE' ) and self.RERAISE is not None:
- raise self.RERAISE
+ if hasattr ( self, 'RERAISE' ) and self.RERAISE:
+ raise self.RERAISE [0], self.RERAISE [1], self.RERAISE [2]
self.remove_empty()
# write manifest files
# fixme: debug print
if write_manifest and ( not manifest_threadsafe ):
- #self.logger.info ( "Writing Manifest files for {}".format ( name ) )
print ( "Writing Manifest files ..." )
for package in self._subdirs.values():
package.write_manifest ( ignore_empty=True )
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-02-09 21:28 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-02-09 21:28 UTC (permalink / raw
To: gentoo-commits
commit: d6bbb89d92aca4dc6cd351d1b249ebbbc2876b69
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb 9 21:27:12 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb 9 21:27:12 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d6bbb89d
overlay/category: fix RERAISE
this fixes a python 2 vs 3 issue
---
roverlay/overlay/category.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 4a03cce..97f87e7 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -258,7 +258,7 @@ class Category ( object ):
for w in workers: w.join()
if hasattr ( self, 'RERAISE' ) and self.RERAISE:
- raise self.RERAISE [0], self.RERAISE [1], self.RERAISE [2]
+ raise ( self.RERAISE [0], self.RERAISE [1], self.RERAISE [2] )
self.remove_empty()
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-04-25 16:44 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-04-25 16:44 UTC (permalink / raw
To: gentoo-commits
commit: 2e601a6f8b3171fde698c7f5284dbc94b35705bc
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Apr 25 15:09:44 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Apr 25 15:09:44 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2e601a6f
overlay/category: stop queue workers
Stop write queue workers when an exception is caught.
Also changed the RERAISE code (again).
---
roverlay/overlay/category.py | 19 ++++++++++++++++---
1 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 97f87e7..dd55c14 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -199,7 +199,7 @@ class Category ( object ):
* q -- queue
* write_kw -- keywords for write(...)
"""
- while not q.empty():
+ while not q.empty() and not hasattr ( self, 'RERAISE' ):
try:
pkg = q.get_nowait()
# remove manifest writing from threaded writing since it's
@@ -207,7 +207,8 @@ class Category ( object ):
pkg.write ( **write_kw )
except queue.Empty:
break
- except:
+ except ( Exception, KeyboardInterrupt ) as err:
+ self.logger.exception ( err )
self.RERAISE = sys.exc_info()
# --- end of run_write_queue (...) ---
@@ -258,7 +259,19 @@ class Category ( object ):
for w in workers: w.join()
if hasattr ( self, 'RERAISE' ) and self.RERAISE:
- raise ( self.RERAISE [0], self.RERAISE [1], self.RERAISE [2] )
+ # ref: PEP 3109
+ # results in correct traceback when running python 3.x
+ # and inaccurate traceback with python 2.x,
+ # which can be tolerated since the exception has been logged
+ try:
+ reraise = self.RERAISE[0] ( self.RERAISE[1] )
+ except TypeError:
+ # "portage.exception.FileNotFound is not subscriptable"
+ reraise = self.RERAISE[1]
+
+ reraise.__traceback__ = self.RERAISE [2]
+ raise reraise
+ # --- end RERAISE;
self.remove_empty()
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-04-25 16:44 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-04-25 16:44 UTC (permalink / raw
To: gentoo-commits
commit: f53903b04fc9f3ae739ea88bfbda616fb5de32e3
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Apr 25 15:07:08 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Apr 25 15:07:08 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f53903b0
overlay/creator: add scan_overlay time stats
---
roverlay/overlay/creator.py | 5 +++++
1 files changed, 5 insertions(+), 0 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 6aa5af9..931a2ff 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -95,6 +95,7 @@ class OverlayCreator ( object ):
# this queue is used to propagate exceptions from threads
self._err_queue = errorqueue.ErrorQueue()
+ time_scan_overlay = time.time()
# init overlay using config values
self.overlay = Overlay.new_configured (
logger = self.logger,
@@ -103,6 +104,7 @@ class OverlayCreator ( object ):
skip_manifest = skip_manifest,
runtime_incremental = immediate_ebuild_writes,
)
+ time_scan_overlay = time.time() - time_scan_overlay
self.depresolver = easyresolver.setup ( self._err_queue )
self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
@@ -138,6 +140,9 @@ class OverlayCreator ( object ):
self._timestats = collections.OrderedDict()
+ if incremental and time_scan_overlay >= 0.1:
+ self._timestats ['scan_overlay'] = time_scan_overlay
+
for k in (
'sync_packages',
'add_packages',
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-06-13 16:34 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-06-13 16:34 UTC (permalink / raw
To: gentoo-commits
commit: fc3eeb0f06119515f8db68311183172923423ded
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun 5 18:02:09 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun 5 18:02:09 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=fc3eeb0f
overlay: fix dynpool rules, remove moved ebuilds
---
roverlay/overlay/category.py | 44 +++++++++++++++++++-----
roverlay/overlay/creator.py | 6 ++++
roverlay/overlay/root.py | 82 ++++++++++++++++++++++++++++++++++++++++++--
3 files changed, 120 insertions(+), 12 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 2d9b93d..baca2d1 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -90,6 +90,12 @@ class Category ( object ):
)
# --- end of add (...) ---
+ def drop_package ( self, name ):
+ p = self._subdirs [name]
+ del self._subdirs [name]
+ p.fs_destroy()
+ # --- end of drop_package (...) ---
+
def empty ( self ):
"""Returns True if this category contains 0 ebuilds."""
return (
@@ -98,6 +104,11 @@ class Category ( object ):
)
# --- end of empty (...) ---
+ def get_nonempty ( self, name ):
+ subdir = self._subdirs.get ( name, None )
+ return subdir if ( subdir and not subdir.empty() ) else None
+ # --- end of has_nonempty (...) ---
+
def has ( self, subdir ):
return subdir in self._subdirs
# --- end of has (...) ---
@@ -106,22 +117,37 @@ class Category ( object ):
return os.path.isdir ( self.physical_location + os.sep + _dir )
# --- end of has_category (...) ---
- def list_packages ( self, for_deprules=False ):
+ def list_package_names ( self ):
+ for name, subdir in self._subdirs.items():
+ if not subdir.empty():
+ yield name
+ # --- end of list_package_names (...) ---
+
+ def list_packages ( self,
+ for_deprules=False, is_default_category=False
+ ):
"""Lists all packages in this category.
Yields <category>/<package name> or a dict (see for_deprules below).
arguments:
- * for_deprules -- if set and True:
- yield keyword args for dependency rules
+ * for_deprules -- if set and True:
+ yield keyword args for dependency rules
+ * is_default_category -- bool indicating whether this category is the
+ default one or not
"""
-
- for name, subdir in self._subdirs.items():
- if not subdir.empty():
- if for_deprules:
+ if for_deprules:
+ for name, subdir in self._subdirs.items():
+ if not subdir.empty():
yield dict (
- dep_str=name, resolving_package=self.name + os.sep + name
+ dep_str = name,
+ resolving_package = ( self.name + "/" + name ),
+ is_selfdep = is_default_category,
+ # prefer packages from the default category (really?)
+ priority = 80 if is_default_category else 90,
)
- else:
+ else:
+ for name, subdir in self._subdirs.items():
+ if not subdir.empty():
yield self.name + os.sep + name
# --- end of list_packages (...) ---
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 8833aef..5ad86d4 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -274,6 +274,12 @@ class OverlayCreator ( object ):
del self.package_rules
# --- end of release_package_rules (...) ---
+ def remove_moved_ebuilds ( self, reverse ):
+ """See overlay.root.Overlay.remove_moved_ebuilds()."""
+ if self.overlay.remove_duplicate_ebuilds ( reverse=reverse ):
+ self.depresolver.reload_pools()
+ # --- end of remove_moved_ebuilds (...) ---
+
def add_package ( self, package_info ):
"""Adds a PackageInfo to the package queue.
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 3adbb6e..b9030d2 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -259,7 +259,9 @@ class Overlay ( object ):
write_profiles_file ( 'repo_name', self.name + '\n' )
# profiles/categories
- cats = '\n'.join ( self._categories.keys() )
+ cats = '\n'.join (
+ k for k, v in self._categories.items() if not v.empty()
+ )
if cats:
write_profiles_file ( 'categories', cats + '\n' )
@@ -319,6 +321,78 @@ class Overlay ( object ):
return os.path.isdir ( self.physical_location + os.sep + _dir )
# --- end of has_category (...) ---
+ def find_duplicate_packages ( self, _default_category=None ):
+ default_category = (
+ _default_category if _default_category is None
+ else self._categories.get ( self.default_category, None )
+ )
+
+ if default_category:
+ duplicate_pkg = set()
+
+ for category in self._categories.values():
+ if category is not default_category:
+ for name in category.list_package_names():
+ if default_category.get_nonempty ( name ):
+ duplicate_pkg.add ( name )
+ # -- end for;
+
+ return frozenset ( duplicate_pkg )
+ else:
+ return None
+ # --- end of find_duplicate_packages (...) ---
+
+ def remove_duplicate_ebuilds ( self, reverse ):
+ default_category = self._categories.get ( self.default_category, None )
+ if default_category:
+ if reverse:
+ d_pkg = self.find_duplicate_packages (
+ _default_category=default_category
+ )
+ for pkg_name in d_pkg:
+ default_category.drop_package ( pkg_name )
+
+ else:
+ d_pkg = set()
+ for category in self._categories.values():
+ if category is not default_category:
+ for name in category.list_package_names():
+ if default_category.get_nonempty ( name ):
+ d_pkg.add ( name )
+ category.drop_package ( pkg_name )
+ # -- end for category;
+
+ # -- end if;
+
+ if d_pkg:
+ self.remove_empty_categories()
+
+ self.logger.info (
+ '{} ebuilds have been removed from the default category, '
+ 'the overlay might be broken now!'.format ( len ( d_pkg ) )
+ )
+ return True
+ else:
+ return False
+ else:
+ return False
+
+ # --- end of remove_duplicate_ebuilds (...) ---
+
+ def remove_empty_categories ( self ):
+ catlist = self._categories.items()
+ for cat in catlist:
+ cat[1].remove_empty()
+ if cat[1].empty():
+ del self._categories [cat[0]]
+
+ try:
+ os.rmdir ( cat [1].physical_location )
+ except OSError as ose:
+ self.logger.exception ( ose )
+
+ # --- end of remove_empty_categories (...) ---
+
def list_packages ( self, for_deprules=True ):
for cat in self._categories.values():
for package in cat.list_packages ( for_deprules=for_deprules ):
@@ -326,9 +400,11 @@ class Overlay ( object ):
# --- end of list_packages (...) ---
def list_rule_kwargs ( self ):
- # FIXME/TODO: depres has to recognize categories
for cat in self._categories.values():
- for kwargs in cat.list_packages ( for_deprules=True ):
+ for kwargs in cat.list_packages (
+ for_deprules = True,
+ is_default_category = ( cat.name is self.default_category )
+ ):
yield kwargs
# --- end of list_rule_kwargs (...) ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-06-18 14:12 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-06-18 14:12 UTC (permalink / raw
To: gentoo-commits
commit: dea12476091298811456b8c8f6ca596ae195e7ee
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Jun 15 08:58:16 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Jun 15 08:58:16 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=dea12476
overlay: write/update USE_EXPAND description files
---
| 13 ++++
roverlay/overlay/root.py | 168 ++++++++++++++++++++++++++++++++++++++++++---
2 files changed, 171 insertions(+), 10 deletions(-)
--git a/roverlay/overlay/header.py b/roverlay/overlay/header.py
index 6cd1f1f..8c7c05a 100644
--- a/roverlay/overlay/header.py
+++ b/roverlay/overlay/header.py
@@ -29,6 +29,19 @@ class EbuildHeader ( object ):
return self._cached_header
# --- end of get (...) ---
+ def get_use_expand_header ( self, use_expand_name ):
+ if self.default_header:
+ return (
+ self.default_header + '\n'
+ + (
+ '# This file contains descriptions of the {} '
+ 'USE_EXPAND flags.'.format ( use_expand_name )
+ ) + '\n'
+ )
+ else:
+ return None
+ # --- end of get_use_expand_header (...) ---
+
def _make ( self ):
if self.eclasses:
inherit = 'inherit ' + ' '.join ( sorted ( self.eclasses ) )
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 98d0b26..71156df 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -17,6 +17,7 @@ due do double-linkage between PackageInfo and PackageDir.
__all__ = [ 'Overlay', ]
+import errno
import logging
import os
import shutil
@@ -313,13 +314,18 @@ class Overlay ( object ):
# profiles/desc/<r_suggests>.desc
# !!! (late) config access (FIXME)
+
+ use_expand_name = roverlay.config.get_or_fail (
+ "EBUILD.USE_EXPAND.name"
+ ).rstrip ( "_" )
+
self._write_rsuggests_use_desc (
(
self._profiles_dir + os.sep + 'desc' + os.sep
- + roverlay.config.get_or_fail (
- "EBUILD.USE_EXPAND.name"
- ).rstrip ( "_" ).lower()+ '.desc'
- )
+ + use_expand_name.lower() + '.desc'
+ ),
+ use_expand_name.upper(),
+ roverlay.config.get ( 'OVERLAY.backup_desc', True )
)
# profiles/use.desc
@@ -342,12 +348,154 @@ class Overlay ( object ):
raise
# --- end of _init_overlay (...) ---
- def _write_rsuggests_use_desc ( self, desc_file ):
- # TODO
- #open ( desc_file, "r" ), errno == errno.ENOENT (2)
- self.logger.error (
- "Cannot write {!r}: code is missing ;)".format ( desc_file )
- )
+ def _write_rsuggests_use_desc (
+ self, desc_file, use_expand_name, backup_file, rewrite=False
+ ):
+ """Creates a USE_EXPAND description file.
+
+ Reads the old file (if it exists) and imports its header / flag desc.
+
+ arguments:
+ * desc_file -- path to the file that should be written/read
+ * use_expand_name -- name of the USE_EXPAND variable, e.g. R_SUGGESTS
+ * backup_file -- move desc_file to backup_file before overwriting it
+ This can also be an int i (=> desc_file + '.<i>')
+ or a bool (if True => desc_file + '.bak').
+ * rewrite -- force recreation of the desc file
+ """
+ FLAG_SEPA = ' - '
+ DESC_UNDEF = 'unknown'
+
+ def do_backup ( dest, makedir=False ):
+ self.logger.debug ( "Moving old desc file to {!r}".format ( dest ) )
+ if makedir:
+ roverlay.util.dodir ( os.path.dirname ( dest ), mkdir_p=True )
+ shutil.move ( desc_file, dest )
+ # --- end of do_backup (...) ---
+
+ def read_desc_file():
+ """Reads the old desc file (if it exists).
+ Returns a 3-tuple ( list header, dict flags, bool file_existed ).
+
+ arguments:
+ * @implicit desc_file --
+
+ Passes all exceptions (IOError, ...) but "file does not exist".
+ """
+
+ FH = None
+ header = list()
+ # flags := dict { <flag name> => <desc>|None }
+ flags = dict()
+ fexist = False
+ try:
+ FH = open ( desc_file, 'rt' )
+
+ addto_header = True
+
+ for line in FH.readlines():
+ rsline = line.rstrip()
+ if rsline and rsline [0] != '#':
+ flag, sepa, desc = rsline.partition ( FLAG_SEPA )
+ # <desc> == DESC_UNDEF -- ignore
+ flags [flag] = desc if sepa else None
+ addto_header = False
+ elif addto_header is True:
+ header.append ( rsline )
+ elif rsline:
+ self.logger.warning (
+ "dropping line from {f!r}: {l}".format (
+ f=desc_file, l=rsline
+ )
+ )
+ # -- end for
+
+ FH.close()
+ fexist = True
+ except IOError as ioerr:
+ if ioerr.errno == errno.ENOENT:
+ pass
+ else:
+ raise
+ finally:
+ if FH: FH.close()
+
+ return ( header, flags, fexist )
+ # --- end of read_desc_file ---
+
+ def gen_desc ( header, flags ):
+ """Creates new text lines for the desc file.
+
+ arguments:
+ * @implicit use_expand_name --
+ * header -- header line(s) to use (a default header will be created
+ if this is empty)
+ * flags -- flag=>desc dict
+ """
+ NEWLINE = '\n'
+
+ if header:
+ for line in header:
+ yield line
+ yield NEWLINE
+ #yield NEWLINE
+ else:
+ defheader = self._header.get_use_expand_header ( use_expand_name )
+ if defheader:
+ yield defheader
+ yield NEWLINE
+
+ for flag, desc in sorted ( flags.items(), key=lambda e: e[0] ):
+ if desc:
+ yield flag + FLAG_SEPA + desc
+ else:
+ yield flag + FLAG_SEPA + DESC_UNDEF
+ yield NEWLINE
+ # --- end of gen_desc (...) ---
+
+ header, old_flags, can_backup = read_desc_file()
+
+ if self._incremental:
+ # incremental: add new flags
+ # Create dict flag=>None that contains all new flags
+ # and copy old_flags "over" it.
+ #
+ flags = dict.fromkeys ( self._rsuggests_flags )
+ flags.update ( old_flags )
+ else:
+ # not incremental: discard old flags
+ # Create a dict that contains the new flags only and use desc
+ # from old_flags if available.
+ #
+ flags = {
+ flag: old_flags.get ( flag, None )
+ for flag in self._rsuggests_flags
+ }
+ # -- end if
+
+ # don't rewrite the desc file if nothing has changed
+ if rewrite or (
+ frozenset ( flags.keys() ) != frozenset ( old_flags.keys() )
+ ):
+ if can_backup:
+ if backup_file is True:
+ do_backup ( desc_file + '.bak' )
+ elif isinstance ( backup_file, int ):
+ do_backuo ( desc_file + '.' + str ( backup_file ) )
+ elif backup_file:
+ do_backup ( str ( backup_file ), makedir=True )
+ # -- end if
+
+ self.logger.debug ( "writing desc file {!r}".format ( desc_file ) )
+
+ roverlay.util.dodir ( os.path.dirname ( desc_file ) )
+ with open ( desc_file, 'wt' ) as FH:
+ for line in gen_desc ( header, flags ):
+ FH.write ( line )
+ else:
+ self.logger.debug (
+ "not writing desc file {!r} (nothing changed)".format ( desc_file )
+ )
# --- end of _write_rsuggests_use_desc (...) ---
def add ( self, package_info ):
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-06-22 15:24 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-06-22 15:24 UTC (permalink / raw
To: gentoo-commits
commit: b3c06fe525a85dbd2820c14a47cd7628739fa78b
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Jun 22 10:19:38 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Jun 22 10:19:38 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b3c06fe5
overlay, scan(): don't filter all exceptions
---
roverlay/overlay/root.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index a3b02f1..35ab5ad 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -682,6 +682,8 @@ class Overlay ( object ):
for cat in scan_categories():
try:
cat.scan ( **kw )
+ except ( RuntimeError, SystemError, KeyboardInterrupt, ):
+ raise
except Exception as e:
self.logger.exception ( e )
# --- end of scan (...) ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
2013-07-10 15:10 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
@ 2013-07-10 16:16 ` André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-07-10 16:16 UTC (permalink / raw
To: gentoo-commits
commit: 35368b2daf90b7be2a76f2af4608961bcbc1eb46
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 14:02:22 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 14:02:22 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=35368b2d
roverlay/overlay/root: remove/replace debug code
---
roverlay/overlay/root.py | 31 +++++++++++++------------------
1 file changed, 13 insertions(+), 18 deletions(-)
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 0c291da..099d6d5 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -1,6 +1,6 @@
# R overlay -- overlay package, overlay root
# -*- coding: utf-8 -*-
-# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
@@ -703,10 +703,9 @@ class Overlay ( object ):
## drop p
##
- #FIXME: PKG_REMOVED*: debug code
- print ( "REMOVE_BROKEN_PACKAGES: 'balance' ... " )
- PKG_REMOVED = list()
- PKG_REMOVED_ADD = PKG_REMOVED.append
+
+ #PKG_REMOVED = list()
+ #PKG_REMOVED_ADD = PKG_REMOVED.append
num_pkg_removed = 0
@@ -714,21 +713,17 @@ class Overlay ( object ):
for pkgdir in cat._subdirs.values():
for pvr, p_info in pkgdir._packages.items():
if not p_info.is_valid():
- # FIXME: debug print
- PKG_REMOVED_ADD ( "{}-{}".format ( pkgdir.name, pvr ) )
+ #PKG_REMOVED_ADD ( "{}-{}".format ( pkgdir.name, pvr ) )
pkgdir.purge_package ( pvr )
num_pkg_removed += 1
# -- end for cat;
- print ( "REMOVE_BROKEN_PACKAGES: 'finalize' ... " )
-
- #FIXME: PKG_REMOVED*
- if PKG_REMOVED:
- with open ( "/tmp/roverlay_selfdep_redux.dbg", 'wt' ) as DEBUG_FH:
- for line in PKG_REMOVED:
- DEBUG_FH.write ( line )
- DEBUG_FH.write ( '\n' )
+# if PKG_REMOVED:
+# with open ( "/tmp/roverlay_selfdep_redux.dbg", 'wt' ) as DEBUG_FH:
+# for line in PKG_REMOVED:
+# DEBUG_FH.write ( line )
+# DEBUG_FH.write ( '\n' )
# FIXME: debug prints (use logging, ...)
@@ -736,13 +731,13 @@ class Overlay ( object ):
# remove_empty_categories() could be done in the loop above
self.remove_empty_categories()
- print (
- 'REMOVE_BROKEN_PACKAGES: {:d} ebuilds have been removed'.format (
+ self.logger.info (
+ 'remove_broken_packages: {:d} ebuilds have been dropped.'.format (
num_pkg_removed
)
)
else:
- print ( "REMOVE_BROKEN_PACKAGES: nothing done ;)" )
+ self.logger.info ( 'remove_broken_packages: no ebuilds removed.' )
return num_pkg_removed
# --- end of remove_broken_packages (...) ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-07-29 14:56 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-07-29 14:56 UTC (permalink / raw
To: gentoo-commits
commit: a1290081b70d34ac70315970cffe5f70c3b31bc5
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 29 09:25:20 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 29 09:25:20 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a1290081
remove old stats code in overlay/creator.py
---
roverlay/overlay/creator.py | 94 ---------------------------------------------
1 file changed, 94 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 4bbedbf..0666335 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -94,100 +94,6 @@ class OverlayCreator ( object ):
# --- end of __init__ (...) ---
- def get_stats ( self ):
- raise NotImplementedError ( "about to be removed!" )
- pkg_added = self.package_added.get_nowait()
- pkg_created = self.create_success.get_nowait()
- pkg_failed = self.create_fail.get_nowait()
- ov_added = self.overlay_added.get_nowait()
- ov_failed = pkg_created - ov_added
- processed = pkg_created + pkg_failed
- failed = pkg_failed + ov_failed
-
- return (
- pkg_added, pkg_created, pkg_failed,
- ov_added, ov_failed,
- processed, failed
- )
- # --- end of get_stats (...) ---
-
- def stats_str ( self, enclose=True ):
- """Returns a string with some overlay creation stats."""
- raise NotImplementedError ( "about to be removed!" )
- def stats_gen():
- """Yields stats strings."""
- stats = self.get_stats()
-
- # the length of the highest number in stats (^=digit count)
- # max_number_len := { 1,...,5 }
- max_number_len = min ( 5, len ( str ( max ( stats ) ) ) )
-
- for stats_tuple in zip (
- stats,
- (
- 'packages added to the ebuild creation queue',
- 'packages passed ebuild creation',
- 'packages failed ebuild creation',
- 'ebuilds could be added to the overlay',
- 'ebuilds couldn\'t be added to the overlay',
- 'packages processed in total',
- 'packages failed in total',
- ),
- ):
- yield "{num:<{l}} {s}".format (
- num = stats_tuple [0],
- s = stats_tuple [1],
- l = max_number_len,
- )
-
- yield ""
-
- k_len = min (
- 39,
- max ( len ( k ) for k in self._timestats.keys() )
- )
-
- for k, v in self._timestats.items():
- if v < 0:
- yield "time for {:<{l}} : <unknown>".format ( k, l=k_len, )
-
- elif v < 1:
- yield "time for {:<{l}} : {} ms".format (
- k,
- round ( v * 1000, 2 ),
- l = k_len,
- )
-
- elif v > 300:
- yield "time for {:<{l}} : {} minutes".format (
- k,
- round ( v / 60., 2 ),
- l = k_len,
- )
-
- else:
- yield "time for {}: {} seconds".format ( k, round ( v, 2 ) )
- # --- end of stats_gen (...) ---
-
- if enclose:
- stats = list ( stats_gen() )
-
- # maxlen := { 2,...,80 }
- maxlen = 2 + min ( 78,
- len ( max ( stats, key=lambda s : len( s ) ) )
- )
-
- return (
- "{0:-^{1}}\n".format ( " Overlay creation stats ", maxlen )
- + '\n'.join ( stats )
- #+ '\n{0:-^{1}}'.format ( '', maxlen )
- + '\n' + ( maxlen * '-' )
- )
-
- else:
- return '\n'.join ( stats_gen() )
- # --- end of stats_str (...) ---
-
def release_package_rules ( self ):
"""Removes all package rules from this object.
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-08-20 21:46 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-08-20 21:46 UTC (permalink / raw
To: gentoo-commits
commit: abb8299807a7df2474ebcf52c7ad0187da94d810
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Aug 20 16:33:39 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Aug 20 16:33:39 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=abb82998
overlay creator: optionally use non-greedy channels
---
roverlay/overlay/creator.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 96c5531..fb57d39 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -50,7 +50,7 @@ class OverlayCreator ( object ):
def __init__ ( self,
skip_manifest, incremental, immediate_ebuild_writes,
- logger=None, allow_write=True
+ logger=None, allow_write=True, greedy_depres=True,
):
if logger is None:
self.logger = self.__class__.LOGGER
@@ -80,6 +80,13 @@ class OverlayCreator ( object ):
self.depresolver = roverlay.recipe.easyresolver.setup ( self._err_queue )
self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
+ if greedy_depres:
+ self._depres_channel_cls = roverlay.depres.channels.EbuildJobChannel
+ else:
+ self._depres_channel_cls = (
+ roverlay.depres.channels.NonGreedyDepresChannel
+ )
+
self.package_rules = PackageRules.get_configured()
self.NUMTHREADS = config.get ( 'EBUILD.jobcount', 0 )
@@ -118,7 +125,7 @@ class OverlayCreator ( object ):
* **channel_kw -- keywords for EbuildJobChannel.__init__
"""
return self.depresolver.register_channel (
- roverlay.depres.channels.EbuildJobChannel ( **channel_kw )
+ self._depres_channel_cls ( **channel_kw )
)
# --- end of _get_resolver_channel (...) ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-08-20 21:46 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-08-20 21:46 UTC (permalink / raw
To: gentoo-commits
commit: 6174d0fd60ff404e54d51ddf0286d1a13c59aa5d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Aug 20 16:33:13 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Aug 20 16:33:13 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6174d0fd
roverlay/overlay/worker: remove unused imports
---
roverlay/overlay/worker.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/roverlay/overlay/worker.py b/roverlay/overlay/worker.py
index 6ec5da6..28b3ab7 100644
--- a/roverlay/overlay/worker.py
+++ b/roverlay/overlay/worker.py
@@ -15,9 +15,6 @@ __all__ = [ 'OverlayWorker', ]
import sys
import threading
-from roverlay.depres.channels import EbuildJobChannel
-from roverlay.ebuild.creation import EbuildCreation
-
# this controls whether debug message from OverlayWorker.run() are printed
# to stderr or suppressed
DEBUG = True
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-08-29 12:36 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-08-29 12:36 UTC (permalink / raw
To: gentoo-commits
commit: 5e738c15639dddb86446e5c749d2c986f4350580
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug 29 12:28:14 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug 29 12:28:14 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=5e738c15
common base object for Overlay,Category,...
---
roverlay/overlay/base.py | 34 ++++++++++++++++++++++++++++++++++
roverlay/overlay/category.py | 10 +++++-----
roverlay/overlay/root.py | 12 ++++++------
3 files changed, 45 insertions(+), 11 deletions(-)
diff --git a/roverlay/overlay/base.py b/roverlay/overlay/base.py
new file mode 100644
index 0000000..b380e8f
--- /dev/null
+++ b/roverlay/overlay/base.py
@@ -0,0 +1,34 @@
+# R overlay -- overlay package, overlay base object
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import logging
+
+import roverlay.util.objects
+
+class OverlayObject ( roverlay.util.objects.ReferenceTree ):
+
+ # always keep a (weak) reference to self:
+ # (a) Overlay: multiple Category objects use this ref
+ # (b) Category: ^ PackageDir ...
+ # (c) PackageDir: ^ PackageInfo ...
+ #
+ CACHE_REF = True
+
+ def __init__ ( self, name, logger, directory, parent ):
+ super ( OverlayObject, self ).__init__ ( parent )
+ self.name = name
+ self.logger = (
+ logger.getChild ( name ) if logger else logging.getLogger ( name )
+ )
+ self.physical_location = directory
+ # --- end of __init__ (...) ---
+
+ # inherited:
+ #def get_parent
+ #def get_upper
+ #def get_ref
+
+# --- end of OverlayObject ---
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 2fc3a4f..d2b2028 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -27,6 +27,7 @@ except ImportError:
import roverlay.stats.collector
import roverlay.overlay.pkgdir.base
+import roverlay.overlay.base
##import roverlay.overlay.pkgdir.packagedir_ebuildmanifest
##import roverlay.overlay.pkgdir.packagedir_newmanifest
@@ -77,14 +78,14 @@ class WriteQueueJob ( object ):
# --- end of WriteQueueJob ---
-class Category ( object ):
+class Category ( roverlay.overlay.base.OverlayObject ):
WRITE_JOBCOUNT = 3
STATS = roverlay.stats.collector.static.overlay
def __init__ ( self,
- name, logger, directory, get_header, runtime_incremental
+ name, logger, directory, get_header, runtime_incremental, parent
):
"""Initializes a overlay/portage category (such as 'app-text', 'sci-R').
@@ -95,12 +96,11 @@ class Category ( object ):
* get_header -- function that returns an ebuild header
* runtime_incremental -- enable/disable runtime incremental writing
for this category (and all created PackageDirs)
+ * parent -- overlay object that created/creates this object
"""
- self.logger = logger.getChild ( name )
- self.name = name
+ super ( Category, self ).__init__ ( name, logger, directory, parent )
self._lock = threading.RLock()
self._subdirs = dict()
- self.physical_location = directory
self.get_header = get_header
self.runtime_incremental = runtime_incremental
self.packagedir_cls = roverlay.overlay.pkgdir.base.get_class()
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 6e3b1d7..1a07091 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -27,15 +27,15 @@ import threading
import roverlay.config
import roverlay.util
import roverlay.overlay.additionsdir
+import roverlay.overlay.base
import roverlay.overlay.category
import roverlay.overlay.header
import roverlay.overlay.pkgdir.base
import roverlay.overlay.pkgdir.distroot.static
-class Overlay ( object ):
+class Overlay ( roverlay.overlay.base.OverlayObject ):
DEFAULT_USE_DESC = (
'byte-compile - enable byte compiling\n'
- 'R_suggests - install recommended packages'
)
@classmethod
@@ -124,9 +124,8 @@ class Overlay ( object ):
* keep_n_ebuilds -- number of ebuilds to keep (per package),
any "false" Value (None, 0, ...) disables this
"""
- self.name = name
- self.logger = logger.getChild ( 'overlay' )
- self.physical_location = directory
+ super ( Overlay, self ).__init__ ( name, logger, directory, None )
+
self.default_category = default_category
self._eclass_files = eclass_files
@@ -203,7 +202,8 @@ class Overlay ( object ):
self.logger,
self.physical_location + os.sep + category,
get_header=self._header.get,
- runtime_incremental=self._runtime_incremental
+ runtime_incremental=self._runtime_incremental,
+ parent=self,
)
self._categories [category] = newcat
finally:
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-09-02 16:21 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-09-02 16:21 UTC (permalink / raw
To: gentoo-commits
commit: 591df99455bd4f4f907e7d96bdca4aedef5e2258
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Sep 2 16:18:29 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Sep 2 16:18:29 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=591df994
roverlay/overlay: use new depres rule generator
---
roverlay/overlay/category.py | 31 +++++++------------------------
roverlay/overlay/creator.py | 4 +++-
roverlay/overlay/root.py | 13 +++++--------
3 files changed, 15 insertions(+), 33 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index d2b2028..4301734 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -198,15 +198,7 @@ class Category ( roverlay.overlay.base.OverlayObject ):
yield p_info
# --- end of iter_package_info (...) ---
- def list_package_names ( self ):
- for name, subdir in self._subdirs.items():
- if not subdir.empty():
- yield name
- # --- end of list_package_names (...) ---
-
- def list_packages ( self,
- for_deprules=False, is_default_category=False
- ):
+ def list_packages ( self, name_only=False ):
"""Lists all packages in this category.
Yields <category>/<package name> or a dict (see for_deprules below).
@@ -216,29 +208,20 @@ class Category ( roverlay.overlay.base.OverlayObject ):
* is_default_category -- bool indicating whether this category is the
default one or not
"""
- if for_deprules:
+ if name_only:
for name, subdir in self._subdirs.items():
if not subdir.empty():
- yield dict (
- dep_str = name,
- resolving_package = ( self.name + "/" + name ),
- is_selfdep = 2 if is_default_category else 1,
- # prefer packages from the default category (really?)
- priority = 80 if is_default_category else 90,
- finalize = True,
- selfdep_package_names = filter (
- None, (
- p.get ( 'package_name', do_fallback=True )
- for p in subdir.iter_package_info()
- )
- ),
- )
+ yield name
else:
for name, subdir in self._subdirs.items():
if not subdir.empty():
yield self.name + os.sep + name
# --- end of list_packages (...) ---
+ def list_package_names ( self ):
+ return self.list_packages ( name_only=True )
+ # --- end of list_package_names (...) ---
+
def supports_threadsafe_manifest_writing ( self, unsafe=True ):
"""Returns True if manifest writing is thread safe for this
category, else False. Also returns True for empty categories.
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index fb57d39..f3157f3 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -78,7 +78,9 @@ class OverlayCreator ( object ):
)
self.depresolver = roverlay.recipe.easyresolver.setup ( self._err_queue )
- self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
+ self.depresolver.make_selfdep_pool (
+ self.overlay.get_depres_rule_generator()
+ )
if greedy_depres:
self._depres_channel_cls = roverlay.depres.channels.EbuildJobChannel
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 1a07091..80bf836 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -32,6 +32,8 @@ import roverlay.overlay.category
import roverlay.overlay.header
import roverlay.overlay.pkgdir.base
import roverlay.overlay.pkgdir.distroot.static
+import roverlay.overlay.rulegen
+
class Overlay ( roverlay.overlay.base.OverlayObject ):
DEFAULT_USE_DESC = (
@@ -701,14 +703,9 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
yield package
# --- end of list_packages (...) ---
- def list_rule_kwargs ( self ):
- for cat in self._categories.values():
- for kwargs in cat.list_packages (
- for_deprules = True,
- is_default_category = ( cat.name is self.default_category )
- ):
- yield kwargs
- # --- end of list_rule_kwargs (...) ---
+ def get_depres_rule_generator ( self ):
+ return roverlay.overlay.rulegen.DepresRuleGenerator ( self )
+ # --- end of get_depres_rule_generator (...) ---
def readonly ( self ):
return not self._writeable
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-09-02 16:21 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-09-02 16:21 UTC (permalink / raw
To: gentoo-commits
commit: 0b3a8fd9aada581d8510e4f2bd4a433702d792b0
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Sep 2 16:18:09 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Sep 2 16:18:09 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=0b3a8fd9
roverlay/overlay: depres rule generator
---
roverlay/overlay/rulegen.py | 71 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 71 insertions(+)
diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
new file mode 100644
index 0000000..7610e9a
--- /dev/null
+++ b/roverlay/overlay/rulegen.py
@@ -0,0 +1,71 @@
+# R overlay -- overlay package, depres rule generator
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+class DepresRuleGenerator ( object ):
+
+ def __init__ ( self, overlay ):
+ super ( DepresRuleGenerator, self ).__init__()
+ self.overlay_ref = overlay.get_ref()
+ self.rule_class = None
+ # --- end of __init__ (...) ---
+
+ def make_rule_args ( self ):
+ overlay = self.overlay_ref.deref_safe()
+ default_category_name = overlay.default_category
+
+ # COULDFIX: direct access to "private" attributes
+ for cat_name, cat in overlay._categories.items():
+ is_default_category = cat_name == default_category_name
+
+ for pkgdir_name, pkgdir in cat._subdirs.items():
+ if not pkgdir.empty():
+ repo_ids = set()
+ package_names = set()
+ for p_info in pkgdir.iter_package_info():
+ package_name = p_info.get (
+ 'package_name', do_fallback=True
+ )
+ if package_name:
+ package_names.add ( package_name )
+
+ repo = p_info.get ( 'origin', do_fallback=True )
+ if repo is not None:
+ repo_ids.add ( repo.get_identifier() )
+ # -- end for <get repo ids / package names>
+
+ yield (
+ repo_ids,
+ dict (
+ dep_str = pkgdir_name,
+ resolving_package = ( cat_name + '/' + pkgdir_name ),
+ is_selfdep = 2 if is_default_category else 1,
+ priority = 90,
+ finalize = True,
+ selfdep_package_names = package_names,
+ )
+ )
+ # -- end for pkgdir
+ # -- end for category
+ # --- end of make_rule_args (...) ---
+
+ def make_rules ( self ):
+ for repo_ids, rule_kwargs in self.make_rule_args():
+ yield ( repo_ids, self.rule_class ( **rule_kwargs ) )
+ # --- end of make_rules (...) ---
+
+ def make_rule_dict ( self ):
+ rule_dict = dict()
+ for repo_ids, rule_kwargs in self.make_rule_args():
+ rule = self.rule_class ( **rule_kwargs )
+ for repo_id in repo_ids:
+ if repo_id in rule_dict:
+ rule_dict [repo_id].append ( rule )
+ else:
+ rule_dict [repo_id] = [ rule ]
+ return rule_dict
+ # --- end of make_rule_dict (...) ---
+
+# --- end of DepresRuleGenerator ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-09-03 8:35 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-09-03 8:35 UTC (permalink / raw
To: gentoo-commits
commit: f874355dd601abd680b0a4978fb1bf6b11259099
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 08:32:57 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 08:32:57 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f874355d
depres rule generator: sort rules
+ don't ignore rules without repo ids
---
roverlay/overlay/rulegen.py | 46 ++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 41 insertions(+), 5 deletions(-)
diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
index 7610e9a..ea15b72 100644
--- a/roverlay/overlay/rulegen.py
+++ b/roverlay/overlay/rulegen.py
@@ -4,6 +4,8 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
+import collections
+
class DepresRuleGenerator ( object ):
def __init__ ( self, overlay ):
@@ -58,14 +60,48 @@ class DepresRuleGenerator ( object ):
def make_rule_dict ( self ):
rule_dict = dict()
+ rules_without_repo = list()
+
for repo_ids, rule_kwargs in self.make_rule_args():
rule = self.rule_class ( **rule_kwargs )
- for repo_id in repo_ids:
- if repo_id in rule_dict:
- rule_dict [repo_id].append ( rule )
- else:
- rule_dict [repo_id] = [ rule ]
+ if repo_ids:
+ for repo_id in repo_ids:
+ if repo_id in rule_dict:
+ rule_dict [repo_id].append ( rule )
+ else:
+ rule_dict [repo_id] = [ rule ]
+ else:
+ rules_without_repo.append ( rule )
+
+ # TODO: use distmap to restore repo ids
+ assert '_' not in rule_dict
+ rule_dict ['_'] = rules_without_repo
+
return rule_dict
# --- end of make_rule_dict (...) ---
+ def make_rule_list ( self, do_sort=False ):
+ if do_sort:
+ rule_dict = self.make_rule_dict()
+ for rules in rule_dict.values():
+ rules.sort ( key=( lambda k: k.priority ) )
+
+ undef_key = -1
+ #undef_key = max ( k for k in rule_dict if k != '_' ) + 1
+
+ return sorted (
+ rule_dict.items(),
+ key=lambda kv: ( undef_key if kv[0] == '_' else kv[0] )
+ )
+ else:
+ return list ( self.make_rule_dict().items() )
+ # --- end of make_rule_list (...) ---
+
+ def make_ordered_rule_dict ( self ):
+ # _not_ efficient:
+ # build a dict -> build a list -> build a dict
+ #
+ return collections.OrderedDict ( self.make_rule_list ( do_sort=True ) )
+ # --- end of make_ordered_rule_dict (...) ---
+
# --- end of DepresRuleGenerator ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-09-03 13:15 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-09-03 13:15 UTC (permalink / raw
To: gentoo-commits
commit: 6dafe81ba380a2c7c01ca048bc7245996692fefa
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 13:14:06 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 13:14:49 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6dafe81b
roverlay/overlay/root: remove get_depres_rule_generator()
no longer used
---
roverlay/overlay/root.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 80bf836..c6b205e 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -32,7 +32,6 @@ import roverlay.overlay.category
import roverlay.overlay.header
import roverlay.overlay.pkgdir.base
import roverlay.overlay.pkgdir.distroot.static
-import roverlay.overlay.rulegen
class Overlay ( roverlay.overlay.base.OverlayObject ):
@@ -703,10 +702,6 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
yield package
# --- end of list_packages (...) ---
- def get_depres_rule_generator ( self ):
- return roverlay.overlay.rulegen.DepresRuleGenerator ( self )
- # --- end of get_depres_rule_generator (...) ---
-
def readonly ( self ):
return not self._writeable
# --- end of readonly (...) ---
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-09-03 15:51 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-09-03 15:51 UTC (permalink / raw
To: gentoo-commits
commit: e6f457f95052e4de5ad349db4b5331956115c7dc
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 15:44:01 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 15:44:01 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=e6f457f9
remove TODO note
---
roverlay/overlay/rulegen.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
index 42b5d09..b2bf1e5 100644
--- a/roverlay/overlay/rulegen.py
+++ b/roverlay/overlay/rulegen.py
@@ -87,7 +87,6 @@ class DepresRuleGenerator ( object ):
else:
rules_without_repo.append ( rule )
- # TODO: use distmap to restore repo ids
assert '_' not in rule_dict
rule_dict ['_'] = rules_without_repo
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-09-17 13:49 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-09-17 13:49 UTC (permalink / raw
To: gentoo-commits
commit: 4502c63561b4e6390ecc7457d11c4fc1c15e2829
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Sep 16 14:56:43 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Sep 16 14:56:43 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=4502c635
initialize overlay before importing ebuilds
from import_ebuilds()'s comment:
do minimal overlay initialization before importing ebuilds.
This is recommended because portage expects metadata/layout.conf to exist,
for example.
---
roverlay/overlay/root.py | 227 +++++++++++++++++++++++++----------------------
1 file changed, 123 insertions(+), 104 deletions(-)
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index b3be5b3..0a3b15e 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -135,7 +135,6 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
self._runtime_incremental = write_allowed and runtime_incremental
self._writeable = write_allowed
- self._profiles_dir = self.physical_location + os.sep + 'profiles'
self._catlock = threading.Lock()
self._categories = dict()
@@ -148,8 +147,8 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
ebuild_header, eapi
)
self._use_desc = (
- use_desc.rstrip() if use_desc is not None else self.DEFAULT_USE_DESC
- )
+ use_desc if use_desc is not None else self.DEFAULT_USE_DESC
+ ).rstrip()
if keep_n_ebuilds:
self.keep_n_ebuilds = keep_n_ebuilds
@@ -271,133 +270,143 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
raise
# --- end of _import_eclass (...) ---
- def _init_overlay ( self, reimport_eclass ):
+ def _generate_layout_conf ( self ):
+ # create layout.conf file
+ # * create lines
+
+ kv_join = lambda k, v: "{k} = {v}".format ( k=k, v=v )
+ v_join = lambda v: ' '.join ( v )
+
+ yield kv_join ( "repo_name", self.name )
+
+ if self._masters:
+ yield kv_join ( "masters", v_join ( self._masters ) )
+ else:
+ yield "masters ="
+
+
+ # strictly speaking,
+ # declaring cache-formats here is not correct since egencache
+ # is run as hook
+ yield kv_join ( "cache-formats", "md5-dict" )
+
+ ##yield kv_join ( "sign-commits", "false" )
+ ##yield kv_join ( "sign-manifests", "false" )
+ ##yield kv_join ( "thin-manifests", "false" )
+
+ hashes = roverlay.overlay.pkgdir.base.get_class().HASH_TYPES
+ if hashes:
+ yield kv_join ( "manifest-hashes", v_join ( hashes ) )
+ # --- end of _generate_layout_conf (...) ---
+
+ def _init_overlay ( self, reimport_eclass, minimal=False ):
"""Initializes the overlay at its physical/filesystem location.
arguments:
- * reimport_eclass -- whether to copy existing eclass files
- again (True) or not
- * make_profiles_dir -- if True: create the profiles/ dir now
+ * reimport_eclass -- whether to copy existing eclass files
+ again (True) or not
+ * minimal -- whether to do full overlay initialization (False)
+ or not (True). Defaults to False.
raises:
- * IOError
+ * IOError, OSError -- passed from file writing / dir creation
"""
- NEWLINE = '\n'
- EMPTY_STR = ""
+ NEWLINE = '\n'
+ CONFIG_GET = roverlay.config.get
+ CONFIG_GET_OR_FAIL = roverlay.config.get_or_fail
+ ROOT = str ( self.physical_location )
+ METADATA_DIR = ROOT + os.sep + 'metadata'
+ PROFILES_DIR = ROOT + os.sep + 'profiles'
- def write_profiles_dir():
- """Creates and updates the profiles/ dir."""
- def write_profiles_file ( filename, to_write ):
- """Writes a file in profiles/.
+ def get_subdir_file_write ( root_dir ):
+ """Returns a function for writing files in the given directory.
+
+ arguments:
+ * root_dir --
+ """
+ def write_subdir_file ( relpath, content, append_newline=True ):
+ """Writes a file (<root_dir>/<relpath>).
arguments:
- * filename -- name of the file to write (including file extension)
- * to_write -- string to write (don't forget newline at the end)
+ * relpath -- file path relative to root_dir
+ * content -- data to write
+ * append_newline -- whether to append a newline at EOF
"""
- fh = None
- try:
- fh = open ( self._profiles_dir + os.sep + filename, 'w' )
- if to_write:
- # else touch file
- fh.write ( to_write )
- except IOError as e:
- self.logger.exception ( e )
- raise
- finally:
- if fh: fh.close()
- # --- end of write_profiles_file (...) ---
-
- # always use the default category (+write it into profiles/categories)
- self._get_category ( self.default_category )
+ content_str = str ( content )
+ if content_str:
+ with open ( root_dir + os.sep + relpath, 'wt' ) as FH:
+ FH.write ( content_str )
+ if append_newline:
+ FH.write ( NEWLINE )
+ else:
+ # touch file
+ with open ( root_dir + os.sep + relpath, 'a' ) as FH:
+ pass
+ # --- end of write_subdir_file (...) ---
+ return write_subdir_file
+ # --- end of get_subdir_file_write (...) ---
- # profiles/
- roverlay.util.dodir ( self._profiles_dir )
- # profiles/repo_name
- write_profiles_file ( 'repo_name', self.name + '\n' )
-
- # profiles/categories
- cats = '\n'.join (
- k for k, v in self._categories.items() if not v.empty()
- )
- if cats:
- write_profiles_file ( 'categories', cats + '\n' )
-
- # profiles/desc/<r_suggests>.desc
- use_expand_name = roverlay.config.get_or_fail (
- "EBUILD.USE_EXPAND.name"
- ).rstrip ( "_" )
-
- self._write_rsuggests_use_desc (
- desc_file = (
- self._profiles_dir + os.sep + 'desc' + os.sep
- + use_expand_name.lower() + '.desc'
- ),
- use_expand_name = use_expand_name.upper(),
- backup_file = roverlay.config.get ( 'OVERLAY.backup_desc', True ),
- flagdesc_file = roverlay.config.get (
- 'EBUILD.USE_EXPAND.desc_file', None
- ),
- )
+ write_profiles_file = get_subdir_file_write ( PROFILES_DIR )
+ write_metadata_file = get_subdir_file_write ( METADATA_DIR )
+ try:
+ layout_conf_str = NEWLINE.join ( self._generate_layout_conf() )
- # profiles/use.desc
- if self._use_desc:
- write_profiles_file ( 'use.desc', self._use_desc + '\n' )
- # --- end of write_profiles_dir (...) ---
+ ## make overlay dirs, root, metadata/, profiles/
+ roverlay.util.dodir ( ROOT, mkdir_p=True )
+ roverlay.util.dodir ( PROFILES_DIR )
+ roverlay.util.dodir ( METADATA_DIR )
- def write_metadata_dir():
- METADATA_DIR = self.physical_location + os.sep + 'metadata'
- roverlay.util.dodir ( METADATA_DIR )
+ ## import eclass files
+ self._import_eclass ( reimport_eclass )
- # create layout.conf file
- # * create lines
- layout_lines = list()
- layout_add = layout_lines.append
- kv_join = lambda k, v: "{k} = {v}".format ( k=k, v=v )
- v_join = lambda v: ' '.join ( v )
- layout_add ( kv_join ( "repo_name", self.name ) )
+ ## populate profiles/
- if self._masters:
- layout_add ( kv_join ( "masters", v_join ( self._masters ) ) )
- else:
- layout_add ( "masters =" )
+ # profiles/repo_name
+ write_profiles_file ( 'repo_name', self.name )
+ # profiles/categories
+ cats = NEWLINE.join (
+ k for k, v in self._categories.items() if not v.empty()
+ )
+ if cats:
+ write_profiles_file ( 'categories', cats )
- # strictly speaking,
- # declaring cache-formats here is not correct since egencache
- # is run as hook
- layout_add ( kv_join ( "cache-formats", "md5-dict" ) )
- ##layout_add ( kv_join ( "sign-commits", "false" ) )
- ##layout_add ( kv_join ( "sign-manifests", "false" ) )
- ##layout_add ( kv_join ( "thin-manifests", "false" ) )
+ if not minimal:
+ # profiles/desc/<r_suggests>.desc
+ use_expand_name = (
+ CONFIG_GET_OR_FAIL ( "EBUILD.USE_EXPAND.name" ).rstrip ( "_" )
+ )
- hashes = roverlay.overlay.pkgdir.base.get_class().HASH_TYPES
- if hashes:
- layout_add ( kv_join ( "manifest-hashes", v_join ( hashes ) ) )
+ self._write_rsuggests_use_desc (
+ desc_file = os.sep.join ([
+ PROFILES_DIR, 'desc', use_expand_name.lower(), '.desc'
+ ]),
+ use_expand_name = use_expand_name.upper(),
+ backup_file = CONFIG_GET ( 'OVERLAY.backup_desc', True ),
+ flagdesc_file = CONFIG_GET (
+ 'EBUILD.USE_EXPAND.desc_file', None
+ ),
+ )
- # * write it
- with open ( METADATA_DIR + os.sep + 'layout.conf', 'wt' ) as FH:
- for line in layout_lines:
- FH.write ( line )
- FH.write ( NEWLINE )
- # --- end of write_metadata_dir (...) ---
- try:
- # mkdir overlay root
- roverlay.util.dodir ( self.physical_location, mkdir_p=True )
+ # profiles/use.desc
+ if self._use_desc:
+ write_profiles_file ( 'use.desc', self._use_desc )
- self._import_eclass ( reimport_eclass )
+ # -- end if not minimal ~ profiles/
- write_profiles_dir()
- write_metadata_dir()
+ ## metadata/
- except IOError as e:
+ # metadata/layout.conf
+ write_metadata_file ( 'layout.conf', layout_conf_str )
+ except ( OSError, IOError ) as e:
self.logger.exception ( e )
self.logger.critical ( "failed to init overlay" )
raise
@@ -710,13 +719,21 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
return not self._writeable
# --- end of readonly (...) ---
- def import_ebuilds ( self, overwrite, nosync=False ):
+ def import_ebuilds ( self, overwrite, nosync=False, init_overlay=True ):
"""Imports ebuilds from the additions dir.
arguments:
- * overwrite -- whether to overwrite existing ebuilds
- * nosync -- if True: don't fetch src files (defaults to False)
+ * overwrite -- whether to overwrite existing ebuilds
+ * nosync -- if True: don't fetch src files (defaults to False)
+ * init_overlay -- if True: do minimal overlay initialization before
+ importing ebuilds. This is recommended
+ because portage expects metadata/layout.conf
+ to exist, for example.
+ Defaults to True.
"""
+ if init_overlay:
+ self._init_overlay ( False, minimal=True )
+
for catview in (
roverlay.overlay.additionsdir.CategoryRootView ( self.additions_dir )
):
@@ -887,7 +904,9 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
if self._writeable and not self.skip_manifest:
# profiles/categories is required for successful Manifest
# creation
- if os.path.isfile ( self._profiles_dir + os.sep + 'categories' ):
+ if os.path.isfile ( os.path.join (
+ str ( self.physical_location ), 'profiles', 'categories'
+ ) ):
for cat in self._categories.values():
cat.write_manifest ( **manifest_kw )
else:
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2013-09-18 14:00 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2013-09-18 14:00 UTC (permalink / raw
To: gentoo-commits
commit: 66b7738ec3d88f0766e785cdf890d71597cd5ba0
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Sep 18 13:56:53 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Sep 18 13:56:53 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=66b7738e
fix 4502c63561b4e6390ecc7457d11c4fc1c15e2829
fix a typo causing _write_rsuggests_use_desc() to fail
---
roverlay/overlay/root.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 0a3b15e..34a15ee 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -385,8 +385,8 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
self._write_rsuggests_use_desc (
desc_file = os.sep.join ([
- PROFILES_DIR, 'desc', use_expand_name.lower(), '.desc'
- ]),
+ PROFILES_DIR, 'desc', use_expand_name.lower(),
+ ]) + '.desc',
use_expand_name = use_expand_name.upper(),
backup_file = CONFIG_GET ( 'OVERLAY.backup_desc', True ),
flagdesc_file = CONFIG_GET (
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2014-04-01 16:38 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2014-04-01 16:38 UTC (permalink / raw
To: gentoo-commits
commit: 6596fb255d1f17017420401a4e39f8a711bcc239
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Mar 31 15:38:36 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Mar 31 15:38:36 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6596fb25
overlay/creator: fix comment
---
roverlay/overlay/creator.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index ea623e6..75119d3 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -404,7 +404,6 @@ class OverlayCreator ( object ):
## num_removed += selfdep.reduce()
## end loop
##
- ## num_removed <- 0
## end loop
##
self.overlay.link_selfdeps ( selfdeps )
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2014-07-18 16:20 André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2014-07-18 16:20 UTC (permalink / raw
To: gentoo-commits
commit: 093b182951ad7ab6eb7e519061c7b95dc218aacb
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 17 19:14:21 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 17 19:14:21 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=093b1829
overlay/abccontrol: convert_str()
---
roverlay/overlay/abccontrol.py | 42 +++++++++++++++++++++++++++++++-----------
1 file changed, 31 insertions(+), 11 deletions(-)
diff --git a/roverlay/overlay/abccontrol.py b/roverlay/overlay/abccontrol.py
index 364a083..d031eec 100644
--- a/roverlay/overlay/abccontrol.py
+++ b/roverlay/overlay/abccontrol.py
@@ -82,17 +82,37 @@ class AdditionControlResult ( object ):
) = _gen_bits(5)
-# PKG_DESCRIPTION_MAP = {
-# PKG_FORCE_DENY : 'force-deny',
-# PKG_DENY_REPLACE : 'deny-replace',
-# PKG_FORCE_REPLACE : 'force-replace',
-# PKG_REPLACE_ONLY : 'replace-only',
-# PKG_REVBUMP_ON_COLLISION : 'revbump-on-collision',
-# PKG_DEFAULT_BEHAVIOR : 'default',
-# }
-#
-# PKG_DESCRIPTION_REVMAP = { v: k for k,v in PKG_DESCRIPTION_MAP.items() }
-#
+ PKG_DESCRIPTION_MAP = {
+ PKG_FORCE_DENY : 'force-deny',
+ PKG_DENY_REPLACE : 'deny-replace',
+ PKG_FORCE_REPLACE : 'force-replace',
+ PKG_REPLACE_ONLY : 'replace-only',
+ PKG_REVBUMP_ON_COLLISION : 'revbump-on-collision',
+ PKG_DEFAULT_BEHAVIOR : 'default',
+ }
+
+ PKG_DESCRIPTION_REVMAP = { v: k for k,v in PKG_DESCRIPTION_MAP.items() }
+
+ @classmethod
+ def convert_str ( cls, s ):
+ desc_rmap = cls.PKG_DESCRIPTION_REVMAP
+
+ if not s:
+ raise ValueError ( "str must not be empty." )
+
+ bitmask = 0
+
+ for arg in filter ( None, s.strip().lower().split(",") ):
+ # ^ strip(),lower(),split() ...
+ print(arg)
+ try:
+ bitmask |= desc_rmap [arg]
+ except KeyError:
+ raise ValueError ( arg )
+ # -- end for
+
+ return bitmask
+ # --- end of convert_str (...) ---
@classmethod
def get_reversed_sort_key ( cls, k ):
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
2014-07-18 2:28 [gentoo-commits] proj/R_overlay:wip/addition_control " André Erdmann
@ 2014-07-18 16:20 ` André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2014-07-18 16:20 UTC (permalink / raw
To: gentoo-commits
commit: 13ae9d788d175488eaa5b4ba2637ed04fb6423bb
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 18 01:00:42 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 18 01:00:42 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=13ae9d78
roverlay/overlay/abccontrol: remove dbg statement
---
roverlay/overlay/abccontrol.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/roverlay/overlay/abccontrol.py b/roverlay/overlay/abccontrol.py
index d031eec..a319e35 100644
--- a/roverlay/overlay/abccontrol.py
+++ b/roverlay/overlay/abccontrol.py
@@ -104,7 +104,6 @@ class AdditionControlResult ( object ):
for arg in filter ( None, s.strip().lower().split(",") ):
# ^ strip(),lower(),split() ...
- print(arg)
try:
bitmask |= desc_rmap [arg]
except KeyError:
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
2014-07-18 2:28 [gentoo-commits] proj/R_overlay:wip/addition_control " André Erdmann
@ 2014-07-18 16:20 ` André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2014-07-18 16:20 UTC (permalink / raw
To: gentoo-commits
commit: 9c4859591aa5d5388c192f932f3e4d8a1344e57d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 18 02:23:31 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 18 02:23:31 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9c485959
roverlay/overlay/abccontrol: comment
---
roverlay/overlay/abccontrol.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/roverlay/overlay/abccontrol.py b/roverlay/overlay/abccontrol.py
index a319e35..991343e 100644
--- a/roverlay/overlay/abccontrol.py
+++ b/roverlay/overlay/abccontrol.py
@@ -81,6 +81,10 @@ class AdditionControlResult ( object ):
PKG_ALL,
) = _gen_bits(5)
+## # virtuals
+## PKG_REVBUMP_ONLY = PKG_REVBUMP_ON_COLLISION|PKG_REPLACE_ONLY
+## PKG_FORCE_REPLACE_ONLY = PKG_FORCE_REPLACE|PKG_REPLACE_ONLY
+
PKG_DESCRIPTION_MAP = {
PKG_FORCE_DENY : 'force-deny',
@@ -89,6 +93,10 @@ class AdditionControlResult ( object ):
PKG_REPLACE_ONLY : 'replace-only',
PKG_REVBUMP_ON_COLLISION : 'revbump-on-collision',
PKG_DEFAULT_BEHAVIOR : 'default',
+
+## # virtuals
+## PKG_REVBUMP_ONLY : 'revbump-on-collision,replace-only',
+## PKG_FORCE_REPLACE_ONLY : 'force-replace,replace-only',
}
PKG_DESCRIPTION_REVMAP = { v: k for k,v in PKG_DESCRIPTION_MAP.items() }
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
2014-08-23 19:03 André Erdmann
@ 2014-07-29 18:29 ` André Erdmann
0 siblings, 0 replies; 42+ messages in thread
From: André Erdmann @ 2014-07-29 18:29 UTC (permalink / raw
To: gentoo-commits
commit: 8c67544ba3552ae6a67a10145036d42d8692d10c
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 29 18:22:04 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 29 18:22:04 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=8c67544b
roverlay/overlay/rulegen: fix dot handling
for scanned ebuilds: replace underscore chars '_' with dot chars '.' and the
resulting str as package name alias to the dyn. dep rule.
---
roverlay/overlay/rulegen.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
index b2bf1e5..dcc4723 100644
--- a/roverlay/overlay/rulegen.py
+++ b/roverlay/overlay/rulegen.py
@@ -40,6 +40,8 @@ class DepresRuleGenerator ( object ):
)
if package_name:
package_names.add ( package_name )
+ else:
+ package_names.add ( pkgdir_name.replace('_','.') )
repo = p_info.get ( 'origin', do_fallback=True )
if repo is not None:
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/
@ 2014-08-23 19:03 André Erdmann
2014-07-29 18:29 ` André Erdmann
0 siblings, 1 reply; 42+ messages in thread
From: André Erdmann @ 2014-08-23 19:03 UTC (permalink / raw
To: gentoo-commits
commit: 8c67544ba3552ae6a67a10145036d42d8692d10c
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 29 18:22:04 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 29 18:22:04 2014 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=8c67544b
roverlay/overlay/rulegen: fix dot handling
for scanned ebuilds: replace underscore chars '_' with dot chars '.' and the
resulting str as package name alias to the dyn. dep rule.
---
roverlay/overlay/rulegen.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
index b2bf1e5..dcc4723 100644
--- a/roverlay/overlay/rulegen.py
+++ b/roverlay/overlay/rulegen.py
@@ -40,6 +40,8 @@ class DepresRuleGenerator ( object ):
)
if package_name:
package_names.add ( package_name )
+ else:
+ package_names.add ( pkgdir_name.replace('_','.') )
repo = p_info.get ( 'origin', do_fallback=True )
if repo is not None:
^ permalink raw reply related [flat|nested] 42+ messages in thread
end of thread, other threads:[~2014-08-23 19:03 UTC | newest]
Thread overview: 42+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2012-07-24 16:59 [gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/ André Erdmann
2012-07-30 8:52 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
-- strict thread matches above, loose matches on Subject: below --
2014-08-23 19:03 André Erdmann
2014-07-29 18:29 ` André Erdmann
2014-07-18 16:20 André Erdmann
2014-07-18 2:28 [gentoo-commits] proj/R_overlay:wip/addition_control " André Erdmann
2014-07-18 16:20 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
2014-07-18 2:28 [gentoo-commits] proj/R_overlay:wip/addition_control " André Erdmann
2014-07-18 16:20 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
2014-04-01 16:38 André Erdmann
2013-09-18 14:00 André Erdmann
2013-09-17 13:49 André Erdmann
2013-09-03 15:51 André Erdmann
2013-09-03 13:15 André Erdmann
2013-09-03 8:35 André Erdmann
2013-09-02 16:21 André Erdmann
2013-09-02 16:21 André Erdmann
2013-08-29 12:36 André Erdmann
2013-08-20 21:46 André Erdmann
2013-08-20 21:46 André Erdmann
2013-07-29 14:56 André Erdmann
2013-07-10 15:10 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
2013-07-10 16:16 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
2013-06-22 15:24 André Erdmann
2013-06-18 14:12 André Erdmann
2013-06-13 16:34 André Erdmann
2013-04-25 16:44 André Erdmann
2013-04-25 16:44 André Erdmann
2013-02-09 21:28 André Erdmann
2013-02-09 20:45 André Erdmann
2013-02-09 20:45 André Erdmann
2013-01-30 20:16 André Erdmann
2012-08-17 17:26 André Erdmann
2012-08-03 13:38 André Erdmann
2012-08-01 21:10 André Erdmann
2012-07-30 15:53 André Erdmann
2012-07-30 8:52 André Erdmann
2012-07-30 8:52 André Erdmann
2012-07-05 16:00 André Erdmann
2012-07-04 18:21 André Erdmann
2012-07-03 17:48 André Erdmann
2012-06-27 14:46 André Erdmann
2012-06-26 15:42 André Erdmann
2012-06-22 18:13 André Erdmann
2012-06-21 16:55 André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox