From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.77) (envelope-from ) id 1SmUCk-00062Z-9G for garchives@archives.gentoo.org; Wed, 04 Jul 2012 18:21:34 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 435F1E062C; Wed, 4 Jul 2012 18:21:12 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 04A07E062C for ; Wed, 4 Jul 2012 18:21:11 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 265AB1B4091 for ; Wed, 4 Jul 2012 18:21:10 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id E3B7AE5436 for ; Wed, 4 Jul 2012 18:21:08 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1341425414.6d1ec878f71025e6681255f8b63cd404f9ce74f3.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/overlay/__init__.py roverlay/overlay/category.py roverlay/overlay/creator.py roverlay/overlay/package.py X-VCS-Directories: roverlay/overlay/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: 6d1ec878f71025e6681255f8b63cd404f9ce74f3 X-VCS-Branch: master Date: Wed, 4 Jul 2012 18:21:08 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: e9c4ae05-ac29-44fc-88b7-32da6b898fde X-Archives-Hash: ed470a27e193b57761766b390d13092d commit: 6d1ec878f71025e6681255f8b63cd404f9ce74f3 Author: Andr=C3=A9 Erdmann mailerd de> AuthorDate: Wed Jul 4 18:10:14 2012 +0000 Commit: Andr=C3=A9 Erdmann mailerd de> CommitDate: Wed Jul 4 18:10:14 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/R_overlay.git= ;a=3Dcommit;h=3D6d1ec878 overlay: collect some stats modified: roverlay/overlay/__init__.py modified: roverlay/overlay/category.py modified: roverlay/overlay/creator.py modified: roverlay/overlay/package.py --- roverlay/overlay/__init__.py | 4 +- roverlay/overlay/category.py | 4 +- roverlay/overlay/creator.py | 167 ++++++++++++++++++++++++++++++++++++= +++--- roverlay/overlay/package.py | 22 +++--- 4 files changed, 171 insertions(+), 26 deletions(-) diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py index a8a939d..13b6615 100644 --- a/roverlay/overlay/__init__.py +++ b/roverlay/overlay/__init__.py @@ -99,9 +99,9 @@ class Overlay ( object ): * category -- category where the pkg should be put in, defaults to self.default_category =20 - returns: None (implicit) + returns: True if successfully added else False """ - self._get_category ( + return self._get_category ( self.default_category if category is None else category ) . add ( package_info ) # --- end of add (...) --- diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py index e938dbf..271b524 100644 --- a/roverlay/overlay/category.py +++ b/roverlay/overlay/category.py @@ -46,7 +46,7 @@ class Category ( object ): arguments: * package_info -- =20 - returns: None (implicit) + returns: success """ pkg_name =3D package_info ['name'] =20 @@ -63,7 +63,7 @@ class Category ( object ): finally: self._lock.release() =20 - self._subdirs [pkg_name].add ( package_info ) + return self._subdirs [pkg_name].add ( package_info ) # --- end of add (...) --- =20 def generate_metadata ( self, **metadata_kw ): diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py index 79f2da3..bb2391b 100644 --- a/roverlay/overlay/creator.py +++ b/roverlay/overlay/creator.py @@ -5,10 +5,12 @@ import time import logging import threading -import signal -import traceback +#import signal +#import traceback import sys =20 +from collections import deque + try: import queue except ImportError: @@ -23,18 +25,61 @@ from roverlay.packageinfo import PackageInfo =20 from roverlay.recipe import easyresolver =20 -LOGGER =3D logging.getLogger ( 'OverlayCreator' ) - +# this is used as the default value for can_write_overlay in OverlayCrea= tor +# instances OVERLAY_WRITE_ALLOWED =3D False =20 +class PseudoAtomicCounter ( object ): + + def __init__ ( self, start=3D0, long_int=3DFalse ): + if long_int and sys.version_info < ( 3, 0 ): + self._value =3D long ( start ) + else: + self._value =3D int ( start ) + self._lock =3D threading.Lock() + + def _get_and_inc ( self, step ): + ret =3D None + self._lock.acquire() + try: + old_val =3D self._value + if step > 0: + self._value +=3D step + ret =3D ( self._value, old_val ) + else: + ret =3D old_val + finally: + self._lock.release() + + return ret + # --- end of _get_and_inc (...) --- + + def inc ( self, step=3D1 ): + self._get_and_inc ( step ) + # --- end of inc (...) --- + + def get ( self ): + return self._get_and_inc ( 0 ) + # --- end of get (...) --- + + def get_nowait ( self ): + return self._value + # --- end of get_nowait (...) --- + + def __str__ ( self ): + return str ( self._value ) + # --- end of __str__ (...) --- + =20 class OverlayCreator ( object ): """This is a 'R packages -> Overlay' interface.""" =20 + LOGGER =3D logging.getLogger ( 'OverlayCreator' ) + def __init__ ( self, logger=3DNone ): =20 if logger is None: - self.logger =3D LOGGER + self.logger =3D self.__class__.LOGGER else: self.logger =3D logger.getChild ( 'OverlayCreator' ) =20 @@ -52,19 +97,97 @@ class OverlayCreator ( object ): # it's self._err_queue =3D queue.Queue() =20 + #self._time_start_run =3D list() + #self._time_stop_run =3D list() + =20 self._workers =3D None self._runlock =3D threading.RLock() =20 self.can_write_overlay =3D OVERLAY_WRITE_ALLOWED =20 - # this is a method that adds PackageInfo objects to the pkg queue - self.add_package =3D self._pkg_queue.put =20 self.depresolver.set_exception_queue ( self._err_queue ) =20 + self.closed =3D False + + + # queued packages counter, + # package_added !=3D (create_success + create_fail) if a thread hangs + # or did not call _pkg_done + self.package_added =3D PseudoAtomicCounter() + + # counts packages that passed ebuild creation + self.create_success =3D PseudoAtomicCounter() + + # counts packages that failed ebuild creation + self.create_fail =3D PseudoAtomicCounter() + + # counts packages that passed adding to overlay + self.overlay_added =3D PseudoAtomicCounter() + # --- end of __init__ (...) --- =20 + def get_stats ( self ): + pkg_added =3D self.package_added.get_nowait() + pkg_created =3D self.create_success.get_nowait() + pkg_failed =3D self.create_fail.get_nowait() + ov_added =3D self.overlay_added.get_nowait() + ov_failed =3D pkg_created - ov_added + processed =3D pkg_created + pkg_failed + failed =3D pkg_failed + ov_failed + + + # namedtuple? TODO + + return ( + pkg_added, pkg_created, pkg_failed, + ov_added, ov_failed, + processed, failed + ) + # --- end of get_stats (...) --- + + def stats_str ( self, enclose=3DTrue ): + """Returns a string with some overlay creation stats.""" + def stats_gen(): + """Yields stats strings.""" + stats =3D self.get_stats() + + # the length of the highest number in stats (^=3Ddigit count) + # max_number_len :=3D { 1,...,5 } + max_number_len =3D min ( 5, len ( str ( max ( stats ) ) ) ) + num_fmt =3D '%(num)-' + str ( max_number_len ) + 's ' + + for i, s in enumerate (( + 'packages added to the ebuild creation queue', + 'packages passed ebuild creation', + 'packages failed ebuild creation', + 'ebuilds could be added to the overlay', + 'ebuilds couldn\'t be added to the overlay', + 'packages processed in total', + 'packages failed in total', + )): + yield num_fmt % { 'num' : stats [i] } + s + # --- end of stats_gen (...) --- + + if enclose: + stats_str =3D deque ( stats_gen() ) + # maxlen :=3D { 2,...,80 } + maxlen =3D 2 + min ( 78, + len ( max ( stats_str, key=3Dlambda s : len( s ) ) ) + ) + + stats_str.appendleft ( + " Overlay creation stats ".center ( maxlen, '-' ) + ) + stats_str.append ( '-' * maxlen ) + + return '\n'.join ( stats_str ) + + else: + return '\n'.join ( stats_gen() ) + # --- end of stats_str (...) --- + def _timestamp ( self, description, start, stop=3DNone ): """Logs a timestamp, used for testing. =20 @@ -82,14 +205,25 @@ class OverlayCreator ( object ): return _stop # --- end of _timestamp (...) --- =20 + def add_package ( self, package_info ): + """Adds a PackageInfo to the package queue. + + arguments: + * package_info -- + """ + self._pkg_queue.put ( package_info ) + self.package_added.inc() + def add_package_file ( self, package_file ): """Adds a single R package.""" self._pkg_queue.put ( PackageInfo ( filepath=3Dpackage_file ) ) + self.package_added.inc() # --- end of add_package (...) --- =20 def add_package_files ( self, *package_files ): """Adds multiple R packages.""" for p in package_files: self.add_package_file ( p ) + self.package_added.inc() # --- end of add_packages (...) --- =20 =20 @@ -117,14 +251,18 @@ class OverlayCreator ( object ): self.overlay.show() # --- end of show_overlay (...) --- =20 - def run ( self ): + def run ( self, close_when_done=3DFalse ): """Starts ebuild creation and waits until done.""" self._runlock.acquire() + #self._time_start_run.append ( time.time() ) try: self.start() self.join() finally: + #self._time_stop_run.append ( time.time() ) self._runlock.release() + if close_when_done: + self.close() # --- end of run (...) --- =20 def start ( self ): @@ -145,11 +283,11 @@ class OverlayCreator ( object ): self._join_workers() # --- end of wait (...) --- =20 - def close ( self, write=3DFalse ): + def close ( self ): """Closes this OverlayCreator.""" self._close_workers() self._close_resolver() - if write: self.write_overlay() + self.closed =3D True # --- end of close (...) --- =20 def _close_resolver ( self ): @@ -266,8 +404,13 @@ class OverlayCreator ( object ): # * request an incremental write to save memory etc. =20 # if <>: - if package_info ['ebuild'] is not None: - self.overlay.add ( package_info ) + if package_info ['ebuild'] is None: + self.create_fail.inc() + else: + self.create_success.inc() + if self.overlay.add ( package_info ): + self.overlay_added.inc() + # --- end of _add_to_overlay (...) --- =20 def _get_worker ( self, start_now=3DFalse, use_threads=3DTrue ): diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py index 77fcbee..429200f 100644 --- a/roverlay/overlay/package.py +++ b/roverlay/overlay/package.py @@ -203,11 +203,9 @@ class PackageDir ( object ): """ shortver =3D package_info ['ebuild_verstr'] =20 - def already_exists ( release=3DFalse ): + def already_exists (): if shortver in self._packages: =20 - if release: self._lock.release() - msg =3D "'%s-%s.ebuild' already exists, cannot add it!" % ( self.name, shortver ) @@ -221,14 +219,18 @@ class PackageDir ( object ): return False # --- end of already_exists (...) --- =20 - if already_exists ( release=3DFalse ): return False - self._lock.acquire() - if already_exists ( release=3DTrue ): return False - - self._packages [shortver] =3D package_info + _success =3D False =20 - self._lock.release() - return True + if not already_exists(): + try: + self._lock.acquire() + if not already_exists(): + self._packages [shortver] =3D package_info + _success =3D True + finally: + self._lock.release() + + return _success # --- end of add (...) --- =20 def _regen_metadata ( self ):