From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 7BFEF1381F3 for ; Fri, 26 Jul 2013 13:02:10 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 13A34E09EC; Fri, 26 Jul 2013 13:02:07 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 0A54FE09F1 for ; Fri, 26 Jul 2013 13:02:05 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id D518233EA17 for ; Fri, 26 Jul 2013 13:02:04 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 67569E468F for ; Fri, 26 Jul 2013 13:02:03 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1374843461.d1ca42185a44d01176d9944e95a112493401f1f6.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/stats/abstract.py roverlay/stats/base.py roverlay/stats/collector.py X-VCS-Directories: roverlay/stats/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: d1ca42185a44d01176d9944e95a112493401f1f6 X-VCS-Branch: master Date: Fri, 26 Jul 2013 13:02:03 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 51903453-8645-4813-9a24-71ea6a94c35a X-Archives-Hash: 5e9bdfc2dcaeccf357b5a05ed2ea18c1 commit: d1ca42185a44d01176d9944e95a112493401f1f6 Author: André Erdmann mailerd de> AuthorDate: Fri Jul 26 12:57:41 2013 +0000 Commit: André Erdmann mailerd de> CommitDate: Fri Jul 26 12:57:41 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d1ca4218 statc collection: data types, time stats --- roverlay/stats/abstract.py | 125 +++++++++++++++++++++++++++++++++++++++----- roverlay/stats/base.py | 54 ++++++++++++++----- roverlay/stats/collector.py | 31 ++++++++--- 3 files changed, 177 insertions(+), 33 deletions(-) diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py index 993d48b..36c81ca 100644 --- a/roverlay/stats/abstract.py +++ b/roverlay/stats/abstract.py @@ -26,15 +26,31 @@ class RoverlayStatsBase ( object ): return cls() # --- end of get_new (...) --- + def __init__ ( self, description=None ): + super ( RoverlayStatsBase, self ).__init__() + if description is not None: + self.description = description + # --- end of __init__ (...) --- + def merge_with ( self, other ): - if hasattr ( self, '_MEMBERS' ): - self.merge_members ( other ) + my_cls = self.__class__ + their_cls = other.__class__ + + if ( + issubclass ( my_cls, their_cls ) + and hasattr ( their_cls, '_MEMBERS' ) + ): + self.merge_members ( other, their_cls._MEMBERS ) + + elif hasattr ( my_cls, '_MEMBERS' ): + self.merge_members ( other, my_cls._MEMBERS ) + else: raise MethodNotImplemented ( self, 'merge_with' ) # --- end of merge_with (...) --- - def merge_members ( self, other ): - for member in self.__class__._MEMBERS: + def merge_members ( self, other, members ): + for member in members: getattr ( self, member ).merge_with ( getattr ( other, member ) ) # --- end of merge_members (...) --- @@ -84,21 +100,96 @@ class RoverlayStatsBase ( object ): class RoverlayStats ( RoverlayStatsBase ): + pass +# --- end of RoverlayStats --- - def get_time ( self ): - return time.time() - # --- end of get_time (...) --- - def get_time_delta ( self, start, stop=None ): - t_stop = self.get_time() if stop is None else stop - return ( start - t_stop, t_stop ) - # --- end of get_time_delta (...) --- +class TimeStatsItem ( RoverlayStatsBase ): + # doc TODO: note somewhere that those timestats are just approximate + # values + + def __init__ ( self, t_begin=None, t_end=None, description=None ): + super ( TimeStatsItem, self ).__init__ ( description=description ) + self.time_begin = t_begin if t_begin is not None else time.time() + self.time_end = t_end + # --- end of __init__ (...) --- + + def end ( self, t_end=None ): + self.time_end = time.time() if t_end is None else t_end + # --- end of end (...) --- + + def get_delta ( self ): + if self.time_begin is None: + return -1.0 + elif self.time_end is None: + return -2.0 + else: + return float ( self.time_end ) - float ( self.time_begin ) + # --- end of get_delta (...) --- + + def __str__ ( self ): + return "{:.3f}s".format ( self.get_delta() ) + # --- end of __str__ (...) --- + +# --- end of TimeStatsItem --- + + +class TimeStats ( RoverlayStats ): + + def __init__ ( self, description=None ): + super ( TimeStats, self ).__init__ ( description=description ) + self._timestats = collections.OrderedDict() + # --- end of __init__ (...) --- + + def merge_with ( self, other ): + self._timestats.update ( other._timestats ) + # --- end of merge_with (...) --- + + def get ( self, key ): + return self._timestats.get ( key, -1.0 ) + # --- end of get (...) --- + + def __getitem__ ( self, key ): + return self._timestats [key] + # --- end of __getitem__ (...) --- + + def begin ( self, key ): + item = TimeStatsItem() + self._timestats [key] = item + return item + # --- end of begin (...) --- + + def end ( self, key ): + item = self._timestats [key] + item.end() + return item + # --- end of end (...) --- + + def get_total ( self ): + return sum ( v.get_delta() for v in self._timestats.values() ) + # --- end of get_total (...) --- + + def gen_str ( self ): + desc = self.get_description_str() + if desc: + yield "{} ({:.3f}s)".format ( desc, self.get_total() ) + else: + yield "total time: {:.3f}s".format ( self.get_total() ) + + for key, value in self._timestats.items(): + yield "* {k}: {v}".format ( k=key, v=str ( value ) ) + # --- end of gen_str (...) --- + + def __str__ ( self ): + return '\n'.join ( self.gen_str() ) + # --- end of __str__ (...) --- + +# --- end of TimeStats --- -# --- end of RoverlayStats --- class Counter ( RoverlayStatsBase ): def __init__ ( self, description=None, initial_value=0 ): - super ( Counter, self ).__init__() + super ( Counter, self ).__init__ ( description=description ) self.description = description self.total_count = initial_value self.underflow = False @@ -112,6 +203,14 @@ class Counter ( RoverlayStatsBase ): return float ( self.total_count ) # --- end of __float__ (...) --- + def __add__ ( self, other ): + return self.total_count + int ( other ) + # --- end of __add__ (...) --- + + def __sub__ ( self, other ): + return self.total_count - int ( other ) + # --- end of __sub__ (...) --- + def reset ( self ): self.total_count = 0 self.underflow = False diff --git a/roverlay/stats/base.py b/roverlay/stats/base.py index b03092b..2f13e73 100644 --- a/roverlay/stats/base.py +++ b/roverlay/stats/base.py @@ -8,11 +8,13 @@ from . import abstract class RepoStats ( abstract.RoverlayStats ): - _MEMBERS = frozenset ({ 'pkg_count', }) + _MEMBERS = ( 'sync_time', 'queue_time', 'pkg_count', ) def __init__ ( self ): super ( RepoStats, self ).__init__() - self.pkg_count = abstract.DetailedCounter ( + self.sync_time = abstract.TimeStats ( 'sync_time' ) + self.queue_time = abstract.TimeStats ( 'queue_time' ) + self.pkg_count = abstract.DetailedCounter ( description="package count" ) # --- end of __init__ (...) --- @@ -26,7 +28,7 @@ class RepoStats ( abstract.RoverlayStats ): class DistmapStats ( abstract.RoverlayStats ): - _MEMBERS = frozenset ({ 'pkg_count', }) + _MEMBERS = ( 'pkg_count', ) def __init__ ( self ): super ( DistmapStats, self ).__init__() @@ -46,31 +48,56 @@ class DistmapStats ( abstract.RoverlayStats ): # --- end of DistmapStats --- -class OverlayCreationStats ( abstract.RoverlayStats ): +class OverlayCreationWorkerStats ( abstract.RoverlayStats ): - #_MEMBERS = frozenset({}) + _MEMBERS = ( 'pkg_processed', 'pkg_fail', 'pkg_success', ) + + def __init__ ( self ): + self.pkg_processed = abstract.Counter ( "processed" ) + self.pkg_fail = abstract.DetailedCounter ( "fail" ) + self.pkg_success = abstract.Counter ( "success" ) + # --- end of __init__ (...) --- + +# --- end of OverlayCreationWorkerStats --- + + +class OverlayCreationStats ( OverlayCreationWorkerStats ): + + DESCRIPTION = "overlay creation" + + _MEMBERS = ( + ( 'creation_time', 'pkg_queued', 'pkg_filtered', ) + + OverlayCreationWorkerStats._MEMBERS + ) def __init__ ( self ): super ( OverlayCreationStats, self ).__init__() + self.pkg_queued = abstract.Counter ( "queued" ) + self.pkg_filtered = abstract.Counter ( "filtered" ) + self.creation_time = abstract.TimeStats ( "ebuild creation" ) # --- end of __init__ (...) --- def get_relevant_package_count ( self ): - print ( "get_relevant_package_count(): method stub" ) - return 0 + return self.pkg_queued + #return self.pkg_queued - self.pkg_fail.get ( "empty_desc" ) # --- end of get_relevant_package_count (...) --- - def __str__ ( self ): - return "*** no overlay creation stats available ***" - # --- end of __str__ (...) --- + @classmethod + def get_new ( cls ): + return OverlayCreationWorkerStats() + # --- end of get_new (...) --- # --- end of OverlayCreationStats --- class OverlayStats ( abstract.RoverlayStats ): - _MEMBERS = frozenset ({ + DESCRIPTION = "overlay" + + _MEMBERS = ( + 'scan_time', 'write_time', 'ebuilds_scanned', 'ebuild_count', 'ebuilds_written', - }) + ) def __init__ ( self ): super ( OverlayStats, self ).__init__() @@ -81,6 +108,9 @@ class OverlayStats ( abstract.RoverlayStats ): self.ebuild_count = abstract.Counter ( "post" ) self.ebuilds_written = abstract.Counter ( "written" ) + + self.write_time = abstract.TimeStats ( "write_time" ) + self.scan_time = abstract.TimeStats ( "scan_time" ) # --- end of __init__ (...) --- # --- end of OverlayStats --- diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py index d3c318e..1b122f4 100644 --- a/roverlay/stats/collector.py +++ b/roverlay/stats/collector.py @@ -12,7 +12,7 @@ class StatsCollector ( abstract.RoverlayStatsBase ): _instance = None - _MEMBERS = [ 'repo', 'distmap', 'overlay_creation', 'overlay', ] + _MEMBERS = ( 'time', 'repo', 'distmap', 'overlay_creation', 'overlay', ) @classmethod def get_instance ( cls ): @@ -25,35 +25,50 @@ class StatsCollector ( abstract.RoverlayStatsBase ): # where unsuitable is e.g. "OS_Type not supported") # return abstract.SuccessRatio ( - num_ebuilds = self.overlay.ebuild_count.get ( "new" ), + num_ebuilds = self.overlay_creation.pkg_success, num_pkg = self.overlay_creation.get_relevant_package_count(), ) # --- end of get_success_ratio (...) --- def get_overall_success_ratio ( self ): # overall success ratio: - # all ebuilds / distmap file count + # "relevant ebuild count" / "guessed package count" + # ratio := / ( + ) # - # *Not* accurate as it includes imported ebuilds and assumes that - # each ebuild has one source file in the distmap. + # + # *Not* accurate as it includes imported ebuilds # (Still better than using the repo package count since that may # not include old package files) # - # !!! useless. distmap includes "successful" packages only - # return abstract.SuccessRatio ( num_ebuilds = self.overlay.ebuild_count, - num_pkg = self.distmap.pkg_count, + num_pkg = ( + self.overlay.ebuild_count + self.overlay_creation.pkg_fail + ), ) # --- end of get_overall_success_ratio (...) --- def __init__ ( self ): + self.time = abstract.TimeStats ( "misc time stats" ) self.distmap = base.DistmapStats() self.overlay = base.OverlayStats() self.overlay_creation = base.OverlayCreationStats() self.repo = base.RepoStats() # --- end of __init__ (...) --- + def gen_str ( self ): + yield "{success}, overall {osuccess}".format ( + success = self.get_success_ratio(), + osuccess = self.get_overall_success_ratio(), + ) + yield "" + + for s in super ( StatsCollector, self ).gen_str(): + yield s + yield "" + # --- end of gen_str (...) --- + + # --- end of StatsCollector --- static = StatsCollector()