From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 8D8F01381F3 for ; Mon, 29 Jul 2013 14:57:31 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 81C8BE0ABB; Mon, 29 Jul 2013 14:57:04 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id D733AE0ABB for ; Mon, 29 Jul 2013 14:56:58 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id B07FD33DAC9 for ; Mon, 29 Jul 2013 14:56:57 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id D492BE5467 for ; Mon, 29 Jul 2013 14:56:54 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1375109585.d1300fc92439e26b6941eec62ec888329835ee77.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/stats/abstract.py roverlay/stats/collector.py roverlay/stats/dbcollector.py roverlay/stats/rrd.py roverlay/stats/visualize.py X-VCS-Directories: roverlay/stats/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: d1300fc92439e26b6941eec62ec888329835ee77 X-VCS-Branch: master Date: Mon, 29 Jul 2013 14:56:54 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 03058538-1919-47b6-932c-ab6c90f6935a X-Archives-Hash: 85ab17ead1e7948a1b496f1d5bb68afc commit: d1300fc92439e26b6941eec62ec888329835ee77 Author: André Erdmann mailerd de> AuthorDate: Mon Jul 29 14:53:05 2013 +0000 Commit: André Erdmann mailerd de> CommitDate: Mon Jul 29 14:53:05 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d1300fc9 write stats to round-robin database (rrdtool) This commit makes (/allows to/) stats data persistent by adding them to a db, which can later be used to draw graphs etc. --- roverlay/stats/abstract.py | 37 +----- roverlay/stats/collector.py | 156 +++++--------------------- roverlay/stats/dbcollector.py | 74 ++++++++++++ roverlay/stats/rrd.py | 63 +++++++++++ roverlay/stats/{collector.py => visualize.py} | 91 +++------------ 5 files changed, 189 insertions(+), 232 deletions(-) diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py index 6d0575a..68728f4 100644 --- a/roverlay/stats/abstract.py +++ b/roverlay/stats/abstract.py @@ -9,34 +9,7 @@ from __future__ import division import collections import time -class MethodNotImplemented ( NotImplementedError ): - def __init__ ( self, obj, method ): - super ( MethodNotImplemented, self ).__init__ ( - "{n}.{f}()".format ( n=obj.__class__.__name__, f=method ) - ) - # --- end of __init__ (...) --- - -# --- end of MethodNotImplemented --- - -class StatsVisualizer ( object ): - - def __init__ ( self, stats ): - super ( StatsVisualizer, self ).__init__() - self.stats = stats - self.lines = None - - self.prepare() - # --- end of __init__ (...) --- - - def prepare ( self ): - raise MethodNotImplemented ( self, 'prepare' ) - # --- end of prepare (...) --- - - def __str__ ( self ): - return '\n'.join ( self.lines ) - # --- end of __str__ (...) --- - -# --- end of StatsVisualizer --- +from roverlay.util.objects import MethodNotImplementedError class RoverlayStatsBase ( object ): @@ -66,7 +39,7 @@ class RoverlayStatsBase ( object ): self.merge_members ( other, my_cls._MEMBERS ) else: - raise MethodNotImplemented ( self, 'merge_with' ) + raise MethodNotImplementedError ( self, 'merge_with' ) # --- end of merge_with (...) --- def merge_members ( self, other, members ): @@ -86,7 +59,7 @@ class RoverlayStatsBase ( object ): if int ( member ) != 0: return member else: - raise MethodNotImplemented ( self, 'has_nonzero' ) + raise MethodNotImplementedError ( self, 'has_nonzero' ) # --- end of has_nonzero (...) --- def reset_members ( self ): @@ -98,7 +71,7 @@ class RoverlayStatsBase ( object ): if hasattr ( self.__class__, '_MEMBERS' ): self.reset_members() else: - raise MethodNotImplemented ( self, 'reset' ) + raise MethodNotImplementedError ( self, 'reset' ) # --- end of reset (...) --- def get_description_str ( self ): @@ -122,7 +95,7 @@ class RoverlayStatsBase ( object ): if ret: return ret else: - raise MethodNotImplemented ( self, '__str__' ) + raise MethodNotImplementedError ( self, '__str__' ) # --- end of __str__ (...) --- # --- end of RoverlayStatsBase --- diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py index 3e3dee7..30b0e76 100644 --- a/roverlay/stats/collector.py +++ b/roverlay/stats/collector.py @@ -4,10 +4,15 @@ # Distributed under the terms of the GNU General Public License; # either version 2 of the License, or (at your option) any later version. -import collections +import time + +import roverlay.config.static from . import abstract from . import base +from . import dbcollector +from . import visualize +from . import rrd class StatsCollector ( abstract.RoverlayStatsBase ): @@ -57,13 +62,29 @@ class StatsCollector ( abstract.RoverlayStatsBase ): # --- end of get_net_gain (...) --- def __init__ ( self ): + super ( StatsCollector, self ).__init__() + self.time = abstract.TimeStats ( "misc time stats" ) self.distmap = base.DistmapStats() self.overlay = base.OverlayStats() self.overlay_creation = base.OverlayCreationStats() self.repo = base.RepoStats() + self.db_collector = None + self._database = None # --- end of __init__ (...) --- + def setup_database ( self, config=None ): + conf = ( + config if config is not None else roverlay.config.static.access() + ) + + self.db_collector = dbcollector.StatsDBCollector ( self ) + self._database = rrd.StatsDB ( + conf.get_or_fail ( "RRD_DB.file" ), self.db_collector + ) + self._database.create_if_missing() + # --- end of setup_database (...) --- + def gen_str ( self ): yield "{success}, overall {osuccess}".format ( success = self.get_success_ratio(), @@ -77,136 +98,17 @@ class StatsCollector ( abstract.RoverlayStatsBase ): # --- end of gen_str (...) --- def get_creation_str ( self ): - return str ( CreationStatsVisualizer ( self ) ) + return str ( visualize.CreationStatsVisualizer ( self ) ) # --- end of to_creation_str (...) --- -# --- end of StatsCollector --- - - -class CreationStatsVisualizer ( abstract.StatsVisualizer ): - - def prepare ( self ): - EMPTY_LINE = "" - - pkg_count = self.stats.repo.pkg_count - pkg_queued = self.stats.overlay_creation.pkg_queued - pkg_fail = self.stats.overlay_creation.pkg_fail - pkg_success = self.stats.overlay_creation.pkg_success - ebuild_delta = self.stats.get_net_gain() - revbumps = self.stats.overlay.revbump_count - - max_number_len = min ( - len ( str ( int ( k ) ) ) for k in ( - pkg_queued, pkg_fail, pkg_success - ) - ) - max_number_len = min ( 5, max_number_len ) - - success_ratio = self.stats.get_success_ratio() - overall_success_ratio = self.stats.get_overall_success_ratio() - + def write_db ( self ): + self.db_collector.update() + self._database.update() + self._database.commit() + # --- end of write_db (...) --- - timestats = ( - ( 'scan_overlay', self.stats.overlay.scan_time.get_total_str() ), - ( 'add_packages', self.stats.repo.queue_time.get_total_str() ), - ( - 'ebuild_creation', - self.stats.overlay_creation.creation_time.get_total_str() - ), - ( 'write_overlay', self.stats.overlay.write_time.get_total_str() ), - ) - - try: - max_time_len = max ( len(k) for k, v in timestats if v is not None ) - except ValueError: - # empty sequence -> no timestats - max_time_len = -1 - else: - # necessary? - max_time_len = min ( 39, max_time_len ) - - - # create lines - lines = collections.deque() - unshift = lines.appendleft - append = lines.append - - numstats = lambda k, s: "{num:<{l}d} {s}".format ( - num=int ( k ), s=s, l=max_number_len - ) - - - append ( - 'success ratio {s_i:.2%} (overall {s_o:.2%})'.format ( - s_i = success_ratio.get_ratio(), - s_o = overall_success_ratio.get_ratio() - ) - ) - append ( EMPTY_LINE ) - - append ( - "{e:+d} ebuilds ({r:d} revbumps)".format ( - e=ebuild_delta, r=int ( revbumps ) - ) - ) - append ( EMPTY_LINE ) - - if int ( pkg_count ) != int ( pkg_queued ): - append ( numstats ( - pkg_queued, - '/ {n:d} packages added to the ebuild creation queue'.format ( - n=int ( pkg_count ) - ) - ) ) - else: - append ( numstats ( - pkg_queued, 'packages added to the ebuild creation queue' - ) ) - - append ( numstats ( - pkg_success, 'packages passed ebuild creation' - ) ) - - append ( numstats ( - pkg_fail, 'packages failed ebuild creation' - ) ) - - if pkg_fail.has_details() and int ( pkg_fail ) != 0: - append ( EMPTY_LINE ) - append ( "Details for ebuild creation failure:" ) - details = sorted ( - ( ( k, int(v) ) for k, v in pkg_fail.iter_details() ), - key=lambda kv: kv[1] - ) - dlen = len ( str ( max ( details, key=lambda kv: kv[1] ) [1] ) ) - - for key, value in details: - append ( "* {v:>{l}d}: {k}".format ( k=key, v=value, l=dlen ) ) - # -- end if - - if max_time_len > 0: - # or >= 0 - append ( EMPTY_LINE ) - for k, v in timestats: - if v is not None: - append ( - "time for {0:<{l}} : {1}".format ( k, v, l=max_time_len ) - ) - # -- end if timestats - - - append ( EMPTY_LINE ) - - # add header/footer line(s) - max_line_len = 2 + min ( 78, max ( len(s) for s in lines ) ) - unshift ( - "{0:-^{1}}\n".format ( " Overlay creation stats ", max_line_len ) - ) - append ( max_line_len * '-' ) +# --- end of StatsCollector --- - self.lines = lines - # --- end of gen_str (...) --- -# --- end of CreationStatsVisualizer --- static = StatsCollector() StatsCollector._instance = static diff --git a/roverlay/stats/dbcollector.py b/roverlay/stats/dbcollector.py new file mode 100644 index 0000000..402e64d --- /dev/null +++ b/roverlay/stats/dbcollector.py @@ -0,0 +1,74 @@ +# R overlay -- stats collection, prepare data for db storage +# -*- coding: utf-8 -*- +# Copyright (C) 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + +import collections +import weakref + +class StatsDBCollector ( object ): + VERSION = 0 + + # 'pc' := package count(er), 'ec' := ebuild _ + NUMSTATS_KEYS = ( + 'pc_repo', 'pc_distmap', 'pc_filtered', 'pc_queued', 'pc_success', + 'pc_fail', 'pc_fail_empty', 'pc_fail_dep', 'pc_fail_selfdep', + 'pc_fail_err', + 'ec_pre', 'ec_post', 'ec_written', 'ec_revbump', + ) + NUMSTATS = collections.namedtuple ( + "numstats", ' '.join ( NUMSTATS_KEYS ) + ) + + TIMESTATS_KEYS = () + TIMESTATS = collections.namedtuple ( + "timestats", ' '.join ( TIMESTATS_KEYS ) + ) + + def __init__ ( self, stats ): + super ( StatsDBCollector, self ).__init__() + self.stats = weakref.ref ( stats ) + self._collected_stats = None + # --- end of __init__ (...) --- + + def update ( self ): + self._collected_stats = self.make_all() + # --- end of update (...) --- + + def make_numstats ( self ): + stats = self.stats() + ov_create = stats.overlay_creation + ov = stats.overlay + + return self.__class__.NUMSTATS ( + pc_repo = int ( stats.repo.pkg_count ), + pc_distmap = int ( stats.distmap.pkg_count ), + pc_filtered = int ( ov_create.pkg_filtered ), + pc_queued = int ( ov_create.pkg_queued ), + pc_success = int ( ov_create.pkg_success ), + pc_fail = int ( ov_create.pkg_fail ), + pc_fail_empty = ov_create.pkg_fail.get ( 'empty_desc' ), + pc_fail_dep = ov_create.pkg_fail.get ( 'unresolved_deps' ), + pc_fail_selfdep = ov_create.pkg_fail.get ( 'bad_selfdeps' ), + pc_fail_err = ov_create.pkg_fail.get ( 'exception' ), + ec_pre = int ( ov.ebuilds_scanned ), + ec_post = int ( ov.ebuild_count ), + ec_written = int ( ov.ebuilds_written ), + ec_revbump = int ( ov.revbump_count ), + ) + # --- end of make_numstats (...) --- + + def make_timestats ( self ): + return () + # --- end of make_timestats (...) --- + + def make_all ( self ): + return self.make_numstats() + self.make_timestats() + # --- end of make_all (...) --- + + def get_all ( self ): + return self._collected_stats + # --- end of get_all (...) --- + +# --- end of StatsDBCollector #v0 --- diff --git a/roverlay/stats/rrd.py b/roverlay/stats/rrd.py new file mode 100644 index 0000000..66da434 --- /dev/null +++ b/roverlay/stats/rrd.py @@ -0,0 +1,63 @@ +# R overlay -- stats collection, rrd database (using rrdtool) +# -*- coding: utf-8 -*- +# Copyright (C) 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + + +# NOT using rrdtool's python bindings as they're available for python 2 only + +import roverlay.db.rrdtool +from roverlay.db.rrdtool import RRDVariable, RRDArchive + + +class StatsDB ( roverlay.db.rrdtool.RRD ): + + # default step + STEP = 300 + + def __init__ ( self, filepath, collector, step=None ): + # COULDFIX: + # vars / RRA creation is only necessary when creating a new database + # + self.collector = collector + self.rrd_vars = self.make_vars() + self.rrd_archives = self.make_rra() + self.step = step if step is not None else self.__class__.STEP + super ( StatsDB, self ).__init__ ( filepath ) + # --- end of __init__ (...) --- + + def _do_create ( self, filepath ): + return self._call_rrdtool ( + ( + 'create', filepath, + '--start', str ( self.INIT_TIME ), + '--step', str ( self.step ), + ) + tuple ( + v.get_key() for v in self.rrd_vars + ) + tuple ( + v.get_key() for v in self.rrd_archives + ) + ) + # --- end of _do_create (...) --- + + def update ( self ): + self.add ( self.collector.get_all() ) + # --- end of update (...) --- + + def make_vars ( self ): + return tuple ( + RRDVariable ( k, 'DERIVE', val_max=0 ) + for k in self.collector.NUMSTATS_KEYS + ) + # --- end of make_vars (...) --- + + def make_rra ( self ): + return ( + RRDArchive.new_day ( 'LAST', 0.7 ), + RRDArchive.new_week ( 'AVERAGE', 0.7 ), + RRDArchive.new_month ( 'AVERAGE', 0.7 ), + ) + # --- end of make_rra (...) --- + +# --- end of StatsDB --- diff --git a/roverlay/stats/collector.py b/roverlay/stats/visualize.py similarity index 61% copy from roverlay/stats/collector.py copy to roverlay/stats/visualize.py index 3e3dee7..09ca48e 100644 --- a/roverlay/stats/collector.py +++ b/roverlay/stats/visualize.py @@ -1,4 +1,4 @@ -# R overlay -- stats collection, stats collector +# R overlay -- stats collection, print stats # -*- coding: utf-8 -*- # Copyright (C) 2013 André Erdmann # Distributed under the terms of the GNU General Public License; @@ -6,84 +6,31 @@ import collections -from . import abstract -from . import base +import roverlay.util.objects -class StatsCollector ( abstract.RoverlayStatsBase ): +class StatsVisualizer ( object ): - _instance = None + def __init__ ( self, stats ): + super ( StatsVisualizer, self ).__init__() + self.stats = stats + self.lines = None - _MEMBERS = ( 'time', 'repo', 'distmap', 'overlay_creation', 'overlay', ) - - @classmethod - def get_instance ( cls ): - return cls._instance - # --- end of instance (...) --- - - def get_success_ratio ( self ): - # success ratio for "this" run: - # new ebuilds / relevant package count (new packages - unsuitable, - # where unsuitable is e.g. "OS_Type not supported") - # - return abstract.SuccessRatio ( - num_ebuilds = self.overlay_creation.pkg_success, - num_pkg = self.overlay_creation.get_relevant_package_count(), - ) - # --- end of get_success_ratio (...) --- - - def get_overall_success_ratio ( self ): - # overall success ratio: - # "relevant ebuild count" / "guessed package count" - # ratio := / ( + ) - # - # - # *Not* accurate as it includes imported ebuilds - # (Still better than using the repo package count since that may - # not include old package files) - # - return abstract.SuccessRatio ( - num_ebuilds = self.overlay.ebuild_count, - num_pkg = ( - self.overlay.ebuild_count + self.overlay_creation.pkg_fail - ), - ) - # --- end of get_overall_success_ratio (...) --- - - def get_net_gain ( self ): - return ( - self.overlay.ebuild_count - self.overlay.ebuilds_scanned - ) - # --- end of get_net_gain (...) --- - - def __init__ ( self ): - self.time = abstract.TimeStats ( "misc time stats" ) - self.distmap = base.DistmapStats() - self.overlay = base.OverlayStats() - self.overlay_creation = base.OverlayCreationStats() - self.repo = base.RepoStats() + self.prepare() # --- end of __init__ (...) --- - def gen_str ( self ): - yield "{success}, overall {osuccess}".format ( - success = self.get_success_ratio(), - osuccess = self.get_overall_success_ratio(), - ) - yield "" - - for s in super ( StatsCollector, self ).gen_str(): - yield s - yield "" - # --- end of gen_str (...) --- - - def get_creation_str ( self ): - return str ( CreationStatsVisualizer ( self ) ) - # --- end of to_creation_str (...) --- + @roverlay.util.objects.not_implemented + def prepare ( self ): + pass + # --- end of prepare (...) --- -# --- end of StatsCollector --- + def __str__ ( self ): + return '\n'.join ( self.lines ) + # --- end of __str__ (...) --- +# --- end of StatsVisualizer --- -class CreationStatsVisualizer ( abstract.StatsVisualizer ): +class CreationStatsVisualizer ( StatsVisualizer ): def prepare ( self ): EMPTY_LINE = "" @@ -206,7 +153,5 @@ class CreationStatsVisualizer ( abstract.StatsVisualizer ): self.lines = lines # --- end of gen_str (...) --- -# --- end of CreationStatsVisualizer --- -static = StatsCollector() -StatsCollector._instance = static +# --- end of CreationStatsVisualizer ---