[gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/

["André Erdmann" <dywi@mailerd.de>]

commit:     d1300fc92439e26b6941eec62ec888329835ee77
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 29 14:53:05 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 29 14:53:05 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d1300fc9

write stats to round-robin database (rrdtool)

This commit makes (/allows to/) stats data persistent by adding them to a db,
which can later be used to draw graphs etc.

---
 roverlay/stats/abstract.py                    |  37 +-----
 roverlay/stats/collector.py                   | 156 +++++---------------------
 roverlay/stats/dbcollector.py                 |  74 ++++++++++++
 roverlay/stats/rrd.py                         |  63 +++++++++++
 roverlay/stats/{collector.py => visualize.py} |  91 +++------------
 5 files changed, 189 insertions(+), 232 deletions(-)

diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
index 6d0575a..68728f4 100644
--- a/roverlay/stats/abstract.py
+++ b/roverlay/stats/abstract.py
@@ -9,34 +9,7 @@ from __future__ import division
 import collections
 import time
 
-class MethodNotImplemented ( NotImplementedError ):
-   def __init__ ( self, obj, method ):
-      super ( MethodNotImplemented, self ).__init__ (
-         "{n}.{f}()".format ( n=obj.__class__.__name__, f=method )
-      )
-   # --- end of __init__ (...) ---
-
-# --- end of MethodNotImplemented ---
-
-class StatsVisualizer ( object ):
-
-   def __init__ ( self, stats ):
-      super ( StatsVisualizer, self ).__init__()
-      self.stats = stats
-      self.lines = None
-
-      self.prepare()
-   # --- end of __init__ (...) ---
-
-   def prepare ( self ):
-      raise MethodNotImplemented ( self, 'prepare' )
-   # --- end of prepare (...) ---
-
-   def __str__ ( self ):
-      return '\n'.join ( self.lines )
-   # --- end of __str__ (...) ---
-
-# --- end of StatsVisualizer ---
+from roverlay.util.objects import MethodNotImplementedError
 
 
 class RoverlayStatsBase ( object ):
@@ -66,7 +39,7 @@ class RoverlayStatsBase ( object ):
          self.merge_members ( other, my_cls._MEMBERS )
 
       else:
-         raise MethodNotImplemented ( self, 'merge_with' )
+         raise MethodNotImplementedError ( self, 'merge_with' )
    # --- end of merge_with (...) ---
 
    def merge_members ( self, other, members ):
@@ -86,7 +59,7 @@ class RoverlayStatsBase ( object ):
             if int ( member ) != 0:
                return member
       else:
-         raise MethodNotImplemented ( self, 'has_nonzero' )
+         raise MethodNotImplementedError ( self, 'has_nonzero' )
    # --- end of has_nonzero (...) ---
 
    def reset_members ( self ):
@@ -98,7 +71,7 @@ class RoverlayStatsBase ( object ):
       if hasattr ( self.__class__, '_MEMBERS' ):
          self.reset_members()
       else:
-         raise MethodNotImplemented ( self, 'reset' )
+         raise MethodNotImplementedError ( self, 'reset' )
    # --- end of reset (...) ---
 
    def get_description_str ( self ):
@@ -122,7 +95,7 @@ class RoverlayStatsBase ( object ):
       if ret:
          return ret
       else:
-         raise MethodNotImplemented ( self, '__str__' )
+         raise MethodNotImplementedError ( self, '__str__' )
    # --- end of __str__ (...) ---
 
 # --- end of RoverlayStatsBase ---

diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py
index 3e3dee7..30b0e76 100644
--- a/roverlay/stats/collector.py
+++ b/roverlay/stats/collector.py
@@ -4,10 +4,15 @@
 # Distributed under the terms of the GNU General Public License;
 # either version 2 of the License, or (at your option) any later version.
 
-import collections
+import time
+
+import roverlay.config.static
 
 from . import abstract
 from . import base
+from . import dbcollector
+from . import visualize
+from . import rrd
 
 
 class StatsCollector ( abstract.RoverlayStatsBase ):
@@ -57,13 +62,29 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
    # --- end of get_net_gain (...) ---
 
    def __init__ ( self ):
+      super ( StatsCollector, self ).__init__()
+
       self.time             = abstract.TimeStats ( "misc time stats" )
       self.distmap          = base.DistmapStats()
       self.overlay          = base.OverlayStats()
       self.overlay_creation = base.OverlayCreationStats()
       self.repo             = base.RepoStats()
+      self.db_collector     = None
+      self._database        = None
    # --- end of __init__ (...) ---
 
+   def setup_database ( self, config=None ):
+      conf = (
+         config if config is not None else roverlay.config.static.access()
+      )
+
+      self.db_collector     = dbcollector.StatsDBCollector ( self )
+      self._database        = rrd.StatsDB (
+         conf.get_or_fail ( "RRD_DB.file" ), self.db_collector
+      )
+      self._database.create_if_missing()
+   # --- end of setup_database (...) ---
+
    def gen_str ( self ):
       yield "{success}, overall {osuccess}".format (
          success  = self.get_success_ratio(),
@@ -77,136 +98,17 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
    # --- end of gen_str (...) ---
 
    def get_creation_str ( self ):
-      return str ( CreationStatsVisualizer ( self ) )
+      return str ( visualize.CreationStatsVisualizer ( self ) )
    # --- end of to_creation_str (...) ---
 
-# --- end of StatsCollector ---
-
-
-class CreationStatsVisualizer ( abstract.StatsVisualizer ):
-
-   def prepare ( self ):
-      EMPTY_LINE = ""
-
-      pkg_count    = self.stats.repo.pkg_count
-      pkg_queued   = self.stats.overlay_creation.pkg_queued
-      pkg_fail     = self.stats.overlay_creation.pkg_fail
-      pkg_success  = self.stats.overlay_creation.pkg_success
-      ebuild_delta = self.stats.get_net_gain()
-      revbumps     = self.stats.overlay.revbump_count
-
-      max_number_len = min (
-         len ( str ( int ( k ) ) ) for k in (
-            pkg_queued, pkg_fail, pkg_success
-         )
-      )
-      max_number_len = min ( 5, max_number_len )
-
-      success_ratio         = self.stats.get_success_ratio()
-      overall_success_ratio = self.stats.get_overall_success_ratio()
-
+   def write_db ( self ):
+      self.db_collector.update()
+      self._database.update()
+      self._database.commit()
+   # --- end of write_db (...) ---
 
-      timestats = (
-         ( 'scan_overlay', self.stats.overlay.scan_time.get_total_str() ),
-         ( 'add_packages', self.stats.repo.queue_time.get_total_str() ),
-         (
-            'ebuild_creation',
-            self.stats.overlay_creation.creation_time.get_total_str()
-         ),
-         ( 'write_overlay',  self.stats.overlay.write_time.get_total_str() ),
-      )
-
-      try:
-         max_time_len = max ( len(k) for k, v in timestats if v is not None )
-      except ValueError:
-         # empty sequence -> no timestats
-         max_time_len = -1
-      else:
-         # necessary?
-         max_time_len = min ( 39, max_time_len )
-
-
-      # create lines
-      lines   = collections.deque()
-      unshift = lines.appendleft
-      append  = lines.append
-
-      numstats = lambda k, s: "{num:<{l}d} {s}".format (
-         num=int ( k ), s=s, l=max_number_len
-      )
-
-
-      append (
-         'success ratio {s_i:.2%} (overall {s_o:.2%})'.format (
-            s_i = success_ratio.get_ratio(),
-            s_o = overall_success_ratio.get_ratio()
-         )
-      )
-      append ( EMPTY_LINE )
-
-      append (
-         "{e:+d} ebuilds ({r:d} revbumps)".format (
-            e=ebuild_delta, r=int ( revbumps )
-         )
-      )
-      append ( EMPTY_LINE )
-
-      if int ( pkg_count ) != int ( pkg_queued ):
-         append ( numstats (
-            pkg_queued,
-            '/ {n:d} packages added to the ebuild creation queue'.format (
-               n=int ( pkg_count )
-            )
-         ) )
-      else:
-         append ( numstats (
-            pkg_queued, 'packages added to the ebuild creation queue'
-         ) )
-
-      append ( numstats (
-         pkg_success, 'packages passed ebuild creation'
-      ) )
-
-      append ( numstats (
-         pkg_fail, 'packages failed ebuild creation'
-      ) )
-
-      if pkg_fail.has_details() and int ( pkg_fail ) != 0:
-         append ( EMPTY_LINE )
-         append ( "Details for ebuild creation failure:" )
-         details = sorted (
-            ( ( k, int(v) ) for k, v in pkg_fail.iter_details() ),
-            key=lambda kv: kv[1]
-         )
-         dlen = len ( str ( max ( details, key=lambda kv: kv[1] ) [1] ) )
-
-         for key, value in details:
-            append ( "* {v:>{l}d}: {k}".format ( k=key, v=value, l=dlen ) )
-      # -- end if <have pkg_fail details>
-
-      if max_time_len > 0:
-         # or >= 0
-         append ( EMPTY_LINE )
-         for k, v in timestats:
-            if v is not None:
-               append (
-                  "time for {0:<{l}} : {1}".format ( k, v, l=max_time_len )
-               )
-      # -- end if timestats
-
-
-      append ( EMPTY_LINE )
-
-      # add header/footer line(s)
-      max_line_len = 2 + min ( 78, max ( len(s) for s in lines ) )
-      unshift (
-         "{0:-^{1}}\n".format ( " Overlay creation stats ", max_line_len )
-      )
-      append ( max_line_len * '-' )
+# --- end of StatsCollector ---
 
-      self.lines = lines
-   # --- end of gen_str (...) ---
-# --- end of CreationStatsVisualizer ---
 
 static = StatsCollector()
 StatsCollector._instance = static

diff --git a/roverlay/stats/dbcollector.py b/roverlay/stats/dbcollector.py
new file mode 100644
index 0000000..402e64d
--- /dev/null
+++ b/roverlay/stats/dbcollector.py
@@ -0,0 +1,74 @@
+# R overlay -- stats collection, prepare data for db storage
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import collections
+import weakref
+
+class StatsDBCollector ( object ):
+   VERSION = 0
+
+   # 'pc' := package count(er), 'ec' := ebuild _
+   NUMSTATS_KEYS = (
+      'pc_repo', 'pc_distmap', 'pc_filtered', 'pc_queued', 'pc_success',
+      'pc_fail', 'pc_fail_empty', 'pc_fail_dep', 'pc_fail_selfdep',
+      'pc_fail_err',
+      'ec_pre', 'ec_post', 'ec_written', 'ec_revbump',
+   )
+   NUMSTATS = collections.namedtuple (
+      "numstats", ' '.join ( NUMSTATS_KEYS )
+   )
+
+   TIMESTATS_KEYS = ()
+   TIMESTATS = collections.namedtuple (
+      "timestats", ' '.join ( TIMESTATS_KEYS )
+   )
+
+   def __init__ ( self, stats ):
+      super ( StatsDBCollector, self ).__init__()
+      self.stats            = weakref.ref ( stats )
+      self._collected_stats = None
+   # --- end of __init__ (...) ---
+
+   def update ( self ):
+      self._collected_stats = self.make_all()
+   # --- end of update (...) ---
+
+   def make_numstats ( self ):
+      stats     = self.stats()
+      ov_create = stats.overlay_creation
+      ov        = stats.overlay
+
+      return self.__class__.NUMSTATS (
+         pc_repo         = int ( stats.repo.pkg_count ),
+         pc_distmap      = int ( stats.distmap.pkg_count ),
+         pc_filtered     = int ( ov_create.pkg_filtered ),
+         pc_queued       = int ( ov_create.pkg_queued ),
+         pc_success      = int ( ov_create.pkg_success ),
+         pc_fail         = int ( ov_create.pkg_fail ),
+         pc_fail_empty   = ov_create.pkg_fail.get ( 'empty_desc' ),
+         pc_fail_dep     = ov_create.pkg_fail.get ( 'unresolved_deps' ),
+         pc_fail_selfdep = ov_create.pkg_fail.get ( 'bad_selfdeps' ),
+         pc_fail_err     = ov_create.pkg_fail.get ( 'exception' ),
+         ec_pre          = int ( ov.ebuilds_scanned ),
+         ec_post         = int ( ov.ebuild_count ),
+         ec_written      = int ( ov.ebuilds_written ),
+         ec_revbump      = int ( ov.revbump_count ),
+      )
+   # --- end of make_numstats (...) ---
+
+   def make_timestats ( self ):
+      return ()
+   # --- end of make_timestats (...) ---
+
+   def make_all ( self ):
+      return self.make_numstats() + self.make_timestats()
+   # --- end of make_all (...) ---
+
+   def get_all ( self ):
+      return self._collected_stats
+   # --- end of get_all (...) ---
+
+# --- end of StatsDBCollector #v0 ---

diff --git a/roverlay/stats/rrd.py b/roverlay/stats/rrd.py
new file mode 100644
index 0000000..66da434
--- /dev/null
+++ b/roverlay/stats/rrd.py
@@ -0,0 +1,63 @@
+# R overlay -- stats collection, rrd database (using rrdtool)
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+
+# NOT using rrdtool's python bindings as they're available for python 2 only
+
+import roverlay.db.rrdtool
+from roverlay.db.rrdtool import RRDVariable, RRDArchive
+
+
+class StatsDB ( roverlay.db.rrdtool.RRD ):
+
+   # default step
+   STEP = 300
+
+   def __init__ ( self, filepath, collector, step=None ):
+      # COULDFIX:
+      #  vars / RRA creation is only necessary when creating a new database
+      #
+      self.collector    = collector
+      self.rrd_vars     = self.make_vars()
+      self.rrd_archives = self.make_rra()
+      self.step         = step if step is not None else self.__class__.STEP
+      super ( StatsDB, self ).__init__ ( filepath )
+   # --- end of __init__ (...) ---
+
+   def _do_create ( self, filepath ):
+      return self._call_rrdtool (
+         (
+            'create', filepath,
+            '--start', str ( self.INIT_TIME ),
+            '--step', str ( self.step ),
+         ) + tuple (
+            v.get_key() for v in self.rrd_vars
+         ) + tuple (
+            v.get_key() for v in self.rrd_archives
+         )
+      )
+   # --- end of _do_create (...) ---
+
+   def update ( self ):
+      self.add ( self.collector.get_all() )
+   # --- end of update (...) ---
+
+   def make_vars ( self ):
+      return tuple (
+         RRDVariable ( k, 'DERIVE', val_max=0 )
+            for k in self.collector.NUMSTATS_KEYS
+      )
+   # --- end of make_vars (...) ---
+
+   def make_rra ( self ):
+      return (
+         RRDArchive.new_day   ( 'LAST',    0.7 ),
+         RRDArchive.new_week  ( 'AVERAGE', 0.7 ),
+         RRDArchive.new_month ( 'AVERAGE', 0.7 ),
+      )
+   # --- end of make_rra (...) ---
+
+# --- end of StatsDB ---

diff --git a/roverlay/stats/collector.py b/roverlay/stats/visualize.py
similarity index 61%
copy from roverlay/stats/collector.py
copy to roverlay/stats/visualize.py
index 3e3dee7..09ca48e 100644
--- a/roverlay/stats/collector.py
+++ b/roverlay/stats/visualize.py
@@ -1,4 +1,4 @@
-# R overlay -- stats collection, stats collector
+# R overlay -- stats collection, print stats
 # -*- coding: utf-8 -*-
 # Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
 # Distributed under the terms of the GNU General Public License;
@@ -6,84 +6,31 @@
 
 import collections
 
-from . import abstract
-from . import base
+import roverlay.util.objects
 
 
-class StatsCollector ( abstract.RoverlayStatsBase ):
+class StatsVisualizer ( object ):
 
-   _instance = None
+   def __init__ ( self, stats ):
+      super ( StatsVisualizer, self ).__init__()
+      self.stats = stats
+      self.lines = None
 
-   _MEMBERS  = ( 'time', 'repo', 'distmap', 'overlay_creation', 'overlay', )
-
-   @classmethod
-   def get_instance ( cls ):
-      return cls._instance
-   # --- end of instance (...) ---
-
-   def get_success_ratio ( self ):
-      # success ratio for "this" run:
-      #  new ebuilds / relevant package count (new packages - unsuitable,
-      #   where unsuitable is e.g. "OS_Type not supported")
-      #
-      return abstract.SuccessRatio (
-         num_ebuilds = self.overlay_creation.pkg_success,
-         num_pkg     = self.overlay_creation.get_relevant_package_count(),
-      )
-   # --- end of get_success_ratio (...) ---
-
-   def get_overall_success_ratio ( self ):
-      # overall success ratio:
-      #  "relevant ebuild count" / "guessed package count"
-      #  ratio := <all ebuilds> / (<all ebuilds> + <failed packages>)
-      #
-      #
-      #  *Not* accurate as it includes imported ebuilds
-      #  (Still better than using the repo package count since that may
-      #   not include old package files)
-      #
-      return abstract.SuccessRatio (
-         num_ebuilds = self.overlay.ebuild_count,
-         num_pkg     = (
-            self.overlay.ebuild_count + self.overlay_creation.pkg_fail
-         ),
-      )
-   # --- end of get_overall_success_ratio (...) ---
-
-   def get_net_gain ( self ):
-      return (
-         self.overlay.ebuild_count - self.overlay.ebuilds_scanned
-      )
-   # --- end of get_net_gain (...) ---
-
-   def __init__ ( self ):
-      self.time             = abstract.TimeStats ( "misc time stats" )
-      self.distmap          = base.DistmapStats()
-      self.overlay          = base.OverlayStats()
-      self.overlay_creation = base.OverlayCreationStats()
-      self.repo             = base.RepoStats()
+      self.prepare()
    # --- end of __init__ (...) ---
 
-   def gen_str ( self ):
-      yield "{success}, overall {osuccess}".format (
-         success  = self.get_success_ratio(),
-         osuccess = self.get_overall_success_ratio(),
-      )
-      yield ""
-
-      for s in super ( StatsCollector, self ).gen_str():
-         yield s
-         yield ""
-   # --- end of gen_str (...) ---
-
-   def get_creation_str ( self ):
-      return str ( CreationStatsVisualizer ( self ) )
-   # --- end of to_creation_str (...) ---
+   @roverlay.util.objects.not_implemented
+   def prepare ( self ):
+      pass
+   # --- end of prepare (...) ---
 
-# --- end of StatsCollector ---
+   def __str__ ( self ):
+      return '\n'.join ( self.lines )
+   # --- end of __str__ (...) ---
 
+# --- end of StatsVisualizer ---
 
-class CreationStatsVisualizer ( abstract.StatsVisualizer ):
+class CreationStatsVisualizer ( StatsVisualizer ):
 
    def prepare ( self ):
       EMPTY_LINE = ""
@@ -206,7 +153,5 @@ class CreationStatsVisualizer ( abstract.StatsVisualizer ):
 
       self.lines = lines
    # --- end of gen_str (...) ---
-# --- end of CreationStatsVisualizer ---
 
-static = StatsCollector()
-StatsCollector._instance = static
+# --- end of CreationStatsVisualizer ---