[gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/

["André Erdmann" <dywi@mailerd.de>]

commit:     a2424732d50d85bfb2bc094bee086af7a79ddd49
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 24 15:54:58 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 24 15:54:58 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a2424732

stats collection: data types

---
 roverlay/stats/abstract.py  | 209 ++++++++++++++++++++++++++++++++++++++++++++
 roverlay/stats/base.py      |  82 +++++++++++++++++
 roverlay/stats/collector.py |  60 +++++++++++++
 3 files changed, 351 insertions(+)

diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
new file mode 100644
index 0000000..b6a4ee9
--- /dev/null
+++ b/roverlay/stats/abstract.py
@@ -0,0 +1,209 @@
+# R overlay -- stats collection, data types
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import collections
+import time
+
+class RoverlayStatsBase ( object ):
+
+   @classmethod
+   def get_new ( cls ):
+      return cls()
+   # --- end of get_new (...) ---
+
+   def merge_with ( self, other ):
+      raise NotImplementedError()
+   # --- end of merge_with (...) ---
+
+   def _iter_members ( self, nofail=False ):
+      if not nofail or hasattr ( self, '_MEMBERS' ):
+         for member in self.__class__._MEMBERS:
+            yield getattr ( self, member )
+   # --- end of _iter_members (...) ---
+
+   def reset_members ( self ):
+      for member in self._iter_members():
+         member.reset()
+   # --- end of reset_members (...) ---
+
+   def reset ( self ):
+      if hasattr ( self.__class__, '_MEMBERS' ):
+         self.reset_members()
+      else:
+         raise NotImplementedError()
+   # --- end of reset (...) ---
+
+   def get_description_str ( self ):
+      return (
+         getattr ( self, 'description', None ) or
+         getattr ( self, 'DESCRIPTION', None )
+      )
+   # --- end of get_description_str (...) ---
+
+   def gen_str ( self ):
+      desc = self.get_description_str()
+      if desc:
+         yield desc
+
+      for member in self._iter_members( nofail=True ):
+         yield str ( member )
+   # --- end of gen_str (...) ---
+
+   def __str__ ( self ):
+      ret = '\n'.join ( self.gen_str() )
+      if ret:
+         return ret
+      else:
+         raise NotImplementedError()
+   # --- end of __str__ (...) ---
+
+# --- end of RoverlayStatsBase ---
+
+
+class RoverlayStats ( RoverlayStatsBase ):
+
+   def get_time ( self ):
+      return time.time()
+   # --- end of get_time (...) ---
+
+   def get_time_delta ( self, start, stop=None ):
+      t_stop = self.get_time() if stop is None else stop
+      return ( start - t_stop, t_stop )
+   # --- end of get_time_delta (...) ---
+
+# --- end of RoverlayStats ---
+
+
+class DetailedCounter ( RoverlayStatsBase ):
+   def __init__ ( self, description=None ):
+      super ( DetailedCounter, self ).__init__()
+      self.description     = description
+      self.total_count     = 0
+      self._detailed_count = collections.defaultdict ( int )
+      self.underflow       = False
+   # --- end of __init__ (...) ---
+
+   def __getitem__ ( self, key ):
+      if key in self._detailed_count:
+         return self._detailed_count [key]
+      else:
+         raise KeyError ( key )
+   # --- end of __getitem__ (...) ---
+
+   def __int__ ( self ):
+      return self.total_count
+   # --- end of __int__ (...) ---
+
+   def __float__ ( self ):
+      return float ( self.total_count )
+   # --- end of __float__ (...) ---
+
+   def get ( self, key=None, fallback=0 ):
+      if key is None:
+         return self.total_count
+      elif key in self._detailed_count:
+         return self._detailed_count [key]
+      else:
+         return fallback
+   # --- end of get (...) ---
+
+   def reset ( self ):
+      self.total_count = 0
+      self.underflow   = False
+      self._detailed_count.clear()
+   # --- end of reset (...) ---
+
+   def inc_details_v ( self, details ):
+      for k in details:
+         self._detailed_count [k] += 1
+   # --- end of inc_details_v (...) ---
+
+   def inc ( self, *details ):
+      self.total_count += 1
+      self.inc_details_v ( details )
+   # --- end of inc (...) ---
+
+   def inc_details ( self, *details ):
+      self.inc_details_v ( details )
+   # --- end of inc_details (...) ---
+
+   def dec_details_v ( self, details ):
+      # do not check / fail if self._detailed_count [k] > 0
+      #  simply keep the negative value
+      for k in details:
+         self._detailed_count [k] -= 1
+         if self._detailed_count [k] < 0:
+            self.underflow = True
+   # --- end of dec_details_v (...) ---
+
+   def dec ( self, *details ):
+      self.total_count -= 1
+      if self.total_count < 0:
+         self.underflow = True
+      self.dec_details_v ( details )
+   # --- end of dec (...) ---
+
+   def dec_details ( self, *details ):
+      self.dec_details_v ( Details )
+   # --- end of dec_details (...) ---
+
+   def merge_with ( self, other ):
+      self.total_count += other.totalcount
+      for key, value in other._detailed_count.items():
+         self._detailed_count [key] += value
+
+      if other.underflow:
+         self.underflow = True
+   # --- end of merge_with (...) ---
+
+   def gen_str ( self ):
+      desc = self.get_description_str()
+      if desc:
+         yield desc
+
+      yield "total: {:d}".format ( self.total_count )
+      for key, value in self._detailed_count.items():
+         yield "* {k}: {v:d}".format ( k=key, v=value )
+
+      if self.underflow:
+         yield "*** underflow detected ***"
+   # --- end of gen_str (...) ---
+
+# --- end of DetailedCounter ---
+
+
+class SuccessRatio ( object ):
+
+   def __init__ ( self, num_ebuilds, num_pkg ):
+      super ( SuccessRatio, self ).__init__()
+      self.ebuild_count = int ( num_ebuilds )
+      self.pkg_count    = int ( num_pkg )
+   # --- end of __init__ (...) ---
+
+   def get_ratio ( self ):
+      if self.pkg_count == self.ebuild_count:
+         # 0/0, ..., K/K: "success" := 1.0
+         return 1.0
+      elif self.pkg_count < 1:
+         # K/0: bad ratio, use fixed value
+         return -1.0
+      else:
+         return float ( self.ebuild_count ) / float ( self.pkg_count )
+   # --- end of get_ratio (...) ---
+
+   def __float__ ( self ):
+      return self.get_ratio()
+   # --- end of __float__ (...) ---
+
+   def __int__ ( self ):
+      return int ( self.get_ratio() )
+   # --- end of __int__ (...) ---
+
+   def __str__ ( self ):
+      return "{:.3%}".format ( self.get_ratio() )
+   # --- end of __str__ (...) ---
+
+# --- end of SuccessRatio ---

diff --git a/roverlay/stats/base.py b/roverlay/stats/base.py
new file mode 100644
index 0000000..6e37581
--- /dev/null
+++ b/roverlay/stats/base.py
@@ -0,0 +1,82 @@
+# R overlay -- stats collection, data types
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+from . import abstract
+
+class RepoStats ( abstract.RoverlayStats ):
+
+   _MEMBERS = frozenset ({ 'pkg_count', })
+
+   def __init__ ( self ):
+      super ( RepoStats, self ).__init__()
+      self.pkg_count = abstract.DetailedCounter (
+         description="package count"
+      )
+   # --- end of __init__ (...) ---
+
+   def package_file_found ( self, repo ):
+      self.pkg_count.inc ( repo.name )
+   # --- end of add_package (...) ---
+
+# --- end of RepoStats ---
+
+
+class DistmapStats ( abstract.RoverlayStats ):
+
+   _MEMBERS = frozenset ({ 'pkg_count', })
+
+   def __init__ ( self ):
+      super ( DistmapStats, self ).__init__()
+      self.pkg_count = abstract.DetailedCounter (
+         description="distmap package count"
+      )
+   # --- end of __init__ (...) ---
+
+   def file_added ( self, *origin ):
+      self.pkg_count.inc ( *origin )
+   # --- end of file_added (...) ---
+
+   def file_removed ( self, *origin ):
+      self.pkg_count.dec ( *origin )
+   # --- end of file_removed (...) ---
+
+# --- end of DistmapStats ---
+
+
+class OverlayCreationStats ( abstract.RoverlayStats ):
+
+   #_MEMBERS = frozenset({})
+
+   def __init__ ( self ):
+      super ( OverlayCreationStats, self ).__init__()
+   # --- end of __init__ (...) ---
+
+   def get_relevant_package_count ( self ):
+      print ( "get_relevant_package_count(): method stub" )
+      return 0
+   # --- end of get_relevant_package_count (...) ---
+
+# --- end of OverlayCreationStats ---
+
+
+class OverlayStats ( abstract.RoverlayStats ):
+
+   _MEMBERS = frozenset({ 'ebuild_count', })
+
+   def __init__ ( self ):
+      super ( OverlayStats, self ).__init__()
+      # ebuild_count counts *physical* ebuilds
+      #  number of created ebuilds is part of overlay creation stats
+      self.ebuild_count = abstract.DetailedCounter (
+         description="ebuild count"
+      )
+   # --- end of __init__ (...) ---
+
+   def ebuild_added ( self, origin ):
+      self.ebuild_count.inc ( origin )
+   # --- end of ebuild_added (...) ---
+
+# --- end of OverlayStats ---

diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py
new file mode 100644
index 0000000..fc48d2a
--- /dev/null
+++ b/roverlay/stats/collector.py
@@ -0,0 +1,60 @@
+# R overlay -- stats collection, stats collector
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+from __future__ import division
+
+from . import abstract
+from . import base
+
+
+class StatsCollector ( abstract.RoverlayStatsBase ):
+
+   _instance = None
+
+   _MEMBERS  = frozenset ({ 'repo', 'overlay', 'overlay_creation', })
+
+   @classmethod
+   def get_instance ( cls ):
+      return cls._instance
+   # --- end of instance (...) ---
+
+   def get_success_ratio ( self ):
+      # success ratio for "this" run:
+      #  new ebuilds / relevant package count (new packages - unsuitable,
+      #   where unsuitable is e.g. "OS_Type not supported")
+      #
+      return abstract.SuccessRatio (
+         num_ebuilds = self.overlay.ebuild_count.get ( "new" ),
+         num_pkg     = self.overlay_creation.get_relevant_package_count(),
+      )
+   # --- end of get_success_ratio (...) ---
+
+   def get_overall_success_ratio ( self ):
+      # overall success ratio:
+      #  all ebuilds / distmap file count
+      #
+      #  *Not* accurate as it includes imported ebuilds and assumes that
+      #  each ebuild has one source file in the distmap.
+      #  (Still better than using the repo package count since that may
+      #   not include old package files)
+      #
+      return abstract.SuccessRatio (
+         num_ebuilds = self.overlay.ebuild_count,
+         num_pkg     = self.distmap.pkg_count,
+      )
+   # --- end of get_overall_success_ratio (...) ---
+
+   def __init__ ( self ):
+      self.distmap          = base.DistmapStats()
+      self.overlay          = base.OverlayStats()
+      self.overlay_creation = base.OverlayCreationStats()
+      self.repo             = base.RepoStats()
+   # --- end of __init__ (...) ---
+
+# --- end of StatsCollector ---
+
+static = StatsCollector()
+StatsCollector._instance = static