* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-07-25 14:28 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-07-25 14:28 UTC (permalink / raw
To: gentoo-commits
commit: 80380eae890dfe4411b61991f942c54f02cf4f15
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 25 14:28:37 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 25 14:28:37 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=80380eae
roverlay/stats/__init__.py
---
roverlay/stats/__init__.py | 0
1 file changed, 0 insertions(+), 0 deletions(-)
diff --git a/roverlay/stats/__init__.py b/roverlay/stats/__init__.py
new file mode 100644
index 0000000..e69de29
^ permalink raw reply [flat|nested] 14+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2014-02-21 17:36 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2014-02-21 17:36 UTC (permalink / raw
To: gentoo-commits
commit: f6cd141477e64a7500366206f4cb098f3a249256
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Feb 20 21:18:14 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Feb 20 21:18:14 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f6cd1414
roverlay/stats, timestats: add reset() method
Fixes a bug when using the remote console ("./bin/debug/console remote") where
the second query_packages command fails due to reset() missing for timestats
objects.
---
roverlay/stats/abstract.py | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
index 5ac87bc..c2ecace 100644
--- a/roverlay/stats/abstract.py
+++ b/roverlay/stats/abstract.py
@@ -120,16 +120,26 @@ class TimeStatsItem ( RoverlayStatsBase ):
# doc TODO: note somewhere that those timestats are just approximate
# values
- def __init__ ( self, t_begin=None, t_end=None, description=None ):
- super ( TimeStatsItem, self ).__init__ ( description=description )
+ def _setup_time_stats_item ( self, t_begin, t_end ):
self.time_begin = t_begin if t_begin is not None else time.time()
self.time_end = t_end
+ # --- end of _setup_time_stats_item (...) ---
+
+ def __init__ ( self, t_begin=None, t_end=None, description=None ):
+ super ( TimeStatsItem, self ).__init__ ( description=description )
+ self.time_begin = None
+ self.time_end = None
+ self._setup_time_stats_item ( t_begin, t_end )
# --- end of __init__ (...) ---
def end ( self, t_end=None ):
self.time_end = time.time() if t_end is None else t_end
# --- end of end (...) ---
+ def reset ( self ):
+ self._setup_time_stats_item ( None, None )
+ # --- end of reset (...) ---
+
def get_delta ( self ):
if self.time_begin is None:
return -1.0
@@ -153,6 +163,11 @@ class TimeStats ( RoverlayStats ):
self._timestats = collections.OrderedDict()
# --- end of __init__ (...) ---
+ def reset ( self ):
+ for tstat in self._timestats.values():
+ tstat.reset()
+ # --- end of reset (...) ---
+
def has_changes ( self ):
return False
# --- end of has_changes (...) ---
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-09-05 10:24 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-09-05 10:24 UTC (permalink / raw
To: gentoo-commits
commit: 01bcf594038714fcd576a9ef2357d16e40ac2e3d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 10:13:51 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 10:13:51 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=01bcf594
dbcollector: fix as_dict
---
roverlay/stats/dbcollector.py | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/roverlay/stats/dbcollector.py b/roverlay/stats/dbcollector.py
index 788110d..2b30aec 100644
--- a/roverlay/stats/dbcollector.py
+++ b/roverlay/stats/dbcollector.py
@@ -9,6 +9,16 @@ import weakref
from . import rating
+def get_dict ( k ):
+ if k:
+ try:
+ return k._asdict()
+ except AttributeError:
+ return dict ( k )
+ else:
+ return dict()
+# --- end of get_dict (...) ---
+
class StatsDBCollector ( object ):
VERSION = 0
@@ -65,11 +75,12 @@ class StatsDBCollector ( object ):
# --- end of make_timestats (...) ---
def get_numstats ( self, as_dict=False ):
- return self._numstats._as_dict() if as_dict else self._numstats
+ #dict(zip(<rating~>.keys(),<>))
+ return get_dict ( self._numstats ) if as_dict else self._numstats
# --- end of get_numstats (...) ---
def get_timestats ( self, as_dict=False ):
- return self._timestats._as_dict() if as_dict else self._timestats
+ return get_dict ( self._timestats ) if as_dict else self._timestats
# --- end of get_timestats (...) ---
def get_all ( self ):
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-09-05 9:25 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-09-05 9:25 UTC (permalink / raw
To: gentoo-commits
commit: 149d64984da9534e73c4e805b3892a2fc925bad4
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 09:07:01 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 09:07:01 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=149d6498
dbcollector: fix typo
---
roverlay/stats/dbcollector.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/stats/dbcollector.py b/roverlay/stats/dbcollector.py
index 4fad61c..788110d 100644
--- a/roverlay/stats/dbcollector.py
+++ b/roverlay/stats/dbcollector.py
@@ -65,7 +65,7 @@ class StatsDBCollector ( object ):
# --- end of make_timestats (...) ---
def get_numstats ( self, as_dict=False ):
- return self._numstats._as_dict() if as_dict self._numstats
+ return self._numstats._as_dict() if as_dict else self._numstats
# --- end of get_numstats (...) ---
def get_timestats ( self, as_dict=False ):
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-09-05 9:25 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-09-05 9:25 UTC (permalink / raw
To: gentoo-commits
commit: 13cdf0835c8a8d2c190e100288d385dd9cad6d54
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 09:04:03 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 09:04:03 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=13cdf083
dbcollector: collect time/numstats separately
---
roverlay/stats/dbcollector.py | 20 +++++++++++++-------
1 file changed, 13 insertions(+), 7 deletions(-)
diff --git a/roverlay/stats/dbcollector.py b/roverlay/stats/dbcollector.py
index 2961aec..4fad61c 100644
--- a/roverlay/stats/dbcollector.py
+++ b/roverlay/stats/dbcollector.py
@@ -25,12 +25,14 @@ class StatsDBCollector ( object ):
def __init__ ( self, stats ):
super ( StatsDBCollector, self ).__init__()
- self.stats = weakref.ref ( stats )
- self._collected_stats = None
+ self.stats = weakref.ref ( stats )
+ self._numstats = None
+ self._timestats = None
# --- end of __init__ (...) ---
def update ( self ):
- self._collected_stats = self.make_all()
+ self._numstats = self.make_numstats()
+ self._timestats = self.make_timestats()
# --- end of update (...) ---
def make_numstats ( self ):
@@ -62,12 +64,16 @@ class StatsDBCollector ( object ):
return ()
# --- end of make_timestats (...) ---
- def make_all ( self ):
- return self.make_numstats() + self.make_timestats()
- # --- end of make_all (...) ---
+ def get_numstats ( self, as_dict=False ):
+ return self._numstats._as_dict() if as_dict self._numstats
+ # --- end of get_numstats (...) ---
+
+ def get_timestats ( self, as_dict=False ):
+ return self._timestats._as_dict() if as_dict else self._timestats
+ # --- end of get_timestats (...) ---
def get_all ( self ):
- return self._collected_stats
+ return self._numstats + self._timestats
# --- end of get_all (...) ---
# --- end of StatsDBCollector #v0 ---
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-08-16 12:42 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-08-16 12:42 UTC (permalink / raw
To: gentoo-commits
commit: 90b6001bfd4225013b36e7a652a6c829446d6fc4
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 16 12:40:59 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 16 12:40:59 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=90b6001b
roverlay/stats/rating: handle UNKNOWNs
---
roverlay/stats/rating.py | 167 +++++++++++++++++++++++++++++------------------
1 file changed, 102 insertions(+), 65 deletions(-)
diff --git a/roverlay/stats/rating.py b/roverlay/stats/rating.py
index ec36002..3f6b9e0 100644
--- a/roverlay/stats/rating.py
+++ b/roverlay/stats/rating.py
@@ -41,6 +41,7 @@ class StatsRating ( object ):
STATUS_CRIT = 2**4
STATUS_TOO_HIGH = 2**5
STATUS_TOO_LOW = 2**6
+ STATUS_UNDEF = 2**7
STATUS_WARN_LOW = STATUS_WARN | STATUS_TOO_LOW
@@ -50,7 +51,7 @@ class StatsRating ( object ):
STATUS_CRIT_LOW = STATUS_CRIT | STATUS_TOO_LOW
STATUS_CRIT_HIGH = STATUS_CRIT | STATUS_TOO_HIGH
- STATUS_FAIL = ( ( 2**7 ) - 1 ) ^ STATUS_OK
+ STATUS_FAIL = ( ( 2**8 ) - 1 ) ^ STATUS_OK
def __init__ ( self, description ):
@@ -79,9 +80,7 @@ class NumStatsCounterRating ( StatsRating ):
self.err_low = err_low
self.crit_low = crit_low
self.value = value
- self.status = (
- self.get_rating ( value ) if value is not None else None
- )
+ self.status = self.get_rating ( value )
# --- end of __init__ (...) ---
@classmethod
@@ -89,30 +88,38 @@ class NumStatsCounterRating ( StatsRating ):
return cls ( description, value, warn, err, crit )
# --- end of new_fail_counter (...) ---
+ def get_value ( self, unknown_value=0 ):
+ return self.value or unknown_value
+ # --- end of get_value (...) ---
+
def get_rating ( self, value ):
- too_high = lambda high, k: ( high is not None and k > high )
- too_low = lambda low, k: ( low is not None and k < low )
- ret = self.STATUS_NONE
+ if value is None:
+ return self.STATUS_UNDEF
+
+ else:
+ too_high = lambda high, k: ( high is not None and k > high )
+ too_low = lambda low, k: ( low is not None and k < low )
+ ret = self.STATUS_NONE
- if too_high ( self.warn_high, value ):
- ret |= self.STATUS_WARN_HIGH
+ if too_high ( self.warn_high, value ):
+ ret |= self.STATUS_WARN_HIGH
- if too_low ( self.warn_low, value ):
- ret |= self.STATUS_WARN_LOW
+ if too_low ( self.warn_low, value ):
+ ret |= self.STATUS_WARN_LOW
- if too_high ( self.err_high, value ):
- ret |= self.STATUS_ERR_HIGH
+ if too_high ( self.err_high, value ):
+ ret |= self.STATUS_ERR_HIGH
- if too_low ( self.err_low, value ):
- ret |= self.STATUS_ERR_LOW
+ if too_low ( self.err_low, value ):
+ ret |= self.STATUS_ERR_LOW
- if too_high ( self.crit_high, value ):
- ret |= self.STATUS_CRIT_HIGH
+ if too_high ( self.crit_high, value ):
+ ret |= self.STATUS_CRIT_HIGH
- if too_low ( self.crit_low, value ):
- ret |= self.STATUS_CRIT_LOW
+ if too_low ( self.crit_low, value ):
+ ret |= self.STATUS_CRIT_LOW
- return self.STATUS_OK if ret == self.STATUS_NONE else ret
+ return self.STATUS_OK if ret == self.STATUS_NONE else ret
# --- end of get_rating (...) ---
def is_warning ( self ):
@@ -132,9 +139,9 @@ class NumStatsCounterRating ( StatsRating ):
# --- end of is_ok (...) ---
def format_value ( self,
- fmt_ok=None, fmt_warn=None, fmt_err=None, fmt_crit=None
+ fmt_ok=None, fmt_warn=None, fmt_err=None, fmt_crit=None, fmt_undef=None
):
- fmt = self.get_item ( fmt_ok, fmt_warn, fmt_err, fmt_crit )
+ fmt = self.get_item ( fmt_ok, fmt_warn, fmt_err, fmt_crit, fmt_undef )
if fmt:
return fmt.format ( str ( self.value ) )
elif fmt == "":
@@ -143,7 +150,7 @@ class NumStatsCounterRating ( StatsRating ):
return str ( self.value )
# --- end of format_value (...) ---
- def get_item ( self, item_ok, item_warn, item_err, item_crit ):
+ def get_item ( self, item_ok, item_warn, item_err, item_crit, item_undef ):
status = self.status
if self.status & self.STATUS_CRIT:
return item_crit
@@ -156,9 +163,12 @@ class NumStatsCounterRating ( StatsRating ):
# --- end of get_item (...) ---
def get_word ( self,
- word_ok="ok", word_warn="warn", word_err="err", word_crit="crit"
+ word_ok="ok", word_warn="warn", word_err="err", word_crit="crit",
+ word_undef="undef",
):
- return str ( self.get_item ( word_ok, word_warn, word_err, word_crit ) )
+ return str ( self.get_item (
+ word_ok, word_warn, word_err, word_crit, word_undef
+ ) )
# --- end of get_word (...) ---
@@ -201,8 +211,28 @@ class RoverlayNumStatsRating ( NumStatsRating ):
# FIXME: err/crit
values = self.values
- v_ec_post = values['ec_post']
- v_pc_repo = values['pc_repo']
+
+ # very efficient.
+ # TODO/COULDFIX: find a better solution for handling UNKNOWNS
+ # * 0 or 0 == 0 == None or 0
+ # * don't use v_* as value when creating NumStatsCounterRating objects,
+ # the "UNKNOWN" state would get lost
+ #
+ v_pc_repo = values['pc_repo'] or 0
+ v_pc_distmap = values['pc_distmap'] or 0
+ v_pc_filtered = values['pc_filtered'] or 0
+ v_pc_queued = values['pc_queued'] or 0
+ v_pc_success = values['pc_success'] or 0
+ v_pc_fail = values['pc_fail'] or 0
+ v_pc_fail_empty = values['pc_fail_empty'] or 0
+ v_pc_fail_dep = values['pc_fail_dep'] or 0
+ v_pc_fail_selfdep = values['pc_fail_selfdep'] or 0
+ v_pc_fail_err = values['pc_fail_err'] or 0
+ v_ec_pre = values['ec_pre'] or 0
+ v_ec_post = values['ec_post'] or 0
+ v_ec_written = values['ec_written'] or 0
+ v_ec_revbump = values['ec_revbump'] or 0
+
# *_high=k -- warn/... if value > k
# *_low=k -- warn/... if value < k
@@ -211,77 +241,76 @@ class RoverlayNumStatsRating ( NumStatsRating ):
)
self.pc_repo = new_numstats ( 'pc_repo',
- warn_low=1,
- err_low=( 1 if values['pc_distmap'] > 0 else 0 ),
+ warn_low = 1,
+ err_low = ( 1 if v_pc_distmap > 0 else 0 ),
)
self.pc_distmap = new_numstats ( 'pc_distmap',
# src files of imported ebuilds don't get written to the distmap
# (can be "fixed" with --distmap-verify)
- warn_low=max ( 1, v_ec_post ),
- err_low=( 1 if v_ec_post > 0 else 0 ),
- warn_high=( 1.01*v_ec_post if v_ec_post > 0 else None ),
- err_high=( 1.1*v_ec_post if v_ec_post > 0 else None ),
+ warn_low = max ( 1, v_ec_post ),
+ err_low = ( 1 if v_ec_post > 0 else 0 ),
+ warn_high = ( ( 1.01 * v_ec_post ) if v_ec_post > 0 else None ),
+ err_high = ( ( 1.1 * v_ec_post ) if v_ec_post > 0 else None ),
)
self.pc_filtered = new_numstats ( 'pc_filtered',
- crit_high=( v_pc_repo - values['pc_queued'] ),
+ crit_high = ( v_pc_repo - v_pc_queued ),
)
self.pc_queued = new_numstats ( 'pc_queued',
# cannot queue more packages than available
- crit_high=( v_pc_repo - values['pc_filtered'] ),
+ crit_high = ( v_pc_repo - v_pc_filtered ),
)
self.pc_success = new_numstats ( 'pc_success',
- crit_high=values['pc_queued'],
+ crit_high = v_pc_queued,
# warn about low pc_success/pc_queued ratio
- warn_low=(
- 0.9*values['pc_queued'] if values['pc_queued'] > 0 else None
- ),
+ warn_low = ( ( 0.9 * v_pc_queued ) if v_pc_queued > 0 else None ),
)
self.pc_fail = new_numstats ( 'pc_fail',
# pc_queued would produce "false" warnings in incremental mode
- warn_high=( max ( 0, 0.15 * v_pc_repo ) ),
- err_high=( max ( 0, 0.3 * v_pc_repo ) ),
- crit_high=( max ( 0, 0.5 * v_pc_repo ) ),
- crit_low=(
- values['pc_fail_empty'] + values['pc_fail_dep']
- + values['pc_fail_selfdep'] + values['pc_fail_err']
- ),
+ warn_high = max ( 0, 0.15 * v_pc_repo ),
+ err_high = max ( 0, 0.3 * v_pc_repo ),
+ crit_high = max ( 0, 0.5 * v_pc_repo ),
+ crit_low = sum ((
+ v_pc_fail_empty, v_pc_fail_dep, v_pc_fail_selfdep, v_pc_fail_err
+ )),
)
self.pc_fail_empty = new_numstats ( 'pc_fail_empty',
- crit_high=values['pc_fail'],
+ crit_high = v_pc_fail,
)
self.pc_fail_dep = new_numstats ( 'pc_fail_dep',
- crit_high=values['pc_fail'],
- warn_high=max ( 10, 0.01*values['pc_repo'] ),
+ crit_high = v_pc_fail,
+ warn_high = max ( 10, ( 0.01 * v_pc_repo ) ),
)
self.pc_fail_selfdep = new_numstats ( 'pc_fail_selfdep',
- crit_high=values['pc_fail'],
+ crit_high = v_pc_fail,
)
self.pc_fail_err = new_numstats ( 'pc_fail_err',
- warn_high=1, err_high=1, crit_high=values['pc_fail'],
+ warn_high = 1,
+ err_high = 1,
+ crit_high = v_pc_fail,
)
- self.ec_pre = new_numstats ( 'ec_pre',
- warn_high=v_ec_post,
- err_high=max ( 0, 1.05*v_ec_post ),
+ self.ec_pre = new_numstats ( 'ec_pre',
+ warn_high = v_ec_post,
+ err_high = max ( 0, ( 1.05 * v_ec_post ) ),
)
- self.ec_post = new_numstats ( 'ec_post',
- warn_low=values['ec_pre'],
+ self.ec_post = new_numstats ( 'ec_post',
+ warn_low = v_ec_pre,
# tolerate 5% ebuild loss (1/1.05 ~= 0.95)
- err_low=max ( 1, 0.95*values['ec_pre'] ),
+ err_low = max ( 1, ( 0.95 * v_ec_pre ) ),
)
- self.ec_written = new_numstats ( 'ec_written',
- err_low=values['pc_success'],
+ self.ec_written = new_numstats ( 'ec_written',
+ err_low = v_pc_success,
)
self.ec_revbump = new_numstats ( 'ec_revbump',
- crit_high=min ( v_pc_repo, values['ec_pre'] ),
- warn_high=min ( 1, 0.1*values['ec_pre'] ),
+ crit_high = min ( v_pc_repo, v_ec_pre ),
+ warn_high = min ( 1, ( 0.1 * v_ec_pre ) ),
)
# --- end of setup (...) ---
@@ -296,6 +325,13 @@ class RoverlayNumStatsRating ( NumStatsRating ):
else:
code_format = { 'cstart': "<code>", 'cend': "</code>", }
+
+ if any ( value is None for value in self.values ):
+ yield (
+ "database contains UNKNOWNS",
+ [ "run roverlay", ]
+ )
+
if not self.pc_repo.is_ok():
yield (
"low repo package count",
@@ -321,7 +357,7 @@ class RoverlayNumStatsRating ( NumStatsRating ):
yield ( topic, details )
- if self.pc_success.value < 1:
+ if self.pc_success.get_value(2) < 1:
if self.pc_success.status & (self.STATUS_ERR|self.STATUS_CRIT):
yield ( "no ebuilds created", None )
else:
@@ -335,9 +371,9 @@ class RoverlayNumStatsRating ( NumStatsRating ):
None
)
- if self.pc_fail.value > 0 or not self.pc_success.is_ok():
+ if self.pc_fail.get_value() > 0 or not self.pc_success.is_ok():
details = []
- if self.pc_fail_dep.value > 0:
+ if self.pc_fail_dep.get_value() > 0:
details.append ( "write dependency rules" )
details.append ( 'configure package ignore rules' )
@@ -351,7 +387,6 @@ class RoverlayNumStatsRating ( NumStatsRating ):
)
-
if not self.pc_fail_err.is_ok():
yield (
"failures due to unknown errors",
@@ -367,6 +402,8 @@ class RoverlayNumStatsRating ( NumStatsRating ):
None
)
+ # +++ UNKNOWNS
+
# --- end of get_suggestions (...) ---
# --- end of RoverlayNumStatsRating ---
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-08-16 10:43 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-08-16 10:43 UTC (permalink / raw
To: gentoo-commits
commit: f400284d5fab7c5157796437fb6d0b47cde16072
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 16 10:21:10 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 16 10:21:10 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f400284d
roverlay/stats: rate stats
check stats for min/max values, get suggestions/notes
---
roverlay/stats/dbcollector.py | 13 +-
roverlay/stats/rating.py | 372 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 377 insertions(+), 8 deletions(-)
diff --git a/roverlay/stats/dbcollector.py b/roverlay/stats/dbcollector.py
index a9169e6..2961aec 100644
--- a/roverlay/stats/dbcollector.py
+++ b/roverlay/stats/dbcollector.py
@@ -7,21 +7,18 @@
import collections
import weakref
+from . import rating
+
class StatsDBCollector ( object ):
VERSION = 0
- # 'pc' := package count(er), 'ec' := ebuild _
- NUMSTATS_KEYS = (
- 'pc_repo', 'pc_distmap', 'pc_filtered', 'pc_queued', 'pc_success',
- 'pc_fail', 'pc_fail_empty', 'pc_fail_dep', 'pc_fail_selfdep',
- 'pc_fail_err',
- 'ec_pre', 'ec_post', 'ec_written', 'ec_revbump',
- )
+ NUMSTATS_KEYS = rating.NUMSTATS.keys()
NUMSTATS = collections.namedtuple (
"numstats", ' '.join ( NUMSTATS_KEYS )
)
- TIMESTATS_KEYS = ()
+
+ TIMESTATS_KEYS = rating.TIMESTATS.keys()
TIMESTATS = collections.namedtuple (
"timestats", ' '.join ( TIMESTATS_KEYS )
)
diff --git a/roverlay/stats/rating.py b/roverlay/stats/rating.py
new file mode 100644
index 0000000..ec36002
--- /dev/null
+++ b/roverlay/stats/rating.py
@@ -0,0 +1,372 @@
+# R overlay -- stats collection, "rate" stats ("good","bad",...)
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import collections
+
+import roverlay.util.objects
+
+# 'pc' := package count(er), 'ec' := ebuild _
+#
+NUMSTATS = collections.OrderedDict ((
+ ( 'pc_repo', "package files found in the repositories" ),
+ ( 'pc_distmap', "package files in the mirror directory" ),
+ ( 'pc_filtered', "packages ignored by package rules" ),
+ ( 'pc_queued', "packages queued for ebuild creation" ),
+ ( 'pc_success', "packages for which ebuild creation succeeded" ),
+ ( 'pc_fail', "packages for which ebuild creation failed" ),
+ # actually, it's not the package that "fails"
+ ( 'pc_fail_empty', "packages failed due to empty/unsuitable DESCRIPTION" ),
+ ( 'pc_fail_dep', "packages failed due to unresolvable dependencies" ),
+ ( 'pc_fail_selfdep', "packages failed due to unsatisfiable selfdeps" ),
+ ( 'pc_fail_err', "packages failed due to unknown errors" ),
+ ( 'ec_pre', "ebuild count prior to overlay creation" ),
+ ( 'ec_post', "ebuild count after overlay creation" ),
+ ( 'ec_written', "ebuilds written" ),
+ ( 'ec_revbump', "ebuilds rev-bumped due to package file content change" ),
+))
+
+
+TIMESTATS = {}
+
+class StatsRating ( object ):
+
+ STATUS_NONE = 0
+ ##STATUS_OK = 2**1
+ STATUS_OK = STATUS_NONE
+ STATUS_WARN = 2**2
+ STATUS_ERR = 2**3
+ STATUS_CRIT = 2**4
+ STATUS_TOO_HIGH = 2**5
+ STATUS_TOO_LOW = 2**6
+
+
+ STATUS_WARN_LOW = STATUS_WARN | STATUS_TOO_LOW
+ STATUS_WARN_HIGH = STATUS_WARN | STATUS_TOO_HIGH
+ STATUS_ERR_LOW = STATUS_ERR | STATUS_TOO_LOW
+ STATUS_ERR_HIGH = STATUS_ERR | STATUS_TOO_HIGH
+ STATUS_CRIT_LOW = STATUS_CRIT | STATUS_TOO_LOW
+ STATUS_CRIT_HIGH = STATUS_CRIT | STATUS_TOO_HIGH
+
+ STATUS_FAIL = ( ( 2**7 ) - 1 ) ^ STATUS_OK
+
+
+ def __init__ ( self, description ):
+ super ( StatsRating, self ).__init__()
+ self.description = description
+ # --- end of __init__ (...) ---
+
+ @roverlay.util.objects.abstractmethod
+ def get_rating ( self, value ):
+ return STATUS_NONE
+ # --- end of get_rating (...) ---
+
+# --- end of StatsRating ---
+
+class NumStatsCounterRating ( StatsRating ):
+
+ def __init__ ( self, description, value,
+ warn_high=None, err_high=None, crit_high=None,
+ warn_low=0, err_low=0, crit_low=0
+ ):
+ super ( NumStatsCounterRating, self ).__init__ ( description )
+ self.warn_high = warn_high
+ self.err_high = err_high
+ self.crit_high = crit_high
+ self.warn_low = warn_low
+ self.err_low = err_low
+ self.crit_low = crit_low
+ self.value = value
+ self.status = (
+ self.get_rating ( value ) if value is not None else None
+ )
+ # --- end of __init__ (...) ---
+
+ @classmethod
+ def new_fail_counter ( cls, description, value, warn=1, err=1, crit=1 ):
+ return cls ( description, value, warn, err, crit )
+ # --- end of new_fail_counter (...) ---
+
+ def get_rating ( self, value ):
+ too_high = lambda high, k: ( high is not None and k > high )
+ too_low = lambda low, k: ( low is not None and k < low )
+ ret = self.STATUS_NONE
+
+ if too_high ( self.warn_high, value ):
+ ret |= self.STATUS_WARN_HIGH
+
+ if too_low ( self.warn_low, value ):
+ ret |= self.STATUS_WARN_LOW
+
+ if too_high ( self.err_high, value ):
+ ret |= self.STATUS_ERR_HIGH
+
+ if too_low ( self.err_low, value ):
+ ret |= self.STATUS_ERR_LOW
+
+ if too_high ( self.crit_high, value ):
+ ret |= self.STATUS_CRIT_HIGH
+
+ if too_low ( self.crit_low, value ):
+ ret |= self.STATUS_CRIT_LOW
+
+ return self.STATUS_OK if ret == self.STATUS_NONE else ret
+ # --- end of get_rating (...) ---
+
+ def is_warning ( self ):
+ return self.status & self.STATUS_WARN
+ # --- end of is_warning (...) ---
+
+ def is_error ( self ):
+ return self.status & self.STATUS_ERR
+ # --- end of is_error (...) ---
+
+ def is_critical ( self ):
+ return self.status & self.STATUS_CRIT
+ # --- end of is_critical (...) ---
+
+ def is_ok ( self ):
+ return self.status == self.STATUS_OK
+ # --- end of is_ok (...) ---
+
+ def format_value ( self,
+ fmt_ok=None, fmt_warn=None, fmt_err=None, fmt_crit=None
+ ):
+ fmt = self.get_item ( fmt_ok, fmt_warn, fmt_err, fmt_crit )
+ if fmt:
+ return fmt.format ( str ( self.value ) )
+ elif fmt == "":
+ return fmt
+ else:
+ return str ( self.value )
+ # --- end of format_value (...) ---
+
+ def get_item ( self, item_ok, item_warn, item_err, item_crit ):
+ status = self.status
+ if self.status & self.STATUS_CRIT:
+ return item_crit
+ elif self.status & self.STATUS_ERR:
+ return item_err
+ elif self.status & self.STATUS_WARN:
+ return item_warn
+ else:
+ return item_ok
+ # --- end of get_item (...) ---
+
+ def get_word ( self,
+ word_ok="ok", word_warn="warn", word_err="err", word_crit="crit"
+ ):
+ return str ( self.get_item ( word_ok, word_warn, word_err, word_crit ) )
+ # --- end of get_word (...) ---
+
+
+# --- end of NumStatsCounterRating ---
+
+
+class NumStatsRating ( StatsRating ):
+
+ def __init__ ( self, values, description=None ):
+ assert isinstance ( values, dict )
+ super ( NumStatsRating, self ).__init__ ( description=description )
+ self.values = values
+ self.setup()
+ # --- end of __init__ (...) ---
+
+ @roverlay.util.objects.abstractmethod
+ def setup ( self ):
+ pass
+ # --- end of setup (...) ---
+
+ def get_rating ( self ):
+ return self
+ # --- end of get_rating (...) ---
+
+# --- end of NumStatsRating ---
+
+
+class RoverlayNumStatsRating ( NumStatsRating ):
+
+ ## COULDFIX: yield str key / int indizes in get_suggestions() and
+ ## optionall "translate" them using a map
+ ## "event" => message
+ ##SUGGESTIONS = {}
+
+
+ def setup ( self ):
+ if __debug__:
+ assert set ( NUMSTATS.keys() ) == set ( self.values.keys() )
+
+ # FIXME: err/crit
+
+ values = self.values
+ v_ec_post = values['ec_post']
+ v_pc_repo = values['pc_repo']
+
+ # *_high=k -- warn/... if value > k
+ # *_low=k -- warn/... if value < k
+ new_numstats = lambda key, **b: (
+ NumStatsCounterRating ( NUMSTATS[key], values[key], **b )
+ )
+
+ self.pc_repo = new_numstats ( 'pc_repo',
+ warn_low=1,
+ err_low=( 1 if values['pc_distmap'] > 0 else 0 ),
+ )
+
+ self.pc_distmap = new_numstats ( 'pc_distmap',
+ # src files of imported ebuilds don't get written to the distmap
+ # (can be "fixed" with --distmap-verify)
+ warn_low=max ( 1, v_ec_post ),
+ err_low=( 1 if v_ec_post > 0 else 0 ),
+ warn_high=( 1.01*v_ec_post if v_ec_post > 0 else None ),
+ err_high=( 1.1*v_ec_post if v_ec_post > 0 else None ),
+ )
+
+ self.pc_filtered = new_numstats ( 'pc_filtered',
+ crit_high=( v_pc_repo - values['pc_queued'] ),
+ )
+
+ self.pc_queued = new_numstats ( 'pc_queued',
+ # cannot queue more packages than available
+ crit_high=( v_pc_repo - values['pc_filtered'] ),
+ )
+
+ self.pc_success = new_numstats ( 'pc_success',
+ crit_high=values['pc_queued'],
+ # warn about low pc_success/pc_queued ratio
+ warn_low=(
+ 0.9*values['pc_queued'] if values['pc_queued'] > 0 else None
+ ),
+ )
+
+ self.pc_fail = new_numstats ( 'pc_fail',
+ # pc_queued would produce "false" warnings in incremental mode
+ warn_high=( max ( 0, 0.15 * v_pc_repo ) ),
+ err_high=( max ( 0, 0.3 * v_pc_repo ) ),
+ crit_high=( max ( 0, 0.5 * v_pc_repo ) ),
+ crit_low=(
+ values['pc_fail_empty'] + values['pc_fail_dep']
+ + values['pc_fail_selfdep'] + values['pc_fail_err']
+ ),
+ )
+
+ self.pc_fail_empty = new_numstats ( 'pc_fail_empty',
+ crit_high=values['pc_fail'],
+ )
+ self.pc_fail_dep = new_numstats ( 'pc_fail_dep',
+ crit_high=values['pc_fail'],
+ warn_high=max ( 10, 0.01*values['pc_repo'] ),
+ )
+ self.pc_fail_selfdep = new_numstats ( 'pc_fail_selfdep',
+ crit_high=values['pc_fail'],
+ )
+ self.pc_fail_err = new_numstats ( 'pc_fail_err',
+ warn_high=1, err_high=1, crit_high=values['pc_fail'],
+ )
+
+ self.ec_pre = new_numstats ( 'ec_pre',
+ warn_high=v_ec_post,
+ err_high=max ( 0, 1.05*v_ec_post ),
+ )
+
+ self.ec_post = new_numstats ( 'ec_post',
+ warn_low=values['ec_pre'],
+ # tolerate 5% ebuild loss (1/1.05 ~= 0.95)
+ err_low=max ( 1, 0.95*values['ec_pre'] ),
+ )
+ self.ec_written = new_numstats ( 'ec_written',
+ err_low=values['pc_success'],
+ )
+ self.ec_revbump = new_numstats ( 'ec_revbump',
+ crit_high=min ( v_pc_repo, values['ec_pre'] ),
+ warn_high=min ( 1, 0.1*values['ec_pre'] ),
+ )
+ # --- end of setup (...) ---
+
+ def __iter__ ( self ):
+ for key in NUMSTATS.keys():
+ yield ( key, getattr ( self, key ) )
+ # --- end of __iter__ (...) ---
+
+ def get_suggestions ( self, pure_text=False ):
+ if pure_text:
+ code_format = { 'cstart': "\'", 'cend': "\'", }
+ else:
+ code_format = { 'cstart': "<code>", 'cend': "</code>", }
+
+ if not self.pc_repo.is_ok():
+ yield (
+ "low repo package count",
+ [ "check the repo config file", "drop repos without packages" ]
+ )
+
+ if not self.pc_distmap.is_ok():
+ details = [
+ 'run {cstart}roverlay --distmap-verify{cend} to fix '
+ 'the distmap'.format ( **code_format )
+ ]
+
+ if self.pc_distmap.status & self.STATUS_TOO_HIGH:
+ topic = "distmap file count is higher than the ebuild count"
+ else:
+ topic = "distmap file count is lower than the ebuild count"
+ details.append (
+ 'run {cstart}roverlay --fixup-category-move[-reverse]{cend} '
+ ' after configuring relocations (in the package rules)'.format (
+ **code_format
+ )
+ )
+ yield ( topic, details )
+
+
+ if self.pc_success.value < 1:
+ if self.pc_success.status & (self.STATUS_ERR|self.STATUS_CRIT):
+ yield ( "no ebuilds created", None )
+ else:
+ yield (
+ "no ebuilds created (not an issue in incremental mode)",
+ None
+ )
+ elif not self.pc_success.is_ok():
+ yield (
+ "only a few ebuilds created (not an issue in incremental mode)",
+ None
+ )
+
+ if self.pc_fail.value > 0 or not self.pc_success.is_ok():
+ details = []
+ if self.pc_fail_dep.value > 0:
+ details.append ( "write dependency rules" )
+ details.append ( 'configure package ignore rules' )
+
+ yield (
+ '{adj} failure rate'.format (
+ adj=self.pc_fail.get_word (
+ "normal/expected", "moderate", "high", "very high"
+ )
+ ),
+ details
+ )
+
+
+
+ if not self.pc_fail_err.is_ok():
+ yield (
+ "failures due to unknown errors",
+ [ 'check the log files for python exceptions and report them', ]
+ )
+
+ if not self.ec_pre.is_ok() or not self.ec_post.is_ok():
+ yield ( "ebuild loss occurred (no suggestions available)", None )
+
+ if not self.ec_revbump.is_ok():
+ yield (
+ "unexpected ebuild revbump count (no suggestions available)",
+ None
+ )
+
+ # --- end of get_suggestions (...) ---
+
+# --- end of RoverlayNumStatsRating ---
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-08-15 9:18 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-08-15 9:18 UTC (permalink / raw
To: gentoo-commits
commit: e469805008ebc3743087694844d319e862f0803d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug 15 09:17:22 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug 15 09:17:22 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=e4698050
roverlay/stats/rrd: use GAUGE for numstats
---
roverlay/stats/dbcollector.py | 2 ++
roverlay/stats/rrd.py | 5 +++--
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/roverlay/stats/dbcollector.py b/roverlay/stats/dbcollector.py
index 402e64d..a9169e6 100644
--- a/roverlay/stats/dbcollector.py
+++ b/roverlay/stats/dbcollector.py
@@ -41,6 +41,8 @@ class StatsDBCollector ( object ):
ov_create = stats.overlay_creation
ov = stats.overlay
+ # note that the int() casts are necessary here
+ #
return self.__class__.NUMSTATS (
pc_repo = int ( stats.repo.pkg_count ),
pc_distmap = int ( stats.distmap.pkg_count ),
diff --git a/roverlay/stats/rrd.py b/roverlay/stats/rrd.py
index ffccbe1..2a09bdd 100644
--- a/roverlay/stats/rrd.py
+++ b/roverlay/stats/rrd.py
@@ -43,14 +43,15 @@ class StatsDB ( roverlay.db.rrdtool.RRD ):
def make_vars ( self ):
heartbeat = 2 * self.step
return tuple (
- RRDVariable ( k, 'DERIVE', val_min=0, heartbeat=heartbeat )
+ RRDVariable ( k, 'GAUGE', val_min=0, heartbeat=heartbeat )
for k in self.collector.NUMSTATS_KEYS
)
# --- end of make_vars (...) ---
def make_rra ( self ):
return (
- RRDArchive.new_day ( 'LAST', 0.7, step=self.step ),
+ #RRDArchive.new_day ( 'MIN', 0.7, step=self.step ),
+ RRDArchive.new_day ( 'MAX', 0.7, step=self.step ),
RRDArchive.new_week ( 'AVERAGE', 0.7, step=self.step ),
RRDArchive.new_month ( 'AVERAGE', 0.7, step=self.step ),
)
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-08-02 10:34 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-08-02 10:34 UTC (permalink / raw
To: gentoo-commits
commit: a343c180f2d22c0411086c3dbca5eb2d85da235e
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 2 10:20:04 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 2 10:20:04 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a343c180
stats collection: has_changes(), count imports
* overlay_has_changes()
* count imported ebuilds
---
roverlay/stats/abstract.py | 54 ++++++++++++++++++++++++++++++++++++++++-----
roverlay/stats/base.py | 36 ++++++++++++++++++++++++------
roverlay/stats/collector.py | 7 ++++++
3 files changed, 84 insertions(+), 13 deletions(-)
diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
index 68728f4..5ac87bc 100644
--- a/roverlay/stats/abstract.py
+++ b/roverlay/stats/abstract.py
@@ -9,7 +9,8 @@ from __future__ import division
import collections
import time
-from roverlay.util.objects import MethodNotImplementedError
+import roverlay.util.objects
+from roverlay.util.objects import MethodNotImplementedError, abstractmethod
class RoverlayStatsBase ( object ):
@@ -47,15 +48,15 @@ class RoverlayStatsBase ( object ):
getattr ( self, member ).merge_with ( getattr ( other, member ) )
# --- end of merge_members (...) ---
- def _iter_members ( self, nofail=False ):
+ def iter_members ( self, nofail=False ):
if not nofail or hasattr ( self, '_MEMBERS' ):
for member in self.__class__._MEMBERS:
yield getattr ( self, member )
- # --- end of _iter_members (...) ---
+ # --- end of iter_members (...) ---
def has_nonzero ( self ):
if hasattr ( self, '_MEMBERS' ):
- for member in self._iter_members():
+ for member in self.iter_members():
if int ( member ) != 0:
return member
else:
@@ -63,7 +64,7 @@ class RoverlayStatsBase ( object ):
# --- end of has_nonzero (...) ---
def reset_members ( self ):
- for member in self._iter_members():
+ for member in self.iter_members():
member.reset()
# --- end of reset_members (...) ---
@@ -86,7 +87,7 @@ class RoverlayStatsBase ( object ):
if desc:
yield desc
- for member in self._iter_members( nofail=True ):
+ for member in self.iter_members( nofail=True ):
yield str ( member )
# --- end of gen_str (...) ---
@@ -103,6 +104,15 @@ class RoverlayStatsBase ( object ):
class RoverlayStats ( RoverlayStatsBase ):
pass
+
+ @abstractmethod
+ def has_changes ( self ):
+ """Returns True if this stats item has any numbers indicating that
+ the overlay has changes.
+ """
+ return False
+ # --- end of has_changes (...) ---
+
# --- end of RoverlayStats ---
@@ -143,6 +153,10 @@ class TimeStats ( RoverlayStats ):
self._timestats = collections.OrderedDict()
# --- end of __init__ (...) ---
+ def has_changes ( self ):
+ return False
+ # --- end of has_changes (...) ---
+
def merge_with ( self, other ):
self._timestats.update ( other._timestats )
# --- end of merge_with (...) ---
@@ -225,6 +239,10 @@ class Counter ( RoverlayStatsBase ):
return float ( self.total_count )
# --- end of __float__ (...) ---
+ def __bool__ ( self ):
+ return bool ( self.total_count )
+ # --- end of __bool__ (...) ---
+
def __add__ ( self, other ):
return self.total_count + int ( other )
# --- end of __add__ (...) ---
@@ -233,6 +251,30 @@ class Counter ( RoverlayStatsBase ):
return self.total_count - int ( other )
# --- end of __sub__ (...) ---
+ def __gt__ ( self, other ):
+ return self.total_count > int ( other )
+ # --- end of __gt__ (...) ---
+
+ def __ge__ ( self, other ):
+ return self.total_count >= int ( other )
+ # --- end of __ge__ (...) ---
+
+ def __lt__ ( self, other ):
+ return self.total_count < int ( other )
+ # --- end of __lt__ (...) ---
+
+ def __le__ ( self, other ):
+ return self.total_count <= int ( other )
+ # --- end of __le__ (...) ---
+
+ def __eq__ ( self, other ):
+ return self.total_count == int ( other )
+ # --- end of __eq__ (...) ---
+
+ def __ne__ ( self, other ):
+ return self.total_count != int ( other )
+ # --- end of __ne__ (...) ---
+
def has_details ( self ):
return False
# --- end of has_details (...) ---
diff --git a/roverlay/stats/base.py b/roverlay/stats/base.py
index 28531c2..282ff78 100644
--- a/roverlay/stats/base.py
+++ b/roverlay/stats/base.py
@@ -19,6 +19,10 @@ class RepoStats ( abstract.RoverlayStats ):
)
# --- end of __init__ (...) ---
+ def has_changes ( self ):
+ return False
+ # --- end of has_changes (...) ---
+
def package_file_found ( self, repo ):
self.pkg_count.inc ( repo.name )
# --- end of add_package (...) ---
@@ -37,6 +41,10 @@ class DistmapStats ( abstract.RoverlayStats ):
)
# --- end of __init__ (...) ---
+ def has_changes ( self ):
+ return False
+ # --- end of has_changes (...) ---
+
def file_added ( self, *origin ):
self.pkg_count.inc ( *origin )
# --- end of file_added (...) ---
@@ -58,6 +66,10 @@ class OverlayCreationWorkerStats ( abstract.RoverlayStats ):
self.pkg_success = abstract.Counter ( "success" )
# --- end of __init__ (...) ---
+ def has_changes ( self ):
+ return bool ( self.pkg_success )
+ # --- end of has_changes (...) ---
+
# --- end of OverlayCreationWorkerStats ---
@@ -103,24 +115,34 @@ class OverlayStats ( abstract.RoverlayStats ):
_MEMBERS = (
'scan_time', 'write_time',
- 'ebuilds_scanned', 'ebuild_count', 'revbump_count', 'ebuilds_written',
+ 'ebuilds_scanned', 'ebuild_count', 'revbump_count',
+ 'ebuilds_imported', 'ebuilds_written',
)
def __init__ ( self ):
super ( OverlayStats, self ).__init__()
# ebuilds_scanned: ebuild count prior to running overlay creation
- self.ebuilds_scanned = abstract.Counter ( "pre" )
+ self.ebuilds_scanned = abstract.Counter ( "pre" )
# ebuild count: ebuild count after writing the overlay
- self.ebuild_count = abstract.Counter ( "post" )
+ self.ebuild_count = abstract.Counter ( "post" )
- self.revbump_count = abstract.Counter ( "revbumps" )
- self.ebuilds_written = abstract.Counter ( "written" )
+ self.revbump_count = abstract.Counter ( "revbumps" )
+ self.ebuilds_written = abstract.Counter ( "written" )
+ self.ebuilds_imported = abstract.Counter ( "imported" )
- self.write_time = abstract.TimeStats ( "write_time" )
- self.scan_time = abstract.TimeStats ( "scan_time" )
+ self.write_time = abstract.TimeStats ( "write_time" )
+ self.scan_time = abstract.TimeStats ( "scan_time" )
# --- end of __init__ (...) ---
+ def has_changes ( self ):
+ return (
+ ( self.ebuild_count - self.ebuilds_scanned ) != 0 or
+ self.ebuilds_written > 0 or
+ self.revbump_count > 0
+ )
+ # --- end of has_changes (...) ---
+
def set_ebuild_written ( self, p_info ):
self.ebuilds_written.inc()
# direct dict access
diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py
index 85eb358..ebae1a6 100644
--- a/roverlay/stats/collector.py
+++ b/roverlay/stats/collector.py
@@ -26,6 +26,13 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
return cls._instance
# --- end of instance (...) ---
+ def overlay_has_any_changes ( self ):
+ """Returns True if the resulting overlay has any changes (according to
+ stats).
+ """
+ return any ( x.has_changes() for x in self.iter_members() )
+ # --- end of overlay_has_any_changes (...) ---
+
def get_success_ratio ( self ):
# success ratio for "this" run:
# new ebuilds / relevant package count (new packages - unsuitable,
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-07-29 14:56 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-07-29 14:56 UTC (permalink / raw
To: gentoo-commits
commit: d1300fc92439e26b6941eec62ec888329835ee77
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 29 14:53:05 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 29 14:53:05 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d1300fc9
write stats to round-robin database (rrdtool)
This commit makes (/allows to/) stats data persistent by adding them to a db,
which can later be used to draw graphs etc.
---
roverlay/stats/abstract.py | 37 +-----
roverlay/stats/collector.py | 156 +++++---------------------
roverlay/stats/dbcollector.py | 74 ++++++++++++
roverlay/stats/rrd.py | 63 +++++++++++
roverlay/stats/{collector.py => visualize.py} | 91 +++------------
5 files changed, 189 insertions(+), 232 deletions(-)
diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
index 6d0575a..68728f4 100644
--- a/roverlay/stats/abstract.py
+++ b/roverlay/stats/abstract.py
@@ -9,34 +9,7 @@ from __future__ import division
import collections
import time
-class MethodNotImplemented ( NotImplementedError ):
- def __init__ ( self, obj, method ):
- super ( MethodNotImplemented, self ).__init__ (
- "{n}.{f}()".format ( n=obj.__class__.__name__, f=method )
- )
- # --- end of __init__ (...) ---
-
-# --- end of MethodNotImplemented ---
-
-class StatsVisualizer ( object ):
-
- def __init__ ( self, stats ):
- super ( StatsVisualizer, self ).__init__()
- self.stats = stats
- self.lines = None
-
- self.prepare()
- # --- end of __init__ (...) ---
-
- def prepare ( self ):
- raise MethodNotImplemented ( self, 'prepare' )
- # --- end of prepare (...) ---
-
- def __str__ ( self ):
- return '\n'.join ( self.lines )
- # --- end of __str__ (...) ---
-
-# --- end of StatsVisualizer ---
+from roverlay.util.objects import MethodNotImplementedError
class RoverlayStatsBase ( object ):
@@ -66,7 +39,7 @@ class RoverlayStatsBase ( object ):
self.merge_members ( other, my_cls._MEMBERS )
else:
- raise MethodNotImplemented ( self, 'merge_with' )
+ raise MethodNotImplementedError ( self, 'merge_with' )
# --- end of merge_with (...) ---
def merge_members ( self, other, members ):
@@ -86,7 +59,7 @@ class RoverlayStatsBase ( object ):
if int ( member ) != 0:
return member
else:
- raise MethodNotImplemented ( self, 'has_nonzero' )
+ raise MethodNotImplementedError ( self, 'has_nonzero' )
# --- end of has_nonzero (...) ---
def reset_members ( self ):
@@ -98,7 +71,7 @@ class RoverlayStatsBase ( object ):
if hasattr ( self.__class__, '_MEMBERS' ):
self.reset_members()
else:
- raise MethodNotImplemented ( self, 'reset' )
+ raise MethodNotImplementedError ( self, 'reset' )
# --- end of reset (...) ---
def get_description_str ( self ):
@@ -122,7 +95,7 @@ class RoverlayStatsBase ( object ):
if ret:
return ret
else:
- raise MethodNotImplemented ( self, '__str__' )
+ raise MethodNotImplementedError ( self, '__str__' )
# --- end of __str__ (...) ---
# --- end of RoverlayStatsBase ---
diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py
index 3e3dee7..30b0e76 100644
--- a/roverlay/stats/collector.py
+++ b/roverlay/stats/collector.py
@@ -4,10 +4,15 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
-import collections
+import time
+
+import roverlay.config.static
from . import abstract
from . import base
+from . import dbcollector
+from . import visualize
+from . import rrd
class StatsCollector ( abstract.RoverlayStatsBase ):
@@ -57,13 +62,29 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
# --- end of get_net_gain (...) ---
def __init__ ( self ):
+ super ( StatsCollector, self ).__init__()
+
self.time = abstract.TimeStats ( "misc time stats" )
self.distmap = base.DistmapStats()
self.overlay = base.OverlayStats()
self.overlay_creation = base.OverlayCreationStats()
self.repo = base.RepoStats()
+ self.db_collector = None
+ self._database = None
# --- end of __init__ (...) ---
+ def setup_database ( self, config=None ):
+ conf = (
+ config if config is not None else roverlay.config.static.access()
+ )
+
+ self.db_collector = dbcollector.StatsDBCollector ( self )
+ self._database = rrd.StatsDB (
+ conf.get_or_fail ( "RRD_DB.file" ), self.db_collector
+ )
+ self._database.create_if_missing()
+ # --- end of setup_database (...) ---
+
def gen_str ( self ):
yield "{success}, overall {osuccess}".format (
success = self.get_success_ratio(),
@@ -77,136 +98,17 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
# --- end of gen_str (...) ---
def get_creation_str ( self ):
- return str ( CreationStatsVisualizer ( self ) )
+ return str ( visualize.CreationStatsVisualizer ( self ) )
# --- end of to_creation_str (...) ---
-# --- end of StatsCollector ---
-
-
-class CreationStatsVisualizer ( abstract.StatsVisualizer ):
-
- def prepare ( self ):
- EMPTY_LINE = ""
-
- pkg_count = self.stats.repo.pkg_count
- pkg_queued = self.stats.overlay_creation.pkg_queued
- pkg_fail = self.stats.overlay_creation.pkg_fail
- pkg_success = self.stats.overlay_creation.pkg_success
- ebuild_delta = self.stats.get_net_gain()
- revbumps = self.stats.overlay.revbump_count
-
- max_number_len = min (
- len ( str ( int ( k ) ) ) for k in (
- pkg_queued, pkg_fail, pkg_success
- )
- )
- max_number_len = min ( 5, max_number_len )
-
- success_ratio = self.stats.get_success_ratio()
- overall_success_ratio = self.stats.get_overall_success_ratio()
-
+ def write_db ( self ):
+ self.db_collector.update()
+ self._database.update()
+ self._database.commit()
+ # --- end of write_db (...) ---
- timestats = (
- ( 'scan_overlay', self.stats.overlay.scan_time.get_total_str() ),
- ( 'add_packages', self.stats.repo.queue_time.get_total_str() ),
- (
- 'ebuild_creation',
- self.stats.overlay_creation.creation_time.get_total_str()
- ),
- ( 'write_overlay', self.stats.overlay.write_time.get_total_str() ),
- )
-
- try:
- max_time_len = max ( len(k) for k, v in timestats if v is not None )
- except ValueError:
- # empty sequence -> no timestats
- max_time_len = -1
- else:
- # necessary?
- max_time_len = min ( 39, max_time_len )
-
-
- # create lines
- lines = collections.deque()
- unshift = lines.appendleft
- append = lines.append
-
- numstats = lambda k, s: "{num:<{l}d} {s}".format (
- num=int ( k ), s=s, l=max_number_len
- )
-
-
- append (
- 'success ratio {s_i:.2%} (overall {s_o:.2%})'.format (
- s_i = success_ratio.get_ratio(),
- s_o = overall_success_ratio.get_ratio()
- )
- )
- append ( EMPTY_LINE )
-
- append (
- "{e:+d} ebuilds ({r:d} revbumps)".format (
- e=ebuild_delta, r=int ( revbumps )
- )
- )
- append ( EMPTY_LINE )
-
- if int ( pkg_count ) != int ( pkg_queued ):
- append ( numstats (
- pkg_queued,
- '/ {n:d} packages added to the ebuild creation queue'.format (
- n=int ( pkg_count )
- )
- ) )
- else:
- append ( numstats (
- pkg_queued, 'packages added to the ebuild creation queue'
- ) )
-
- append ( numstats (
- pkg_success, 'packages passed ebuild creation'
- ) )
-
- append ( numstats (
- pkg_fail, 'packages failed ebuild creation'
- ) )
-
- if pkg_fail.has_details() and int ( pkg_fail ) != 0:
- append ( EMPTY_LINE )
- append ( "Details for ebuild creation failure:" )
- details = sorted (
- ( ( k, int(v) ) for k, v in pkg_fail.iter_details() ),
- key=lambda kv: kv[1]
- )
- dlen = len ( str ( max ( details, key=lambda kv: kv[1] ) [1] ) )
-
- for key, value in details:
- append ( "* {v:>{l}d}: {k}".format ( k=key, v=value, l=dlen ) )
- # -- end if <have pkg_fail details>
-
- if max_time_len > 0:
- # or >= 0
- append ( EMPTY_LINE )
- for k, v in timestats:
- if v is not None:
- append (
- "time for {0:<{l}} : {1}".format ( k, v, l=max_time_len )
- )
- # -- end if timestats
-
-
- append ( EMPTY_LINE )
-
- # add header/footer line(s)
- max_line_len = 2 + min ( 78, max ( len(s) for s in lines ) )
- unshift (
- "{0:-^{1}}\n".format ( " Overlay creation stats ", max_line_len )
- )
- append ( max_line_len * '-' )
+# --- end of StatsCollector ---
- self.lines = lines
- # --- end of gen_str (...) ---
-# --- end of CreationStatsVisualizer ---
static = StatsCollector()
StatsCollector._instance = static
diff --git a/roverlay/stats/dbcollector.py b/roverlay/stats/dbcollector.py
new file mode 100644
index 0000000..402e64d
--- /dev/null
+++ b/roverlay/stats/dbcollector.py
@@ -0,0 +1,74 @@
+# R overlay -- stats collection, prepare data for db storage
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import collections
+import weakref
+
+class StatsDBCollector ( object ):
+ VERSION = 0
+
+ # 'pc' := package count(er), 'ec' := ebuild _
+ NUMSTATS_KEYS = (
+ 'pc_repo', 'pc_distmap', 'pc_filtered', 'pc_queued', 'pc_success',
+ 'pc_fail', 'pc_fail_empty', 'pc_fail_dep', 'pc_fail_selfdep',
+ 'pc_fail_err',
+ 'ec_pre', 'ec_post', 'ec_written', 'ec_revbump',
+ )
+ NUMSTATS = collections.namedtuple (
+ "numstats", ' '.join ( NUMSTATS_KEYS )
+ )
+
+ TIMESTATS_KEYS = ()
+ TIMESTATS = collections.namedtuple (
+ "timestats", ' '.join ( TIMESTATS_KEYS )
+ )
+
+ def __init__ ( self, stats ):
+ super ( StatsDBCollector, self ).__init__()
+ self.stats = weakref.ref ( stats )
+ self._collected_stats = None
+ # --- end of __init__ (...) ---
+
+ def update ( self ):
+ self._collected_stats = self.make_all()
+ # --- end of update (...) ---
+
+ def make_numstats ( self ):
+ stats = self.stats()
+ ov_create = stats.overlay_creation
+ ov = stats.overlay
+
+ return self.__class__.NUMSTATS (
+ pc_repo = int ( stats.repo.pkg_count ),
+ pc_distmap = int ( stats.distmap.pkg_count ),
+ pc_filtered = int ( ov_create.pkg_filtered ),
+ pc_queued = int ( ov_create.pkg_queued ),
+ pc_success = int ( ov_create.pkg_success ),
+ pc_fail = int ( ov_create.pkg_fail ),
+ pc_fail_empty = ov_create.pkg_fail.get ( 'empty_desc' ),
+ pc_fail_dep = ov_create.pkg_fail.get ( 'unresolved_deps' ),
+ pc_fail_selfdep = ov_create.pkg_fail.get ( 'bad_selfdeps' ),
+ pc_fail_err = ov_create.pkg_fail.get ( 'exception' ),
+ ec_pre = int ( ov.ebuilds_scanned ),
+ ec_post = int ( ov.ebuild_count ),
+ ec_written = int ( ov.ebuilds_written ),
+ ec_revbump = int ( ov.revbump_count ),
+ )
+ # --- end of make_numstats (...) ---
+
+ def make_timestats ( self ):
+ return ()
+ # --- end of make_timestats (...) ---
+
+ def make_all ( self ):
+ return self.make_numstats() + self.make_timestats()
+ # --- end of make_all (...) ---
+
+ def get_all ( self ):
+ return self._collected_stats
+ # --- end of get_all (...) ---
+
+# --- end of StatsDBCollector #v0 ---
diff --git a/roverlay/stats/rrd.py b/roverlay/stats/rrd.py
new file mode 100644
index 0000000..66da434
--- /dev/null
+++ b/roverlay/stats/rrd.py
@@ -0,0 +1,63 @@
+# R overlay -- stats collection, rrd database (using rrdtool)
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+
+# NOT using rrdtool's python bindings as they're available for python 2 only
+
+import roverlay.db.rrdtool
+from roverlay.db.rrdtool import RRDVariable, RRDArchive
+
+
+class StatsDB ( roverlay.db.rrdtool.RRD ):
+
+ # default step
+ STEP = 300
+
+ def __init__ ( self, filepath, collector, step=None ):
+ # COULDFIX:
+ # vars / RRA creation is only necessary when creating a new database
+ #
+ self.collector = collector
+ self.rrd_vars = self.make_vars()
+ self.rrd_archives = self.make_rra()
+ self.step = step if step is not None else self.__class__.STEP
+ super ( StatsDB, self ).__init__ ( filepath )
+ # --- end of __init__ (...) ---
+
+ def _do_create ( self, filepath ):
+ return self._call_rrdtool (
+ (
+ 'create', filepath,
+ '--start', str ( self.INIT_TIME ),
+ '--step', str ( self.step ),
+ ) + tuple (
+ v.get_key() for v in self.rrd_vars
+ ) + tuple (
+ v.get_key() for v in self.rrd_archives
+ )
+ )
+ # --- end of _do_create (...) ---
+
+ def update ( self ):
+ self.add ( self.collector.get_all() )
+ # --- end of update (...) ---
+
+ def make_vars ( self ):
+ return tuple (
+ RRDVariable ( k, 'DERIVE', val_max=0 )
+ for k in self.collector.NUMSTATS_KEYS
+ )
+ # --- end of make_vars (...) ---
+
+ def make_rra ( self ):
+ return (
+ RRDArchive.new_day ( 'LAST', 0.7 ),
+ RRDArchive.new_week ( 'AVERAGE', 0.7 ),
+ RRDArchive.new_month ( 'AVERAGE', 0.7 ),
+ )
+ # --- end of make_rra (...) ---
+
+# --- end of StatsDB ---
diff --git a/roverlay/stats/collector.py b/roverlay/stats/visualize.py
similarity index 61%
copy from roverlay/stats/collector.py
copy to roverlay/stats/visualize.py
index 3e3dee7..09ca48e 100644
--- a/roverlay/stats/collector.py
+++ b/roverlay/stats/visualize.py
@@ -1,4 +1,4 @@
-# R overlay -- stats collection, stats collector
+# R overlay -- stats collection, print stats
# -*- coding: utf-8 -*-
# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
# Distributed under the terms of the GNU General Public License;
@@ -6,84 +6,31 @@
import collections
-from . import abstract
-from . import base
+import roverlay.util.objects
-class StatsCollector ( abstract.RoverlayStatsBase ):
+class StatsVisualizer ( object ):
- _instance = None
+ def __init__ ( self, stats ):
+ super ( StatsVisualizer, self ).__init__()
+ self.stats = stats
+ self.lines = None
- _MEMBERS = ( 'time', 'repo', 'distmap', 'overlay_creation', 'overlay', )
-
- @classmethod
- def get_instance ( cls ):
- return cls._instance
- # --- end of instance (...) ---
-
- def get_success_ratio ( self ):
- # success ratio for "this" run:
- # new ebuilds / relevant package count (new packages - unsuitable,
- # where unsuitable is e.g. "OS_Type not supported")
- #
- return abstract.SuccessRatio (
- num_ebuilds = self.overlay_creation.pkg_success,
- num_pkg = self.overlay_creation.get_relevant_package_count(),
- )
- # --- end of get_success_ratio (...) ---
-
- def get_overall_success_ratio ( self ):
- # overall success ratio:
- # "relevant ebuild count" / "guessed package count"
- # ratio := <all ebuilds> / (<all ebuilds> + <failed packages>)
- #
- #
- # *Not* accurate as it includes imported ebuilds
- # (Still better than using the repo package count since that may
- # not include old package files)
- #
- return abstract.SuccessRatio (
- num_ebuilds = self.overlay.ebuild_count,
- num_pkg = (
- self.overlay.ebuild_count + self.overlay_creation.pkg_fail
- ),
- )
- # --- end of get_overall_success_ratio (...) ---
-
- def get_net_gain ( self ):
- return (
- self.overlay.ebuild_count - self.overlay.ebuilds_scanned
- )
- # --- end of get_net_gain (...) ---
-
- def __init__ ( self ):
- self.time = abstract.TimeStats ( "misc time stats" )
- self.distmap = base.DistmapStats()
- self.overlay = base.OverlayStats()
- self.overlay_creation = base.OverlayCreationStats()
- self.repo = base.RepoStats()
+ self.prepare()
# --- end of __init__ (...) ---
- def gen_str ( self ):
- yield "{success}, overall {osuccess}".format (
- success = self.get_success_ratio(),
- osuccess = self.get_overall_success_ratio(),
- )
- yield ""
-
- for s in super ( StatsCollector, self ).gen_str():
- yield s
- yield ""
- # --- end of gen_str (...) ---
-
- def get_creation_str ( self ):
- return str ( CreationStatsVisualizer ( self ) )
- # --- end of to_creation_str (...) ---
+ @roverlay.util.objects.not_implemented
+ def prepare ( self ):
+ pass
+ # --- end of prepare (...) ---
-# --- end of StatsCollector ---
+ def __str__ ( self ):
+ return '\n'.join ( self.lines )
+ # --- end of __str__ (...) ---
+# --- end of StatsVisualizer ---
-class CreationStatsVisualizer ( abstract.StatsVisualizer ):
+class CreationStatsVisualizer ( StatsVisualizer ):
def prepare ( self ):
EMPTY_LINE = ""
@@ -206,7 +153,5 @@ class CreationStatsVisualizer ( abstract.StatsVisualizer ):
self.lines = lines
# --- end of gen_str (...) ---
-# --- end of CreationStatsVisualizer ---
-static = StatsCollector()
-StatsCollector._instance = static
+# --- end of CreationStatsVisualizer ---
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-07-29 8:55 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-07-29 8:55 UTC (permalink / raw
To: gentoo-commits
commit: 55537b3dd0f8f1624f882af50f5dc9ea0de65678
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 29 08:54:35 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 29 08:54:35 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=55537b3d
visualize stats (formatted str output)
---
roverlay/stats/abstract.py | 68 +++++++++++++++++++++-
roverlay/stats/collector.py | 137 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 204 insertions(+), 1 deletion(-)
diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
index 36c81ca..6d0575a 100644
--- a/roverlay/stats/abstract.py
+++ b/roverlay/stats/abstract.py
@@ -18,6 +18,26 @@ class MethodNotImplemented ( NotImplementedError ):
# --- end of MethodNotImplemented ---
+class StatsVisualizer ( object ):
+
+ def __init__ ( self, stats ):
+ super ( StatsVisualizer, self ).__init__()
+ self.stats = stats
+ self.lines = None
+
+ self.prepare()
+ # --- end of __init__ (...) ---
+
+ def prepare ( self ):
+ raise MethodNotImplemented ( self, 'prepare' )
+ # --- end of prepare (...) ---
+
+ def __str__ ( self ):
+ return '\n'.join ( self.lines )
+ # --- end of __str__ (...) ---
+
+# --- end of StatsVisualizer ---
+
class RoverlayStatsBase ( object ):
@@ -60,6 +80,15 @@ class RoverlayStatsBase ( object ):
yield getattr ( self, member )
# --- end of _iter_members (...) ---
+ def has_nonzero ( self ):
+ if hasattr ( self, '_MEMBERS' ):
+ for member in self._iter_members():
+ if int ( member ) != 0:
+ return member
+ else:
+ raise MethodNotImplemented ( self, 'has_nonzero' )
+ # --- end of has_nonzero (...) ---
+
def reset_members ( self ):
for member in self._iter_members():
member.reset()
@@ -166,9 +195,29 @@ class TimeStats ( RoverlayStats ):
# --- end of end (...) ---
def get_total ( self ):
- return sum ( v.get_delta() for v in self._timestats.values() )
+ return float (
+ sum ( filter (
+ lambda k: k > 0.0,
+ ( v.get_delta() for v in self._timestats.values() )
+ ) )
+ )
# --- end of get_total (...) ---
+ def get_total_str ( self,
+ unknown_threshold=0.00001, ms_threshold=1.0, min_threshold=300.0,
+ unknown_return=None
+ ):
+ t = self.get_total()
+ if t < unknown_threshold:
+ return unknown_return
+ elif t < ms_threshold:
+ return "{:.2f} ms".format ( t * 1000 )
+ elif t > min_threshold:
+ return "{:.2f} minutes".format ( t / 60.0 )
+ else:
+ return "{:.2f} seconds".format ( t )
+ # --- end of get_total_str (...) ---
+
def gen_str ( self ):
desc = self.get_description_str()
if desc:
@@ -211,6 +260,10 @@ class Counter ( RoverlayStatsBase ):
return self.total_count - int ( other )
# --- end of __sub__ (...) ---
+ def has_details ( self ):
+ return False
+ # --- end of has_details (...) ---
+
def reset ( self ):
self.total_count = 0
self.underflow = False
@@ -256,6 +309,14 @@ class DetailedCounter ( Counter ):
self._detailed_count = collections.defaultdict ( int )
# --- end of __init__ (...) ---
+ def has_details ( self ):
+ return any ( int(v) != 0 for v in self._detailed_count.values() )
+ # --- end of has_details (...) ---
+
+ def iter_details ( self ):
+ return ( kv for kv in self._detailed_count.items() if int(kv[1]) != 0 )
+ # --- end of iter_details (...) ---
+
def __getitem__ ( self, key ):
if key in self._detailed_count:
return self._detailed_count [key]
@@ -350,6 +411,11 @@ class SuccessRatio ( object ):
return float ( self.ebuild_count ) / float ( self.pkg_count )
# --- end of get_ratio (...) ---
+ @property
+ def ratio ( self ):
+ return self.get_ratio()
+ # --- end of ratio (...) ---
+
def __float__ ( self ):
return self.get_ratio()
# --- end of __float__ (...) ---
diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py
index 1b122f4..3e3dee7 100644
--- a/roverlay/stats/collector.py
+++ b/roverlay/stats/collector.py
@@ -4,6 +4,8 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
+import collections
+
from . import abstract
from . import base
@@ -48,6 +50,12 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
)
# --- end of get_overall_success_ratio (...) ---
+ def get_net_gain ( self ):
+ return (
+ self.overlay.ebuild_count - self.overlay.ebuilds_scanned
+ )
+ # --- end of get_net_gain (...) ---
+
def __init__ ( self ):
self.time = abstract.TimeStats ( "misc time stats" )
self.distmap = base.DistmapStats()
@@ -68,8 +76,137 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
yield ""
# --- end of gen_str (...) ---
+ def get_creation_str ( self ):
+ return str ( CreationStatsVisualizer ( self ) )
+ # --- end of to_creation_str (...) ---
# --- end of StatsCollector ---
+
+class CreationStatsVisualizer ( abstract.StatsVisualizer ):
+
+ def prepare ( self ):
+ EMPTY_LINE = ""
+
+ pkg_count = self.stats.repo.pkg_count
+ pkg_queued = self.stats.overlay_creation.pkg_queued
+ pkg_fail = self.stats.overlay_creation.pkg_fail
+ pkg_success = self.stats.overlay_creation.pkg_success
+ ebuild_delta = self.stats.get_net_gain()
+ revbumps = self.stats.overlay.revbump_count
+
+ max_number_len = min (
+ len ( str ( int ( k ) ) ) for k in (
+ pkg_queued, pkg_fail, pkg_success
+ )
+ )
+ max_number_len = min ( 5, max_number_len )
+
+ success_ratio = self.stats.get_success_ratio()
+ overall_success_ratio = self.stats.get_overall_success_ratio()
+
+
+ timestats = (
+ ( 'scan_overlay', self.stats.overlay.scan_time.get_total_str() ),
+ ( 'add_packages', self.stats.repo.queue_time.get_total_str() ),
+ (
+ 'ebuild_creation',
+ self.stats.overlay_creation.creation_time.get_total_str()
+ ),
+ ( 'write_overlay', self.stats.overlay.write_time.get_total_str() ),
+ )
+
+ try:
+ max_time_len = max ( len(k) for k, v in timestats if v is not None )
+ except ValueError:
+ # empty sequence -> no timestats
+ max_time_len = -1
+ else:
+ # necessary?
+ max_time_len = min ( 39, max_time_len )
+
+
+ # create lines
+ lines = collections.deque()
+ unshift = lines.appendleft
+ append = lines.append
+
+ numstats = lambda k, s: "{num:<{l}d} {s}".format (
+ num=int ( k ), s=s, l=max_number_len
+ )
+
+
+ append (
+ 'success ratio {s_i:.2%} (overall {s_o:.2%})'.format (
+ s_i = success_ratio.get_ratio(),
+ s_o = overall_success_ratio.get_ratio()
+ )
+ )
+ append ( EMPTY_LINE )
+
+ append (
+ "{e:+d} ebuilds ({r:d} revbumps)".format (
+ e=ebuild_delta, r=int ( revbumps )
+ )
+ )
+ append ( EMPTY_LINE )
+
+ if int ( pkg_count ) != int ( pkg_queued ):
+ append ( numstats (
+ pkg_queued,
+ '/ {n:d} packages added to the ebuild creation queue'.format (
+ n=int ( pkg_count )
+ )
+ ) )
+ else:
+ append ( numstats (
+ pkg_queued, 'packages added to the ebuild creation queue'
+ ) )
+
+ append ( numstats (
+ pkg_success, 'packages passed ebuild creation'
+ ) )
+
+ append ( numstats (
+ pkg_fail, 'packages failed ebuild creation'
+ ) )
+
+ if pkg_fail.has_details() and int ( pkg_fail ) != 0:
+ append ( EMPTY_LINE )
+ append ( "Details for ebuild creation failure:" )
+ details = sorted (
+ ( ( k, int(v) ) for k, v in pkg_fail.iter_details() ),
+ key=lambda kv: kv[1]
+ )
+ dlen = len ( str ( max ( details, key=lambda kv: kv[1] ) [1] ) )
+
+ for key, value in details:
+ append ( "* {v:>{l}d}: {k}".format ( k=key, v=value, l=dlen ) )
+ # -- end if <have pkg_fail details>
+
+ if max_time_len > 0:
+ # or >= 0
+ append ( EMPTY_LINE )
+ for k, v in timestats:
+ if v is not None:
+ append (
+ "time for {0:<{l}} : {1}".format ( k, v, l=max_time_len )
+ )
+ # -- end if timestats
+
+
+ append ( EMPTY_LINE )
+
+ # add header/footer line(s)
+ max_line_len = 2 + min ( 78, max ( len(s) for s in lines ) )
+ unshift (
+ "{0:-^{1}}\n".format ( " Overlay creation stats ", max_line_len )
+ )
+ append ( max_line_len * '-' )
+
+ self.lines = lines
+ # --- end of gen_str (...) ---
+# --- end of CreationStatsVisualizer ---
+
static = StatsCollector()
StatsCollector._instance = static
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-07-26 13:02 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-07-26 13:02 UTC (permalink / raw
To: gentoo-commits
commit: d1ca42185a44d01176d9944e95a112493401f1f6
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 26 12:57:41 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 26 12:57:41 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d1ca4218
statc collection: data types, time stats
---
roverlay/stats/abstract.py | 125 +++++++++++++++++++++++++++++++++++++++-----
roverlay/stats/base.py | 54 ++++++++++++++-----
roverlay/stats/collector.py | 31 ++++++++---
3 files changed, 177 insertions(+), 33 deletions(-)
diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
index 993d48b..36c81ca 100644
--- a/roverlay/stats/abstract.py
+++ b/roverlay/stats/abstract.py
@@ -26,15 +26,31 @@ class RoverlayStatsBase ( object ):
return cls()
# --- end of get_new (...) ---
+ def __init__ ( self, description=None ):
+ super ( RoverlayStatsBase, self ).__init__()
+ if description is not None:
+ self.description = description
+ # --- end of __init__ (...) ---
+
def merge_with ( self, other ):
- if hasattr ( self, '_MEMBERS' ):
- self.merge_members ( other )
+ my_cls = self.__class__
+ their_cls = other.__class__
+
+ if (
+ issubclass ( my_cls, their_cls )
+ and hasattr ( their_cls, '_MEMBERS' )
+ ):
+ self.merge_members ( other, their_cls._MEMBERS )
+
+ elif hasattr ( my_cls, '_MEMBERS' ):
+ self.merge_members ( other, my_cls._MEMBERS )
+
else:
raise MethodNotImplemented ( self, 'merge_with' )
# --- end of merge_with (...) ---
- def merge_members ( self, other ):
- for member in self.__class__._MEMBERS:
+ def merge_members ( self, other, members ):
+ for member in members:
getattr ( self, member ).merge_with ( getattr ( other, member ) )
# --- end of merge_members (...) ---
@@ -84,21 +100,96 @@ class RoverlayStatsBase ( object ):
class RoverlayStats ( RoverlayStatsBase ):
+ pass
+# --- end of RoverlayStats ---
- def get_time ( self ):
- return time.time()
- # --- end of get_time (...) ---
- def get_time_delta ( self, start, stop=None ):
- t_stop = self.get_time() if stop is None else stop
- return ( start - t_stop, t_stop )
- # --- end of get_time_delta (...) ---
+class TimeStatsItem ( RoverlayStatsBase ):
+ # doc TODO: note somewhere that those timestats are just approximate
+ # values
+
+ def __init__ ( self, t_begin=None, t_end=None, description=None ):
+ super ( TimeStatsItem, self ).__init__ ( description=description )
+ self.time_begin = t_begin if t_begin is not None else time.time()
+ self.time_end = t_end
+ # --- end of __init__ (...) ---
+
+ def end ( self, t_end=None ):
+ self.time_end = time.time() if t_end is None else t_end
+ # --- end of end (...) ---
+
+ def get_delta ( self ):
+ if self.time_begin is None:
+ return -1.0
+ elif self.time_end is None:
+ return -2.0
+ else:
+ return float ( self.time_end ) - float ( self.time_begin )
+ # --- end of get_delta (...) ---
+
+ def __str__ ( self ):
+ return "{:.3f}s".format ( self.get_delta() )
+ # --- end of __str__ (...) ---
+
+# --- end of TimeStatsItem ---
+
+
+class TimeStats ( RoverlayStats ):
+
+ def __init__ ( self, description=None ):
+ super ( TimeStats, self ).__init__ ( description=description )
+ self._timestats = collections.OrderedDict()
+ # --- end of __init__ (...) ---
+
+ def merge_with ( self, other ):
+ self._timestats.update ( other._timestats )
+ # --- end of merge_with (...) ---
+
+ def get ( self, key ):
+ return self._timestats.get ( key, -1.0 )
+ # --- end of get (...) ---
+
+ def __getitem__ ( self, key ):
+ return self._timestats [key]
+ # --- end of __getitem__ (...) ---
+
+ def begin ( self, key ):
+ item = TimeStatsItem()
+ self._timestats [key] = item
+ return item
+ # --- end of begin (...) ---
+
+ def end ( self, key ):
+ item = self._timestats [key]
+ item.end()
+ return item
+ # --- end of end (...) ---
+
+ def get_total ( self ):
+ return sum ( v.get_delta() for v in self._timestats.values() )
+ # --- end of get_total (...) ---
+
+ def gen_str ( self ):
+ desc = self.get_description_str()
+ if desc:
+ yield "{} ({:.3f}s)".format ( desc, self.get_total() )
+ else:
+ yield "total time: {:.3f}s".format ( self.get_total() )
+
+ for key, value in self._timestats.items():
+ yield "* {k}: {v}".format ( k=key, v=str ( value ) )
+ # --- end of gen_str (...) ---
+
+ def __str__ ( self ):
+ return '\n'.join ( self.gen_str() )
+ # --- end of __str__ (...) ---
+
+# --- end of TimeStats ---
-# --- end of RoverlayStats ---
class Counter ( RoverlayStatsBase ):
def __init__ ( self, description=None, initial_value=0 ):
- super ( Counter, self ).__init__()
+ super ( Counter, self ).__init__ ( description=description )
self.description = description
self.total_count = initial_value
self.underflow = False
@@ -112,6 +203,14 @@ class Counter ( RoverlayStatsBase ):
return float ( self.total_count )
# --- end of __float__ (...) ---
+ def __add__ ( self, other ):
+ return self.total_count + int ( other )
+ # --- end of __add__ (...) ---
+
+ def __sub__ ( self, other ):
+ return self.total_count - int ( other )
+ # --- end of __sub__ (...) ---
+
def reset ( self ):
self.total_count = 0
self.underflow = False
diff --git a/roverlay/stats/base.py b/roverlay/stats/base.py
index b03092b..2f13e73 100644
--- a/roverlay/stats/base.py
+++ b/roverlay/stats/base.py
@@ -8,11 +8,13 @@ from . import abstract
class RepoStats ( abstract.RoverlayStats ):
- _MEMBERS = frozenset ({ 'pkg_count', })
+ _MEMBERS = ( 'sync_time', 'queue_time', 'pkg_count', )
def __init__ ( self ):
super ( RepoStats, self ).__init__()
- self.pkg_count = abstract.DetailedCounter (
+ self.sync_time = abstract.TimeStats ( 'sync_time' )
+ self.queue_time = abstract.TimeStats ( 'queue_time' )
+ self.pkg_count = abstract.DetailedCounter (
description="package count"
)
# --- end of __init__ (...) ---
@@ -26,7 +28,7 @@ class RepoStats ( abstract.RoverlayStats ):
class DistmapStats ( abstract.RoverlayStats ):
- _MEMBERS = frozenset ({ 'pkg_count', })
+ _MEMBERS = ( 'pkg_count', )
def __init__ ( self ):
super ( DistmapStats, self ).__init__()
@@ -46,31 +48,56 @@ class DistmapStats ( abstract.RoverlayStats ):
# --- end of DistmapStats ---
-class OverlayCreationStats ( abstract.RoverlayStats ):
+class OverlayCreationWorkerStats ( abstract.RoverlayStats ):
- #_MEMBERS = frozenset({})
+ _MEMBERS = ( 'pkg_processed', 'pkg_fail', 'pkg_success', )
+
+ def __init__ ( self ):
+ self.pkg_processed = abstract.Counter ( "processed" )
+ self.pkg_fail = abstract.DetailedCounter ( "fail" )
+ self.pkg_success = abstract.Counter ( "success" )
+ # --- end of __init__ (...) ---
+
+# --- end of OverlayCreationWorkerStats ---
+
+
+class OverlayCreationStats ( OverlayCreationWorkerStats ):
+
+ DESCRIPTION = "overlay creation"
+
+ _MEMBERS = (
+ ( 'creation_time', 'pkg_queued', 'pkg_filtered', )
+ + OverlayCreationWorkerStats._MEMBERS
+ )
def __init__ ( self ):
super ( OverlayCreationStats, self ).__init__()
+ self.pkg_queued = abstract.Counter ( "queued" )
+ self.pkg_filtered = abstract.Counter ( "filtered" )
+ self.creation_time = abstract.TimeStats ( "ebuild creation" )
# --- end of __init__ (...) ---
def get_relevant_package_count ( self ):
- print ( "get_relevant_package_count(): method stub" )
- return 0
+ return self.pkg_queued
+ #return self.pkg_queued - self.pkg_fail.get ( "empty_desc" )
# --- end of get_relevant_package_count (...) ---
- def __str__ ( self ):
- return "*** no overlay creation stats available ***"
- # --- end of __str__ (...) ---
+ @classmethod
+ def get_new ( cls ):
+ return OverlayCreationWorkerStats()
+ # --- end of get_new (...) ---
# --- end of OverlayCreationStats ---
class OverlayStats ( abstract.RoverlayStats ):
- _MEMBERS = frozenset ({
+ DESCRIPTION = "overlay"
+
+ _MEMBERS = (
+ 'scan_time', 'write_time',
'ebuilds_scanned', 'ebuild_count', 'ebuilds_written',
- })
+ )
def __init__ ( self ):
super ( OverlayStats, self ).__init__()
@@ -81,6 +108,9 @@ class OverlayStats ( abstract.RoverlayStats ):
self.ebuild_count = abstract.Counter ( "post" )
self.ebuilds_written = abstract.Counter ( "written" )
+
+ self.write_time = abstract.TimeStats ( "write_time" )
+ self.scan_time = abstract.TimeStats ( "scan_time" )
# --- end of __init__ (...) ---
# --- end of OverlayStats ---
diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py
index d3c318e..1b122f4 100644
--- a/roverlay/stats/collector.py
+++ b/roverlay/stats/collector.py
@@ -12,7 +12,7 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
_instance = None
- _MEMBERS = [ 'repo', 'distmap', 'overlay_creation', 'overlay', ]
+ _MEMBERS = ( 'time', 'repo', 'distmap', 'overlay_creation', 'overlay', )
@classmethod
def get_instance ( cls ):
@@ -25,35 +25,50 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
# where unsuitable is e.g. "OS_Type not supported")
#
return abstract.SuccessRatio (
- num_ebuilds = self.overlay.ebuild_count.get ( "new" ),
+ num_ebuilds = self.overlay_creation.pkg_success,
num_pkg = self.overlay_creation.get_relevant_package_count(),
)
# --- end of get_success_ratio (...) ---
def get_overall_success_ratio ( self ):
# overall success ratio:
- # all ebuilds / distmap file count
+ # "relevant ebuild count" / "guessed package count"
+ # ratio := <all ebuilds> / (<all ebuilds> + <failed packages>)
#
- # *Not* accurate as it includes imported ebuilds and assumes that
- # each ebuild has one source file in the distmap.
+ #
+ # *Not* accurate as it includes imported ebuilds
# (Still better than using the repo package count since that may
# not include old package files)
#
- # !!! useless. distmap includes "successful" packages only
- #
return abstract.SuccessRatio (
num_ebuilds = self.overlay.ebuild_count,
- num_pkg = self.distmap.pkg_count,
+ num_pkg = (
+ self.overlay.ebuild_count + self.overlay_creation.pkg_fail
+ ),
)
# --- end of get_overall_success_ratio (...) ---
def __init__ ( self ):
+ self.time = abstract.TimeStats ( "misc time stats" )
self.distmap = base.DistmapStats()
self.overlay = base.OverlayStats()
self.overlay_creation = base.OverlayCreationStats()
self.repo = base.RepoStats()
# --- end of __init__ (...) ---
+ def gen_str ( self ):
+ yield "{success}, overall {osuccess}".format (
+ success = self.get_success_ratio(),
+ osuccess = self.get_overall_success_ratio(),
+ )
+ yield ""
+
+ for s in super ( StatsCollector, self ).gen_str():
+ yield s
+ yield ""
+ # --- end of gen_str (...) ---
+
+
# --- end of StatsCollector ---
static = StatsCollector()
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-07-25 15:20 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-07-25 15:20 UTC (permalink / raw
To: gentoo-commits
commit: 4b9358dbf616678865d53b699e4ba6734b00b6d2
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 25 14:31:32 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 25 14:31:32 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=4b9358db
stats collection: data types, overlay stats
---
roverlay/stats/abstract.py | 109 ++++++++++++++++++++++++++++++++------------
roverlay/stats/base.py | 24 ++++++----
roverlay/stats/collector.py | 6 +--
3 files changed, 98 insertions(+), 41 deletions(-)
diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
index b6a4ee9..a8a2200 100644
--- a/roverlay/stats/abstract.py
+++ b/roverlay/stats/abstract.py
@@ -4,9 +4,21 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
+from __future__ import division
+
import collections
import time
+class MethodNotImplemented ( NotImplementedError ):
+ def __init__ ( self, obj, method ):
+ super ( MethodNotImplemented, self ).__init__ (
+ "{n}.{f}()".format ( n=obj.__class__.__name__, f=method )
+ )
+ # --- end of __init__ (...) ---
+
+# --- end of MethodNotImplemented ---
+
+
class RoverlayStatsBase ( object ):
@classmethod
@@ -15,9 +27,17 @@ class RoverlayStatsBase ( object ):
# --- end of get_new (...) ---
def merge_with ( self, other ):
- raise NotImplementedError()
+ if hasattr ( self, '_MEMBERS' ):
+ self.merge_members ( other )
+ else:
+ raise MethodNotImplemented ( self, 'merge_with' )
# --- end of merge_with (...) ---
+ def merge_members ( self, other ):
+ for member in self.__class__._MEMBERS:
+ getattr ( self, member ).merge_with ( getattr ( other, member ) )
+ # --- end of merge_members (...) ---
+
def _iter_members ( self, nofail=False ):
if not nofail or hasattr ( self, '_MEMBERS' ):
for member in self.__class__._MEMBERS:
@@ -33,7 +53,7 @@ class RoverlayStatsBase ( object ):
if hasattr ( self.__class__, '_MEMBERS' ):
self.reset_members()
else:
- raise NotImplementedError()
+ raise MethodNotImplemented ( self, 'reset' )
# --- end of reset (...) ---
def get_description_str ( self ):
@@ -57,7 +77,7 @@ class RoverlayStatsBase ( object ):
if ret:
return ret
else:
- raise NotImplementedError()
+ raise MethodNotImplemented ( self, '__str__' )
# --- end of __str__ (...) ---
# --- end of RoverlayStatsBase ---
@@ -76,14 +96,61 @@ class RoverlayStats ( RoverlayStatsBase ):
# --- end of RoverlayStats ---
+class Counter ( RoverlayStatsBase ):
+ def __init__ ( self, description=None, initial_value=0 ):
+ super ( Counter, self ).__init__()
+ self.description = description
+ self.total_count = initial_value
+ self.underflow = False
+ # --- end of __init__ (...) ---
+
+ def __int__ ( self ):
+ return self.total_count
+ # --- end of __int__ (...) ---
+
+ def __float__ ( self ):
+ return float ( self.total_count )
+ # --- end of __float__ (...) ---
+
+ def reset ( self ):
+ self.total_count = 0
+ self.underflow = False
+ # --- end of reset (...) ---
+
+ def inc ( self, step=1 ):
+ self.total_count += step
+ # --- end of inc (...) ---
+
+ def dec ( self, step=1 ):
+ self.total_count -= step
+ if self.total_count < 0:
+ self.underflow = True
+ # --- end of dec (...) ---
+
+ def merge_with ( self, other ):
+ self.total_count += other.total_count
+ if other.underflow:
+ self.underflow = True
+ # --- end of merge_with (...) ---
+
+ def gen_str ( self ):
+ desc = self.get_description_str()
+ if desc:
+ yield "{}: {:d}".format ( desc, self.total_count )
+ else:
+ yield str ( self.total_count )
+
+ if self.underflow:
+ yield "*** underflow detected ***"
+ # --- end of gen_str (...) ---
+
+# --- end of Counter ---
+
-class DetailedCounter ( RoverlayStatsBase ):
+class DetailedCounter ( Counter ):
def __init__ ( self, description=None ):
- super ( DetailedCounter, self ).__init__()
- self.description = description
- self.total_count = 0
+ super ( DetailedCounter, self ).__init__ ( description=description )
self._detailed_count = collections.defaultdict ( int )
- self.underflow = False
# --- end of __init__ (...) ---
def __getitem__ ( self, key ):
@@ -93,14 +160,6 @@ class DetailedCounter ( RoverlayStatsBase ):
raise KeyError ( key )
# --- end of __getitem__ (...) ---
- def __int__ ( self ):
- return self.total_count
- # --- end of __int__ (...) ---
-
- def __float__ ( self ):
- return float ( self.total_count )
- # --- end of __float__ (...) ---
-
def get ( self, key=None, fallback=0 ):
if key is None:
return self.total_count
@@ -111,8 +170,7 @@ class DetailedCounter ( RoverlayStatsBase ):
# --- end of get (...) ---
def reset ( self ):
- self.total_count = 0
- self.underflow = False
+ super ( DetailedCounter, self ).reset()
self._detailed_count.clear()
# --- end of reset (...) ---
@@ -121,8 +179,8 @@ class DetailedCounter ( RoverlayStatsBase ):
self._detailed_count [k] += 1
# --- end of inc_details_v (...) ---
- def inc ( self, *details ):
- self.total_count += 1
+ def inc ( self, *details, **kw ):
+ super ( DetailedCounter, self ).inc ( **kw )
self.inc_details_v ( details )
# --- end of inc (...) ---
@@ -139,10 +197,8 @@ class DetailedCounter ( RoverlayStatsBase ):
self.underflow = True
# --- end of dec_details_v (...) ---
- def dec ( self, *details ):
- self.total_count -= 1
- if self.total_count < 0:
- self.underflow = True
+ def dec ( self, *details, **kw ):
+ super ( DetailedCounter, self ).dec ( **kw )
self.dec_details_v ( details )
# --- end of dec (...) ---
@@ -151,12 +207,9 @@ class DetailedCounter ( RoverlayStatsBase ):
# --- end of dec_details (...) ---
def merge_with ( self, other ):
- self.total_count += other.totalcount
+ super ( DetailedCounter, self ).merge_with ( other )
for key, value in other._detailed_count.items():
self._detailed_count [key] += value
-
- if other.underflow:
- self.underflow = True
# --- end of merge_with (...) ---
def gen_str ( self ):
diff --git a/roverlay/stats/base.py b/roverlay/stats/base.py
index 6e37581..b03092b 100644
--- a/roverlay/stats/base.py
+++ b/roverlay/stats/base.py
@@ -59,24 +59,28 @@ class OverlayCreationStats ( abstract.RoverlayStats ):
return 0
# --- end of get_relevant_package_count (...) ---
+ def __str__ ( self ):
+ return "*** no overlay creation stats available ***"
+ # --- end of __str__ (...) ---
+
# --- end of OverlayCreationStats ---
class OverlayStats ( abstract.RoverlayStats ):
- _MEMBERS = frozenset({ 'ebuild_count', })
+ _MEMBERS = frozenset ({
+ 'ebuilds_scanned', 'ebuild_count', 'ebuilds_written',
+ })
def __init__ ( self ):
super ( OverlayStats, self ).__init__()
- # ebuild_count counts *physical* ebuilds
- # number of created ebuilds is part of overlay creation stats
- self.ebuild_count = abstract.DetailedCounter (
- description="ebuild count"
- )
- # --- end of __init__ (...) ---
+ # ebuilds_scanned: ebuild count prior to running overlay creation
+ self.ebuilds_scanned = abstract.Counter ( "pre" )
- def ebuild_added ( self, origin ):
- self.ebuild_count.inc ( origin )
- # --- end of ebuild_added (...) ---
+ # ebuild count: ebuild count after writing the overlay
+ self.ebuild_count = abstract.Counter ( "post" )
+
+ self.ebuilds_written = abstract.Counter ( "written" )
+ # --- end of __init__ (...) ---
# --- end of OverlayStats ---
diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py
index fc48d2a..d3c318e 100644
--- a/roverlay/stats/collector.py
+++ b/roverlay/stats/collector.py
@@ -4,8 +4,6 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
-from __future__ import division
-
from . import abstract
from . import base
@@ -14,7 +12,7 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
_instance = None
- _MEMBERS = frozenset ({ 'repo', 'overlay', 'overlay_creation', })
+ _MEMBERS = [ 'repo', 'distmap', 'overlay_creation', 'overlay', ]
@classmethod
def get_instance ( cls ):
@@ -41,6 +39,8 @@ class StatsCollector ( abstract.RoverlayStatsBase ):
# (Still better than using the repo package count since that may
# not include old package files)
#
+ # !!! useless. distmap includes "successful" packages only
+ #
return abstract.SuccessRatio (
num_ebuilds = self.overlay.ebuild_count,
num_pkg = self.distmap.pkg_count,
^ permalink raw reply related [flat|nested] 14+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/
@ 2013-07-24 16:52 André Erdmann
0 siblings, 0 replies; 14+ messages in thread
From: André Erdmann @ 2013-07-24 16:52 UTC (permalink / raw
To: gentoo-commits
commit: a2424732d50d85bfb2bc094bee086af7a79ddd49
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 24 15:54:58 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 24 15:54:58 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a2424732
stats collection: data types
---
roverlay/stats/abstract.py | 209 ++++++++++++++++++++++++++++++++++++++++++++
roverlay/stats/base.py | 82 +++++++++++++++++
roverlay/stats/collector.py | 60 +++++++++++++
3 files changed, 351 insertions(+)
diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py
new file mode 100644
index 0000000..b6a4ee9
--- /dev/null
+++ b/roverlay/stats/abstract.py
@@ -0,0 +1,209 @@
+# R overlay -- stats collection, data types
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import collections
+import time
+
+class RoverlayStatsBase ( object ):
+
+ @classmethod
+ def get_new ( cls ):
+ return cls()
+ # --- end of get_new (...) ---
+
+ def merge_with ( self, other ):
+ raise NotImplementedError()
+ # --- end of merge_with (...) ---
+
+ def _iter_members ( self, nofail=False ):
+ if not nofail or hasattr ( self, '_MEMBERS' ):
+ for member in self.__class__._MEMBERS:
+ yield getattr ( self, member )
+ # --- end of _iter_members (...) ---
+
+ def reset_members ( self ):
+ for member in self._iter_members():
+ member.reset()
+ # --- end of reset_members (...) ---
+
+ def reset ( self ):
+ if hasattr ( self.__class__, '_MEMBERS' ):
+ self.reset_members()
+ else:
+ raise NotImplementedError()
+ # --- end of reset (...) ---
+
+ def get_description_str ( self ):
+ return (
+ getattr ( self, 'description', None ) or
+ getattr ( self, 'DESCRIPTION', None )
+ )
+ # --- end of get_description_str (...) ---
+
+ def gen_str ( self ):
+ desc = self.get_description_str()
+ if desc:
+ yield desc
+
+ for member in self._iter_members( nofail=True ):
+ yield str ( member )
+ # --- end of gen_str (...) ---
+
+ def __str__ ( self ):
+ ret = '\n'.join ( self.gen_str() )
+ if ret:
+ return ret
+ else:
+ raise NotImplementedError()
+ # --- end of __str__ (...) ---
+
+# --- end of RoverlayStatsBase ---
+
+
+class RoverlayStats ( RoverlayStatsBase ):
+
+ def get_time ( self ):
+ return time.time()
+ # --- end of get_time (...) ---
+
+ def get_time_delta ( self, start, stop=None ):
+ t_stop = self.get_time() if stop is None else stop
+ return ( start - t_stop, t_stop )
+ # --- end of get_time_delta (...) ---
+
+# --- end of RoverlayStats ---
+
+
+class DetailedCounter ( RoverlayStatsBase ):
+ def __init__ ( self, description=None ):
+ super ( DetailedCounter, self ).__init__()
+ self.description = description
+ self.total_count = 0
+ self._detailed_count = collections.defaultdict ( int )
+ self.underflow = False
+ # --- end of __init__ (...) ---
+
+ def __getitem__ ( self, key ):
+ if key in self._detailed_count:
+ return self._detailed_count [key]
+ else:
+ raise KeyError ( key )
+ # --- end of __getitem__ (...) ---
+
+ def __int__ ( self ):
+ return self.total_count
+ # --- end of __int__ (...) ---
+
+ def __float__ ( self ):
+ return float ( self.total_count )
+ # --- end of __float__ (...) ---
+
+ def get ( self, key=None, fallback=0 ):
+ if key is None:
+ return self.total_count
+ elif key in self._detailed_count:
+ return self._detailed_count [key]
+ else:
+ return fallback
+ # --- end of get (...) ---
+
+ def reset ( self ):
+ self.total_count = 0
+ self.underflow = False
+ self._detailed_count.clear()
+ # --- end of reset (...) ---
+
+ def inc_details_v ( self, details ):
+ for k in details:
+ self._detailed_count [k] += 1
+ # --- end of inc_details_v (...) ---
+
+ def inc ( self, *details ):
+ self.total_count += 1
+ self.inc_details_v ( details )
+ # --- end of inc (...) ---
+
+ def inc_details ( self, *details ):
+ self.inc_details_v ( details )
+ # --- end of inc_details (...) ---
+
+ def dec_details_v ( self, details ):
+ # do not check / fail if self._detailed_count [k] > 0
+ # simply keep the negative value
+ for k in details:
+ self._detailed_count [k] -= 1
+ if self._detailed_count [k] < 0:
+ self.underflow = True
+ # --- end of dec_details_v (...) ---
+
+ def dec ( self, *details ):
+ self.total_count -= 1
+ if self.total_count < 0:
+ self.underflow = True
+ self.dec_details_v ( details )
+ # --- end of dec (...) ---
+
+ def dec_details ( self, *details ):
+ self.dec_details_v ( Details )
+ # --- end of dec_details (...) ---
+
+ def merge_with ( self, other ):
+ self.total_count += other.totalcount
+ for key, value in other._detailed_count.items():
+ self._detailed_count [key] += value
+
+ if other.underflow:
+ self.underflow = True
+ # --- end of merge_with (...) ---
+
+ def gen_str ( self ):
+ desc = self.get_description_str()
+ if desc:
+ yield desc
+
+ yield "total: {:d}".format ( self.total_count )
+ for key, value in self._detailed_count.items():
+ yield "* {k}: {v:d}".format ( k=key, v=value )
+
+ if self.underflow:
+ yield "*** underflow detected ***"
+ # --- end of gen_str (...) ---
+
+# --- end of DetailedCounter ---
+
+
+class SuccessRatio ( object ):
+
+ def __init__ ( self, num_ebuilds, num_pkg ):
+ super ( SuccessRatio, self ).__init__()
+ self.ebuild_count = int ( num_ebuilds )
+ self.pkg_count = int ( num_pkg )
+ # --- end of __init__ (...) ---
+
+ def get_ratio ( self ):
+ if self.pkg_count == self.ebuild_count:
+ # 0/0, ..., K/K: "success" := 1.0
+ return 1.0
+ elif self.pkg_count < 1:
+ # K/0: bad ratio, use fixed value
+ return -1.0
+ else:
+ return float ( self.ebuild_count ) / float ( self.pkg_count )
+ # --- end of get_ratio (...) ---
+
+ def __float__ ( self ):
+ return self.get_ratio()
+ # --- end of __float__ (...) ---
+
+ def __int__ ( self ):
+ return int ( self.get_ratio() )
+ # --- end of __int__ (...) ---
+
+ def __str__ ( self ):
+ return "{:.3%}".format ( self.get_ratio() )
+ # --- end of __str__ (...) ---
+
+# --- end of SuccessRatio ---
diff --git a/roverlay/stats/base.py b/roverlay/stats/base.py
new file mode 100644
index 0000000..6e37581
--- /dev/null
+++ b/roverlay/stats/base.py
@@ -0,0 +1,82 @@
+# R overlay -- stats collection, data types
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+from . import abstract
+
+class RepoStats ( abstract.RoverlayStats ):
+
+ _MEMBERS = frozenset ({ 'pkg_count', })
+
+ def __init__ ( self ):
+ super ( RepoStats, self ).__init__()
+ self.pkg_count = abstract.DetailedCounter (
+ description="package count"
+ )
+ # --- end of __init__ (...) ---
+
+ def package_file_found ( self, repo ):
+ self.pkg_count.inc ( repo.name )
+ # --- end of add_package (...) ---
+
+# --- end of RepoStats ---
+
+
+class DistmapStats ( abstract.RoverlayStats ):
+
+ _MEMBERS = frozenset ({ 'pkg_count', })
+
+ def __init__ ( self ):
+ super ( DistmapStats, self ).__init__()
+ self.pkg_count = abstract.DetailedCounter (
+ description="distmap package count"
+ )
+ # --- end of __init__ (...) ---
+
+ def file_added ( self, *origin ):
+ self.pkg_count.inc ( *origin )
+ # --- end of file_added (...) ---
+
+ def file_removed ( self, *origin ):
+ self.pkg_count.dec ( *origin )
+ # --- end of file_removed (...) ---
+
+# --- end of DistmapStats ---
+
+
+class OverlayCreationStats ( abstract.RoverlayStats ):
+
+ #_MEMBERS = frozenset({})
+
+ def __init__ ( self ):
+ super ( OverlayCreationStats, self ).__init__()
+ # --- end of __init__ (...) ---
+
+ def get_relevant_package_count ( self ):
+ print ( "get_relevant_package_count(): method stub" )
+ return 0
+ # --- end of get_relevant_package_count (...) ---
+
+# --- end of OverlayCreationStats ---
+
+
+class OverlayStats ( abstract.RoverlayStats ):
+
+ _MEMBERS = frozenset({ 'ebuild_count', })
+
+ def __init__ ( self ):
+ super ( OverlayStats, self ).__init__()
+ # ebuild_count counts *physical* ebuilds
+ # number of created ebuilds is part of overlay creation stats
+ self.ebuild_count = abstract.DetailedCounter (
+ description="ebuild count"
+ )
+ # --- end of __init__ (...) ---
+
+ def ebuild_added ( self, origin ):
+ self.ebuild_count.inc ( origin )
+ # --- end of ebuild_added (...) ---
+
+# --- end of OverlayStats ---
diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py
new file mode 100644
index 0000000..fc48d2a
--- /dev/null
+++ b/roverlay/stats/collector.py
@@ -0,0 +1,60 @@
+# R overlay -- stats collection, stats collector
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+from __future__ import division
+
+from . import abstract
+from . import base
+
+
+class StatsCollector ( abstract.RoverlayStatsBase ):
+
+ _instance = None
+
+ _MEMBERS = frozenset ({ 'repo', 'overlay', 'overlay_creation', })
+
+ @classmethod
+ def get_instance ( cls ):
+ return cls._instance
+ # --- end of instance (...) ---
+
+ def get_success_ratio ( self ):
+ # success ratio for "this" run:
+ # new ebuilds / relevant package count (new packages - unsuitable,
+ # where unsuitable is e.g. "OS_Type not supported")
+ #
+ return abstract.SuccessRatio (
+ num_ebuilds = self.overlay.ebuild_count.get ( "new" ),
+ num_pkg = self.overlay_creation.get_relevant_package_count(),
+ )
+ # --- end of get_success_ratio (...) ---
+
+ def get_overall_success_ratio ( self ):
+ # overall success ratio:
+ # all ebuilds / distmap file count
+ #
+ # *Not* accurate as it includes imported ebuilds and assumes that
+ # each ebuild has one source file in the distmap.
+ # (Still better than using the repo package count since that may
+ # not include old package files)
+ #
+ return abstract.SuccessRatio (
+ num_ebuilds = self.overlay.ebuild_count,
+ num_pkg = self.distmap.pkg_count,
+ )
+ # --- end of get_overall_success_ratio (...) ---
+
+ def __init__ ( self ):
+ self.distmap = base.DistmapStats()
+ self.overlay = base.OverlayStats()
+ self.overlay_creation = base.OverlayCreationStats()
+ self.repo = base.RepoStats()
+ # --- end of __init__ (...) ---
+
+# --- end of StatsCollector ---
+
+static = StatsCollector()
+StatsCollector._instance = static
^ permalink raw reply related [flat|nested] 14+ messages in thread
end of thread, other threads:[~2014-02-21 17:37 UTC | newest]
Thread overview: 14+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-07-25 14:28 [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/ André Erdmann
-- strict thread matches above, loose matches on Subject: below --
2014-02-21 17:36 André Erdmann
2013-09-05 10:24 André Erdmann
2013-09-05 9:25 André Erdmann
2013-09-05 9:25 André Erdmann
2013-08-16 12:42 André Erdmann
2013-08-16 10:43 André Erdmann
2013-08-15 9:18 André Erdmann
2013-08-02 10:34 André Erdmann
2013-07-29 14:56 André Erdmann
2013-07-29 8:55 André Erdmann
2013-07-26 13:02 André Erdmann
2013-07-25 15:20 André Erdmann
2013-07-24 16:52 André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox