From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id C94F81381F3 for ; Wed, 14 Aug 2013 14:56:31 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id AC74EE0B79; Wed, 14 Aug 2013 14:56:28 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id E7F40E0B64 for ; Wed, 14 Aug 2013 14:56:27 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id BFCAD33EC6D for ; Wed, 14 Aug 2013 14:56:26 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 5093DE546B for ; Wed, 14 Aug 2013 14:56:24 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1376491272.4d26054ebd65c210cf75c5f6257c8db9ca00560e.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/, roverlay/stats/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/db/rrdtool.py roverlay/stats/rrd.py X-VCS-Directories: roverlay/db/ roverlay/stats/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: 4d26054ebd65c210cf75c5f6257c8db9ca00560e X-VCS-Branch: master Date: Wed, 14 Aug 2013 14:56:24 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 7ef7efed-d3df-45cd-bb97-914928aaee50 X-Archives-Hash: 9e295ff296af0db4a406a42308b0a1fd commit: 4d26054ebd65c210cf75c5f6257c8db9ca00560e Author: André Erdmann mailerd de> AuthorDate: Wed Aug 14 14:41:12 2013 +0000 Commit: André Erdmann mailerd de> CommitDate: Wed Aug 14 14:41:12 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=4d26054e fixup rrd database creation * normalize/balance steps/rows when creating RRAs so that values can actually be used for updating them * create the database file exactly once (!) * _call_rrdtool() log command that is run * let rrdtool control timestamps by default --- roverlay/db/rrdtool.py | 60 ++++++++++++++++++++++++++++++++++++++------------ roverlay/stats/rrd.py | 6 ++--- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/roverlay/db/rrdtool.py b/roverlay/db/rrdtool.py index 76bded7..98c3bf5 100644 --- a/roverlay/db/rrdtool.py +++ b/roverlay/db/rrdtool.py @@ -25,7 +25,8 @@ class RRDVariable ( object ): DST = frozenset ({ 'GAUGE', 'COUNTER', 'DERIVE', 'ABSOLUTE', }) def __init__ ( self, - name, val_type, val_min=None, val_max=None, heartbeat=None, step=300 + name, val_type, val_min=None, val_max=None, heartbeat=None, step=300, + heartbeat_factor=2, ): if val_type not in self.DST: raise ValueError ( "invalid DS type: {!r}".format ( val_type ) ) @@ -34,7 +35,7 @@ class RRDVariable ( object ): self.val_type = val_type self.val_min = val_min self.val_max = val_max - self.heartbeat = heartbeat or ( 2 * step ) + self.heartbeat = heartbeat or ( heartbeat_factor * step ) # --- end of __init__ (...) --- def get_key ( self ): @@ -84,25 +85,49 @@ class RRDArchive ( object ): __str__ = get_key @classmethod + def _balance_cdp_rows ( cls, step, cdp_time, row_count, min_row_count=2 ): + my_cdp_time = cdp_time + my_rows = row_count + my_minrows = min_row_count - 1 + + while step > my_cdp_time and my_rows > my_minrows: + my_cdp_time *= 2 + my_rows //= 2 + + if step <= my_cdp_time and my_rows > my_minrows: + return ( my_cdp_time // step, my_rows ) + else: + raise Exception ( "cannot get steps/rows." ) + # --- end of _balance_cdp_rows (...) --- + + @classmethod + def get_new ( cls, cf, xff, step, cdp_time, row_count ): + steps, rows = cls._balance_cdp_rows ( step, cdp_time, row_count ) + return cls ( cf, xff, steps, rows ) + # --- end of get_new (...) --- + + + @classmethod def new_day ( cls, cf, xff, step=300 ): - # one CDP per hour (24 rows) - return cls ( cf, xff, 3600 // step, 24 ) + # by default, one CDP per hour (24 rows) + return cls.get_new ( cf, xff, step, 3600, 24 ) # --- end of new_day (...) --- @classmethod def new_week ( cls, cf, xff, step=300 ): # one CDP per 6h (28 rows) - return cls ( cf, xff, 21600 // step, 42 ) + return cls.get_new ( cf, xff, step, 21600, 28 ) # --- end of new_week (...) --- @classmethod def new_month ( cls, cf, xff, step=300 ): # one CDP per day (31 rows) - return cls ( cf, xff, (24*3600) // step, 31 ) + return cls.get_new ( cf, xff, step, 86400, 31 ) # --- end of new_month (...) --- # --- end of RRDArchive --- + class RRD ( object ): # should be subclassed 'cause _do_create() is not implemented here @@ -151,10 +176,15 @@ class RRD ( object ): ) # --- end of _call_rrdtool_command (...) --- - def _call_rrdtool ( self, args, return_success=True, get_output=False ): + def _call_rrdtool ( self, + args, return_success=True, get_output=False, binary_stdout=False + ): + cmdv = self.RRDTOOL_CMDV_HEAD + args + self.logger.info ( "calling rrdtool: {!r}".format ( cmdv ) ) + if get_output: cmd_call, output = run_command_get_output ( - self.RRDTOOL_CMDV_HEAD + args, None + cmdv, env=None, binary_stdout=binary_stdout ) if output[1]: logger = self.logger.getChild ( 'rrdtool_call' ) @@ -166,8 +196,7 @@ class RRD ( object ): else: return run_command ( - self.RRDTOOL_CMDV_HEAD + args, None, self.logger, - return_success=return_success + cmdv, env=None, logger=self.logger, return_success=return_success ) # --- end of _call_rrdtool (...) --- @@ -203,8 +232,11 @@ class RRD ( object ): # --- end of check_readonly (...) --- def create_if_missing ( self ): - if self._dbfile_exists or not os.access ( self.filepath, os.F_OK ): - return self.create() + if not self._dbfile_exists: + if os.access ( self.filepath, os.F_OK ): + self._dbfile_exists = True + else: + self.create() # --- end of create_if_missing (...) --- def create ( self ): @@ -218,9 +250,9 @@ class RRD ( object ): # --- end of create (...) --- def add ( self, values, timestamp=None ): - if timestamp is False: + if timestamp is None: t = 'N' - elif timestamp is None: + elif timestamp is True: t = str ( self.time_now() ) else: t = str ( timestamp ) diff --git a/roverlay/stats/rrd.py b/roverlay/stats/rrd.py index 2f9cce3..ffccbe1 100644 --- a/roverlay/stats/rrd.py +++ b/roverlay/stats/rrd.py @@ -28,7 +28,7 @@ class StatsDB ( roverlay.db.rrdtool.RRD ): return self._call_rrdtool ( ( 'create', filepath, - '--start', str ( self.INIT_TIME ), + ##'--start', str ( self.INIT_TIME ), '--step', str ( self.step ), ) + tuple ( v.get_key() for v in self.rrd_vars ) @@ -43,14 +43,14 @@ class StatsDB ( roverlay.db.rrdtool.RRD ): def make_vars ( self ): heartbeat = 2 * self.step return tuple ( - RRDVariable ( k, 'DERIVE', val_max=0, heartbeat=heartbeat ) + RRDVariable ( k, 'DERIVE', val_min=0, heartbeat=heartbeat ) for k in self.collector.NUMSTATS_KEYS ) # --- end of make_vars (...) --- def make_rra ( self ): return ( - #RRDArchive.new_day ( 'LAST', 0.7, step=self.step ), + RRDArchive.new_day ( 'LAST', 0.7, step=self.step ), RRDArchive.new_week ( 'AVERAGE', 0.7, step=self.step ), RRDArchive.new_month ( 'AVERAGE', 0.7, step=self.step ), )