From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id C2C981381F3 for ; Wed, 24 Jul 2013 16:52:39 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 3CA8AE0A90; Wed, 24 Jul 2013 16:52:39 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 794D4E0A90 for ; Wed, 24 Jul 2013 16:52:38 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 4F54433DBC6 for ; Wed, 24 Jul 2013 16:52:37 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id A2123E5468 for ; Wed, 24 Jul 2013 16:52:34 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1374681298.a2424732d50d85bfb2bc094bee086af7a79ddd49.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/stats/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/stats/abstract.py roverlay/stats/base.py roverlay/stats/collector.py X-VCS-Directories: roverlay/stats/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: a2424732d50d85bfb2bc094bee086af7a79ddd49 X-VCS-Branch: master Date: Wed, 24 Jul 2013 16:52:34 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 5dc6b837-42b1-4639-8b2c-f1e10cd663fc X-Archives-Hash: 1a89a50ccebad2913c4cb6ae414c919f commit: a2424732d50d85bfb2bc094bee086af7a79ddd49 Author: André Erdmann mailerd de> AuthorDate: Wed Jul 24 15:54:58 2013 +0000 Commit: André Erdmann mailerd de> CommitDate: Wed Jul 24 15:54:58 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a2424732 stats collection: data types --- roverlay/stats/abstract.py | 209 ++++++++++++++++++++++++++++++++++++++++++++ roverlay/stats/base.py | 82 +++++++++++++++++ roverlay/stats/collector.py | 60 +++++++++++++ 3 files changed, 351 insertions(+) diff --git a/roverlay/stats/abstract.py b/roverlay/stats/abstract.py new file mode 100644 index 0000000..b6a4ee9 --- /dev/null +++ b/roverlay/stats/abstract.py @@ -0,0 +1,209 @@ +# R overlay -- stats collection, data types +# -*- coding: utf-8 -*- +# Copyright (C) 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + +import collections +import time + +class RoverlayStatsBase ( object ): + + @classmethod + def get_new ( cls ): + return cls() + # --- end of get_new (...) --- + + def merge_with ( self, other ): + raise NotImplementedError() + # --- end of merge_with (...) --- + + def _iter_members ( self, nofail=False ): + if not nofail or hasattr ( self, '_MEMBERS' ): + for member in self.__class__._MEMBERS: + yield getattr ( self, member ) + # --- end of _iter_members (...) --- + + def reset_members ( self ): + for member in self._iter_members(): + member.reset() + # --- end of reset_members (...) --- + + def reset ( self ): + if hasattr ( self.__class__, '_MEMBERS' ): + self.reset_members() + else: + raise NotImplementedError() + # --- end of reset (...) --- + + def get_description_str ( self ): + return ( + getattr ( self, 'description', None ) or + getattr ( self, 'DESCRIPTION', None ) + ) + # --- end of get_description_str (...) --- + + def gen_str ( self ): + desc = self.get_description_str() + if desc: + yield desc + + for member in self._iter_members( nofail=True ): + yield str ( member ) + # --- end of gen_str (...) --- + + def __str__ ( self ): + ret = '\n'.join ( self.gen_str() ) + if ret: + return ret + else: + raise NotImplementedError() + # --- end of __str__ (...) --- + +# --- end of RoverlayStatsBase --- + + +class RoverlayStats ( RoverlayStatsBase ): + + def get_time ( self ): + return time.time() + # --- end of get_time (...) --- + + def get_time_delta ( self, start, stop=None ): + t_stop = self.get_time() if stop is None else stop + return ( start - t_stop, t_stop ) + # --- end of get_time_delta (...) --- + +# --- end of RoverlayStats --- + + +class DetailedCounter ( RoverlayStatsBase ): + def __init__ ( self, description=None ): + super ( DetailedCounter, self ).__init__() + self.description = description + self.total_count = 0 + self._detailed_count = collections.defaultdict ( int ) + self.underflow = False + # --- end of __init__ (...) --- + + def __getitem__ ( self, key ): + if key in self._detailed_count: + return self._detailed_count [key] + else: + raise KeyError ( key ) + # --- end of __getitem__ (...) --- + + def __int__ ( self ): + return self.total_count + # --- end of __int__ (...) --- + + def __float__ ( self ): + return float ( self.total_count ) + # --- end of __float__ (...) --- + + def get ( self, key=None, fallback=0 ): + if key is None: + return self.total_count + elif key in self._detailed_count: + return self._detailed_count [key] + else: + return fallback + # --- end of get (...) --- + + def reset ( self ): + self.total_count = 0 + self.underflow = False + self._detailed_count.clear() + # --- end of reset (...) --- + + def inc_details_v ( self, details ): + for k in details: + self._detailed_count [k] += 1 + # --- end of inc_details_v (...) --- + + def inc ( self, *details ): + self.total_count += 1 + self.inc_details_v ( details ) + # --- end of inc (...) --- + + def inc_details ( self, *details ): + self.inc_details_v ( details ) + # --- end of inc_details (...) --- + + def dec_details_v ( self, details ): + # do not check / fail if self._detailed_count [k] > 0 + # simply keep the negative value + for k in details: + self._detailed_count [k] -= 1 + if self._detailed_count [k] < 0: + self.underflow = True + # --- end of dec_details_v (...) --- + + def dec ( self, *details ): + self.total_count -= 1 + if self.total_count < 0: + self.underflow = True + self.dec_details_v ( details ) + # --- end of dec (...) --- + + def dec_details ( self, *details ): + self.dec_details_v ( Details ) + # --- end of dec_details (...) --- + + def merge_with ( self, other ): + self.total_count += other.totalcount + for key, value in other._detailed_count.items(): + self._detailed_count [key] += value + + if other.underflow: + self.underflow = True + # --- end of merge_with (...) --- + + def gen_str ( self ): + desc = self.get_description_str() + if desc: + yield desc + + yield "total: {:d}".format ( self.total_count ) + for key, value in self._detailed_count.items(): + yield "* {k}: {v:d}".format ( k=key, v=value ) + + if self.underflow: + yield "*** underflow detected ***" + # --- end of gen_str (...) --- + +# --- end of DetailedCounter --- + + +class SuccessRatio ( object ): + + def __init__ ( self, num_ebuilds, num_pkg ): + super ( SuccessRatio, self ).__init__() + self.ebuild_count = int ( num_ebuilds ) + self.pkg_count = int ( num_pkg ) + # --- end of __init__ (...) --- + + def get_ratio ( self ): + if self.pkg_count == self.ebuild_count: + # 0/0, ..., K/K: "success" := 1.0 + return 1.0 + elif self.pkg_count < 1: + # K/0: bad ratio, use fixed value + return -1.0 + else: + return float ( self.ebuild_count ) / float ( self.pkg_count ) + # --- end of get_ratio (...) --- + + def __float__ ( self ): + return self.get_ratio() + # --- end of __float__ (...) --- + + def __int__ ( self ): + return int ( self.get_ratio() ) + # --- end of __int__ (...) --- + + def __str__ ( self ): + return "{:.3%}".format ( self.get_ratio() ) + # --- end of __str__ (...) --- + +# --- end of SuccessRatio --- diff --git a/roverlay/stats/base.py b/roverlay/stats/base.py new file mode 100644 index 0000000..6e37581 --- /dev/null +++ b/roverlay/stats/base.py @@ -0,0 +1,82 @@ +# R overlay -- stats collection, data types +# -*- coding: utf-8 -*- +# Copyright (C) 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + +from . import abstract + +class RepoStats ( abstract.RoverlayStats ): + + _MEMBERS = frozenset ({ 'pkg_count', }) + + def __init__ ( self ): + super ( RepoStats, self ).__init__() + self.pkg_count = abstract.DetailedCounter ( + description="package count" + ) + # --- end of __init__ (...) --- + + def package_file_found ( self, repo ): + self.pkg_count.inc ( repo.name ) + # --- end of add_package (...) --- + +# --- end of RepoStats --- + + +class DistmapStats ( abstract.RoverlayStats ): + + _MEMBERS = frozenset ({ 'pkg_count', }) + + def __init__ ( self ): + super ( DistmapStats, self ).__init__() + self.pkg_count = abstract.DetailedCounter ( + description="distmap package count" + ) + # --- end of __init__ (...) --- + + def file_added ( self, *origin ): + self.pkg_count.inc ( *origin ) + # --- end of file_added (...) --- + + def file_removed ( self, *origin ): + self.pkg_count.dec ( *origin ) + # --- end of file_removed (...) --- + +# --- end of DistmapStats --- + + +class OverlayCreationStats ( abstract.RoverlayStats ): + + #_MEMBERS = frozenset({}) + + def __init__ ( self ): + super ( OverlayCreationStats, self ).__init__() + # --- end of __init__ (...) --- + + def get_relevant_package_count ( self ): + print ( "get_relevant_package_count(): method stub" ) + return 0 + # --- end of get_relevant_package_count (...) --- + +# --- end of OverlayCreationStats --- + + +class OverlayStats ( abstract.RoverlayStats ): + + _MEMBERS = frozenset({ 'ebuild_count', }) + + def __init__ ( self ): + super ( OverlayStats, self ).__init__() + # ebuild_count counts *physical* ebuilds + # number of created ebuilds is part of overlay creation stats + self.ebuild_count = abstract.DetailedCounter ( + description="ebuild count" + ) + # --- end of __init__ (...) --- + + def ebuild_added ( self, origin ): + self.ebuild_count.inc ( origin ) + # --- end of ebuild_added (...) --- + +# --- end of OverlayStats --- diff --git a/roverlay/stats/collector.py b/roverlay/stats/collector.py new file mode 100644 index 0000000..fc48d2a --- /dev/null +++ b/roverlay/stats/collector.py @@ -0,0 +1,60 @@ +# R overlay -- stats collection, stats collector +# -*- coding: utf-8 -*- +# Copyright (C) 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + +from __future__ import division + +from . import abstract +from . import base + + +class StatsCollector ( abstract.RoverlayStatsBase ): + + _instance = None + + _MEMBERS = frozenset ({ 'repo', 'overlay', 'overlay_creation', }) + + @classmethod + def get_instance ( cls ): + return cls._instance + # --- end of instance (...) --- + + def get_success_ratio ( self ): + # success ratio for "this" run: + # new ebuilds / relevant package count (new packages - unsuitable, + # where unsuitable is e.g. "OS_Type not supported") + # + return abstract.SuccessRatio ( + num_ebuilds = self.overlay.ebuild_count.get ( "new" ), + num_pkg = self.overlay_creation.get_relevant_package_count(), + ) + # --- end of get_success_ratio (...) --- + + def get_overall_success_ratio ( self ): + # overall success ratio: + # all ebuilds / distmap file count + # + # *Not* accurate as it includes imported ebuilds and assumes that + # each ebuild has one source file in the distmap. + # (Still better than using the repo package count since that may + # not include old package files) + # + return abstract.SuccessRatio ( + num_ebuilds = self.overlay.ebuild_count, + num_pkg = self.distmap.pkg_count, + ) + # --- end of get_overall_success_ratio (...) --- + + def __init__ ( self ): + self.distmap = base.DistmapStats() + self.overlay = base.OverlayStats() + self.overlay_creation = base.OverlayCreationStats() + self.repo = base.RepoStats() + # --- end of __init__ (...) --- + +# --- end of StatsCollector --- + +static = StatsCollector() +StatsCollector._instance = static