* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
2013-06-22 15:14 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
@ 2013-06-22 15:24 ` André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-06-22 15:24 UTC (permalink / raw
To: gentoo-commits
commit: 75694bb74ee537993c03601ef96a31e9406d55d2
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Jun 22 10:07:11 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Jun 22 10:07:11 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=75694bb7
distmap: don't revbump if repo names don't match
Given a package P that is available from two repos A and B, roverlay would add
P_A ("P from A") as -r0 and P_B as -r1 (or vice versa).
In subsequent runs, P_A_new's checksum would be compared to P_B and P_B_new's
checksum to P_A_new, resulting in a revbump for both packages even if there are
no real changes (P_A == P_A_new and/or P_B == P_B_new).
Prior to supporting revbumps, roverlay would not accept P_B.
This commit restores this behavior.
---
roverlay/db/distmap.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index dfdcd01..2af3e57 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -165,6 +165,10 @@ class _DistMapBase ( object ):
if info is None:
# new file, no revbump required
return False
+ elif info.repo_name != package_info['origin'].name:
+ # don't revbump if repo names don't match, this likely results in
+ # infinite revbumps if a package is available from more than one repo
+ return False
elif info.compare_digest ( package_info ) [0] is True:
# old digest == new digest, no revbump
# (package_info should be filtered out)
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
2013-07-10 15:10 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
@ 2013-07-10 16:16 ` André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-07-10 16:16 UTC (permalink / raw
To: gentoo-commits
commit: a62e98887135863e2dce19e77eb97958f54d72a9
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 14:26:00 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 14:26:00 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a62e9888
roverlay/db/distmap: add todo note
---
roverlay/db/distmap.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 3ada05a..6d2e3b5 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -19,6 +19,8 @@ import roverlay.util
__all__ = [ 'DistMapInfo', 'get_distmap' ]
+#COULDFIX: distmap could use roverlay.util.fileio
+
class DistMapInfo ( object ):
"""Distmap entry"""
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-07-30 18:40 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-07-30 18:40 UTC (permalink / raw
To: gentoo-commits
commit: 3b92458a280855f69a6aabc8a6c84e4868562444
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 30 16:02:12 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 30 16:02:12 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=3b92458a
roverlay/db: rrdtool.py
---
roverlay/db/rrdtool.py | 176 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 176 insertions(+)
diff --git a/roverlay/db/rrdtool.py b/roverlay/db/rrdtool.py
new file mode 100644
index 0000000..6d450fe
--- /dev/null
+++ b/roverlay/db/rrdtool.py
@@ -0,0 +1,176 @@
+# R overlay -- stats collection, wrapper for writing rrd databases
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+from __future__ import division
+
+# NOT using rrdtool's python bindings as they're available for python 2 only
+
+import logging
+import os
+import time
+
+import roverlay.tools.runcmd
+from roverlay.tools.runcmd import run_command
+
+import roverlay.util
+
+import roverlay.util.objects
+
+class RRDVariable ( object ):
+
+ DST = frozenset ({ 'GAUGE', 'COUNTER', 'DERIVE', 'ABSOLUTE', })
+
+ def __init__ ( self,
+ name, val_type, val_min=None, val_max=None, heartbeat=None, step=300
+ ):
+ if val_type not in self.DST:
+ raise ValueError ( "invalid DS type: {!r}".format ( val_type ) )
+
+ self.name = name
+ self.val_type = val_type
+ self.val_min = val_min
+ self.val_max = val_max
+ self.heartbeat = heartbeat or ( 2 * step )
+ # --- end of __init__ (...) ---
+
+ def get_key ( self ):
+ mstr = lambda k: 'U' if k is None else str ( k )
+
+ return "DS:{ds_name}:{DST}:{heartbeat}:{vmin}:{vmax}".format (
+ ds_name=self.name, DST=self.val_type, heartbeat=self.heartbeat,
+ vmin=mstr ( self.val_min ), vmax=mstr ( self.val_max )
+ )
+ # --- end of get_key (...) ---
+
+ __str__ = get_key
+
+# --- end of RRDVariable ---
+
+class RRDArchive ( object ):
+
+ CF_TYPES = frozenset ({ 'AVERAGE', 'MIN', 'MAX', 'LAST', })
+
+ def __init__ ( self, cf, xff, steps, rows ):
+ if cf not in self.CF_TYPES:
+ raise ValueError ( "unknown CF: {!r}".format ( cf ) )
+ elif not isinstance ( xff, float ):
+ raise TypeError ( "xff must be a float." )
+ elif xff < 0.0 or xff >= 1.0:
+ raise ValueError (
+ "xff not in range: 0.0 <= {:f} <= 1.0?".format ( xff )
+ )
+ elif not isinstance ( steps, int ) or steps <= 0:
+ raise ValueError ( "steps must be an integer > 0." )
+
+ elif not isinstance ( rows, int ) or rows <= 0:
+ raise ValueError ( "rows must be an integer > 0." )
+
+ self.cf = cf
+ self.xff = float ( xff )
+ self.steps = int ( steps )
+ self.rows = int ( rows )
+ # --- end of __init__ (...) ---
+
+ def get_key ( self ):
+ return "RRA:{cf}:{xff}:{steps}:{rows}".format (
+ cf=self.cf, xff=self.xff, steps=self.steps, rows=self.rows
+ )
+ # --- end of get_key (...) ---
+
+ __str__ = get_key
+
+ @classmethod
+ def new_day ( cls, cf, xff, step=300 ):
+ # one CDP per hour (24 rows)
+ return cls ( cf, xff, 3600 // step, 24 )
+ # --- end of new_day (...) ---
+
+ @classmethod
+ def new_week ( cls, cf, xff, step=300 ):
+ # one CDP per 6h (28 rows)
+ return cls ( cf, xff, 21600 // step, 42 )
+ # --- end of new_week (...) ---
+
+ @classmethod
+ def new_month ( cls, cf, xff, step=300 ):
+ # one CDP per day (31 rows)
+ return cls ( cf, xff, (24*3600) // step, 31 )
+ # --- end of new_month (...) ---
+
+# --- end of RRDArchive ---
+
+class RRD ( object ):
+ # should be subclassed 'cause _do_create() is not implemented here
+
+ RRDTOOL_CMDV_HEAD = ( 'rrdtool', )
+
+ LOGGER = logging.getLogger ( 'RRD' )
+
+ def __init__ ( self, filepath ):
+ self.filepath = filepath
+ self._commit_buffer = []
+ self._dbfile_exists = False
+ self.logger = self.__class__.LOGGER
+ self.INIT_TIME = self.time_now() - 10
+ # --- end of __init__ (...) ---
+
+ def time_now ( self ):
+ return int ( time.time() )
+ # --- end of time_now (...) ---
+
+ def _call_rrdtool ( self, args, return_success=True ):
+ return run_command (
+ self.RRDTOOL_CMDV_HEAD + args, None, self.logger,
+ return_success=return_success
+ )
+ # --- end of _call_rrdtool (...) ---
+
+ @roverlay.util.objects.abstractmethod
+ def _do_create ( self, filepath ):
+ pass
+ # --- end of _do_create (...) ---
+
+ def create_if_missing ( self ):
+ if self._dbfile_exists or not os.access ( self.filepath, os.F_OK ):
+ return self.create()
+ # --- end of create_if_missing (...) ---
+
+ def create ( self ):
+ roverlay.util.dodir_for_file ( self.filepath )
+ self._do_create ( self.filepath )
+ if os.access ( self.filepath, os.F_OK ):
+ self._dbfile_exists = True
+ else:
+ raise Exception ( "database file does not exist." )
+ # --- end of create (...) ---
+
+ def add ( self, values, timestamp=None ):
+ if timestamp is False:
+ t = 'N'
+ elif timestamp is None:
+ t = str ( self.time_now() )
+ else:
+ t = str ( timestamp )
+
+ self._commit_buffer.append (
+ t + ':' + ':'.join ( str(v) for v in values )
+ )
+ # --- end of add (...) ---
+
+ def clear ( self ):
+ self._commit_buffer = []
+ # --- end of clear (...) ---
+
+ def commit ( self ):
+ if self._commit_buffer:
+ self.create_if_missing()
+ self._call_rrdtool (
+ ( 'update', self.filepath ) + tuple ( self._commit_buffer )
+ )
+ self.clear()
+ # --- end of commit (...) ---
+
+# --- end of RRD ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-13 8:56 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-13 8:56 UTC (permalink / raw
To: gentoo-commits
commit: 52fb4a9c2e71de6a02c75194c405037c33ba0a3c
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Aug 13 08:45:08 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Aug 13 08:48:59 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=52fb4a9c
roverlay/db/rrdtool: rudimentary db reading
Read stats from the database. Using "rrdtool dump" (-> xml dump) as that's
easier to extend (read/parser RRA etc.; compared to calling rrdtool multiple
times).
Additionally, this commit adds some read-only checks to the RRD database object.
---
roverlay/db/rrdtool.py | 106 +++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 99 insertions(+), 7 deletions(-)
diff --git a/roverlay/db/rrdtool.py b/roverlay/db/rrdtool.py
index 6d450fe..e21f592 100644
--- a/roverlay/db/rrdtool.py
+++ b/roverlay/db/rrdtool.py
@@ -12,11 +12,12 @@ import logging
import os
import time
+import xml.etree.ElementTree
+
import roverlay.tools.runcmd
-from roverlay.tools.runcmd import run_command
+from roverlay.tools.runcmd import run_command, run_command_get_output
import roverlay.util
-
import roverlay.util.objects
class RRDVariable ( object ):
@@ -109,23 +110,65 @@ class RRD ( object ):
LOGGER = logging.getLogger ( 'RRD' )
- def __init__ ( self, filepath ):
+ def __init__ ( self, filepath, readonly=False ):
+ super ( RRD, self ).__init__()
+ self.readonly = bool ( readonly )
self.filepath = filepath
self._commit_buffer = []
self._dbfile_exists = False
self.logger = self.__class__.LOGGER
self.INIT_TIME = self.time_now() - 10
+ self.cache = None
# --- end of __init__ (...) ---
def time_now ( self ):
return int ( time.time() )
# --- end of time_now (...) ---
- def _call_rrdtool ( self, args, return_success=True ):
- return run_command (
- self.RRDTOOL_CMDV_HEAD + args, None, self.logger,
- return_success=return_success
+ def get_xml_dump ( self ):
+ return self._get_rrdtool_output ( "dump" )
+ # --- end of get_xml_dump (...) ---
+
+ def _get_rrdtool_output ( self, command, *args ):
+ retcode, output = self._call_rrdtool_command (
+ command, args, return_success=True, get_output=True
)
+ if retcode == os.EX_OK:
+ return output[0]
+ else:
+ return None
+ # --- end of _get_rrdtool_output (...) ---
+
+ def _call_rrdtool_command (
+ self, command, args, return_success=True, get_output=False
+ ):
+ """Creates an arg tuple ( command, <stats db file>, *args ) and
+ calls _call_rrdtool() afterwards.
+ """
+ return self._call_rrdtool (
+ ( command, self.filepath, ) + args,
+ return_success=return_success, get_output=get_output
+ )
+ # --- end of _call_rrdtool_command (...) ---
+
+ def _call_rrdtool ( self, args, return_success=True, get_output=False ):
+ if get_output:
+ cmd_call, output = run_command_get_output (
+ self.RRDTOOL_CMDV_HEAD + args, None
+ )
+ if output[1]:
+ logger = self.logger.getChild ( 'rrdtool_call' )
+ for line in output[1]:
+ logger.warning ( line )
+ return (
+ cmd_call.returncode if return_success else cmd_call, output
+ )
+
+ else:
+ return run_command (
+ self.RRDTOOL_CMDV_HEAD + args, None, self.logger,
+ return_success=return_success
+ )
# --- end of _call_rrdtool (...) ---
@roverlay.util.objects.abstractmethod
@@ -133,12 +176,39 @@ class RRD ( object ):
pass
# --- end of _do_create (...) ---
+ def check_readonly ( self,
+ raise_exception=True, error_msg=None, error_msg_append=True
+ ):
+ """Verifies that database writing is allowed.
+
+ Returns True if writing is allowed, else False.
+
+ Raises an Exception if the database is readonly and raise_exception
+ evaluates to True.
+ """
+ if self.readonly:
+ if raise_exception:
+ msg = error_msg
+ if error_msg_append:
+ if msg:
+ msg += " - database is read-only"
+ else:
+ msg = "database is read-only"
+
+ raise Exception ( msg )
+ else:
+ return False
+ else:
+ return True
+ # --- end of check_readonly (...) ---
+
def create_if_missing ( self ):
if self._dbfile_exists or not os.access ( self.filepath, os.F_OK ):
return self.create()
# --- end of create_if_missing (...) ---
def create ( self ):
+ self.check_readonly()
roverlay.util.dodir_for_file ( self.filepath )
self._do_create ( self.filepath )
if os.access ( self.filepath, os.F_OK ):
@@ -165,6 +235,7 @@ class RRD ( object ):
# --- end of clear (...) ---
def commit ( self ):
+ self.check_readonly()
if self._commit_buffer:
self.create_if_missing()
self._call_rrdtool (
@@ -173,4 +244,25 @@ class RRD ( object ):
self.clear()
# --- end of commit (...) ---
+ def make_cache ( self, mask=-1, clear_cache=False ):
+ if clear_cache or not self.cache:
+ self.cache = dict()
+
+ xml_dump = self.get_xml_dump()
+ if not xml_dump:
+ return False
+
+ eroot = xml.etree.ElementTree.fromstring ( '\n'.join ( xml_dump ) )
+
+ self.cache ['lastupdate'] = eroot.find ( "./lastupdate" ).text.strip()
+ self.cache ['values'] = {
+ node.find ( "name" ).text.strip():
+ node.find ( "last_ds" ).text.strip()
+ for node in eroot.findall ( "./ds" )
+ }
+
+
+ return True
+ # --- end of make_cache (...) ---
+
# --- end of RRD ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-14 14:56 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-14 14:56 UTC (permalink / raw
To: gentoo-commits
commit: dee7555a9e0f9cfa59c4fbad3e7d9a50f0192a9a
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Aug 13 12:05:12 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Aug 13 12:05:12 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=dee7555a
roverlay/db/rrdtool: convert values when importing
---
roverlay/db/rrdtool.py | 28 +++++++++++++++++++++++-----
1 file changed, 23 insertions(+), 5 deletions(-)
diff --git a/roverlay/db/rrdtool.py b/roverlay/db/rrdtool.py
index e21f592..76bded7 100644
--- a/roverlay/db/rrdtool.py
+++ b/roverlay/db/rrdtool.py
@@ -245,6 +245,20 @@ class RRD ( object ):
# --- end of commit (...) ---
def make_cache ( self, mask=-1, clear_cache=False ):
+ def convert_value ( str_value, value_type ):
+ try:
+ if value_type == 'GAUGE':
+ ret = float ( str_value )
+ elif value_type in { 'COUNTER', 'DERIVE', 'ABSOLUTE' }:
+ ret = int ( str_value )
+ else:
+ ret = str_value
+ except ValueError:
+ return str_value
+ else:
+ return ret
+ # --- end of convert_value (...) ---
+
if clear_cache or not self.cache:
self.cache = dict()
@@ -255,12 +269,16 @@ class RRD ( object ):
eroot = xml.etree.ElementTree.fromstring ( '\n'.join ( xml_dump ) )
self.cache ['lastupdate'] = eroot.find ( "./lastupdate" ).text.strip()
- self.cache ['values'] = {
- node.find ( "name" ).text.strip():
- node.find ( "last_ds" ).text.strip()
- for node in eroot.findall ( "./ds" )
- }
+ self.cache ['values'] = dict()
+ for ds_node in eroot.findall ( "./ds" ):
+ ds_name = ds_node.find ( "name" ).text.strip()
+ ds_type = ds_node.find ( "type" ).text.strip()
+ if ds_type != 'COMPUTE':
+ ds_value = ds_node.find ( "last_ds" ).text.strip()
+ self.cache ['values'] [ds_name] = convert_value (
+ ds_value, ds_type
+ )
return True
# --- end of make_cache (...) ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-14 14:56 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-14 14:56 UTC (permalink / raw
To: gentoo-commits
commit: cd4a553f69b3da9519310ba5f528aac8541f5bd9
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Aug 14 14:46:17 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Aug 14 14:46:17 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=cd4a553f
roverlay/db/rrdgraph: graph creation
This module provides an interface for calling "rrdtool graphv".
It features include:
* a "protocol" for parsing rrdtool graphv's output into a dict
* objects representing data/graph statements (RRDGraphArg_*)
work in progress -- module is not "stable" yet.
---
roverlay/db/rrdgraph.py | 520 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 520 insertions(+)
diff --git a/roverlay/db/rrdgraph.py b/roverlay/db/rrdgraph.py
new file mode 100644
index 0000000..7464165
--- /dev/null
+++ b/roverlay/db/rrdgraph.py
@@ -0,0 +1,520 @@
+# R overlay -- stats collection, rrdtool graph creation
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import os
+import sys
+import time
+
+import roverlay.util.objects
+
+
+# a few colors for drawing graphs (as str)
+# RRGGBB[AA], rgb+alpha
+COLORS = {
+ 'white' : 'ffffff',
+ 'silver' : 'c0c0c0',
+ 'gray' : '808080',
+ 'grey' : '808080',
+ 'black' : '000000',
+ 'red' : 'ff0000',
+ 'maroon' : '800000',
+ 'yellow' : 'ffff00',
+ 'olive' : '808000',
+ 'lime' : '00ff00',
+ 'green' : '008000',
+ 'aqua' : '00ffff',
+ 'teal' : '008080',
+ 'blue' : '0000ff',
+ 'navy' : '000080',
+ 'fuchsia' : 'ff00ff',
+ 'purple' : '800080',
+}
+
+
+class RRDGraphArg ( object ):
+
+ def gen_options ( self, options, no_key=False ):
+ for attr_name in options:
+ #attr = getattr ( self, attr_name, None )
+ attr = getattr ( self, attr_name.lower().replace ( '-', '_' ) )
+ if attr is not None:
+ if no_key:
+ yield ':' + str ( attr )
+ elif attr is True:
+ yield ':' + attr_name
+ else:
+ yield ':{k}={v}'.format ( k=attr_name, v=str ( attr ) )
+ # --- end of gen_options (...) ---
+
+ def get_options_str ( self, *options ):
+ return ''.join ( self.gen_options ( options, no_key=False ) )
+ # --- end of get_options_str (...) ---
+
+ def get_options_str_nokey ( self, *options ):
+ return ''.join ( self.gen_options ( options, no_key=True ) )
+ # --- end of get_options_str_nokey (...) ---
+
+
+ @roverlay.util.objects.abstractmethod
+ def get_str ( self, rrd_db ):
+ pass
+ # --- end of get_str (...) ---
+
+ def get_color_suffix ( self, color ):
+ if color is None:
+ return ''
+ elif color in COLORS:
+ return '#' + COLORS [color]
+ else:
+ return '#' + str ( color ).lstrip ( '#' )
+ # --- end of get_color_suffix (...) ---
+
+# --- end of RRDGraphArg ---
+
+class RRDGraphArg_DEF ( RRDGraphArg ):
+
+ def __init__ ( self,
+ name, ds_name, cf, rrdfile=None, step=None, start=None, stop=None
+ ):
+ super ( RRDGraphArg_DEF, self ).__init__()
+ self.name = name
+ self.rrdfile = rrdfile
+ self.ds_name = ds_name
+ self.cf = cf
+ self.step = step
+ self.start = start
+ self.stop = stop
+ # --- end of __init__ (...) ---
+
+ def get_str ( self, rrd_db ):
+ rrdfile = self.rrdfile if self.rrdfile is not None else rrd_db.filepath
+ return (
+ "DEF:{vname}={rrdfile}:{ds_name}:{cf}".format (
+ vname=self.name, rrdfile=rrdfile, ds_name=self.ds_name, cf=self.cf
+ )
+ + self.get_options_str ( 'step', 'start', 'stop' )
+ )
+ # --- end of get_str (...) ---
+
+# --- end of RRDGraphArg_DEF ---
+
+class RRDGraphArg_CDEF ( RRDGraphArg ):
+
+ DEFTYPE = 'CDEF'
+
+ def __init__ ( self, name, rpn, expression=None ):
+ super ( RRDGraphArg_CDEF, self ).__init__()
+ self.name = name
+ self.rpn = rpn
+ self.expression = expression
+ # --- end of __init__ (...) ---
+
+ def get_str ( self, rrd_db ):
+ if self.expression is None:
+ return "{deftype}:{vname}={rpn}".format (
+ deftype=self.__class__.DEFTYPE, vname=self.name, rpn=self.rpn
+ )
+ else:
+ return "{deftype}:{vname}={rpn} {expr}".format (
+ deftype=self.__class__.DEFTYPE,
+ vname=self.name, rpn=self.rpn, expr=self.expression
+ )
+ # --- end of get_str (...) ---
+
+# --- end of RRDGraphArg_CDEF ---
+
+class RRDGraphArg_VDEF ( RRDGraphArg_CDEF ):
+ DEFTYPE = 'VDEF'
+# --- end of RRDGraphArg_VDEF ---
+
+class RRDGraphArg_AREA ( RRDGraphArg ):
+
+ def __init__ ( self,
+ value, color=None, legend=None, stack=None, skipscale=None
+ ):
+ super ( RRDGraphArg_AREA, self ).__init__()
+ self.value = value
+ self.color = color
+ self.legend = legend
+ self.stack = stack
+ self.skipscale = skipscale
+ # --- end of __init__ (...) ---
+
+ def get_str ( self, rrd_db ):
+ return (
+ "AREA:{value}{vcolor}".format (
+ value=self.value, vcolor=self.get_color_suffix ( self.color )
+ )
+ + self.get_options_str_nokey ( 'legend' )
+ + self.get_options_str ( 'STACK', 'skipscale' )
+ )
+ # --- end of get_str (...) ---
+
+# --- end of RRDGraphArg_AREA ---
+
+class RRDGraphArg_LINE ( RRDGraphArg ):
+
+ def __init__ ( self,
+ value, color=None, width=None, legend=None, stack=None, skipscale=None,
+ dashes=None, dash_offset=None,
+ ):
+ super ( RRDGraphArg_LINE, self ).__init__()
+ self.width = width
+ self.value = value
+ self.color = color
+ self.legend = legend
+ self.stack = stack
+ self.skipscale = skipscale
+ self.dashes = dashes
+ self.dash_offset = dash_offset
+ # --- end of __init__ (...) ---
+
+ def get_str ( self, rrd_db ):
+ if self.width is not None:
+ if isinstance ( self.width, str ):
+ width = self.width
+ else:
+ width = str ( float ( self.width ) )
+ else:
+ width = ""
+
+ width
+ return (
+ 'LINE' + width + ':{value}{vcolor}'.format (
+ value=self.value, vcolor=self.get_color_suffix ( self.color ),
+ ) + self.get_options_str_nokey ( 'legend' )
+ + self.get_options_str (
+ 'STACK', 'skipscale', 'dashes', 'dash-offset'
+ )
+ )
+ # --- end of get_str (...) ---
+
+# --- end of RRDGraphArg_AREA ---
+
+class RRDGraphArg_PRINT ( RRDGraphArg ):
+
+ ARG_TYPE = 'PRINT'
+
+ def __init__ ( self, name, format_str=None ):
+ super ( RRDGraphArg_PRINT, self ).__init__()
+ self.name = name
+ if format_str is None:
+ self.format_str = '%.0lf %S'
+ else:
+ self.format_str = format_str
+ # --- end of __init__ (...) ---
+
+ def get_str ( self, rrd_db ):
+ return self.ARG_TYPE + ':' + self.name + ':' + self.format_str
+ # --- end of get_str (...) ---
+
+# --- end of RRDGraphArg_PRINT ---
+
+class RRDGraphArg_GPRINT ( RRDGraphArg_PRINT ):
+ ARG_TYPE = 'GPRINT'
+# --- end of RRDGraphArg_GPRINT ---
+
+
+class RRDGraph ( object ):
+
+ GRAPH_COMMAND = 'graphv'
+
+ SECONDS_MINUTE = 60
+ SECONDS_HOUR = SECONDS_MINUTE*60
+ SECONDS_DAY = SECONDS_HOUR*24
+ SECONDS_WEEK = SECONDS_DAY*7
+ SECONDS_MONTH = SECONDS_DAY*31
+ SECONDS_YEAR = SECONDS_DAY*365
+
+ OPTIONS_ATTR = (
+ 'start', 'end', 'imgformat', 'title', 'vertical_label'
+ )
+
+ def __init__ ( self, rrd_db, **kwargs ):
+ super ( RRDGraph, self ).__init__()
+ self.image_data = None
+ self.rrd_db = rrd_db
+ self.extra_options = list()
+ self.args = list()
+
+ self.start = None
+ self.end = None
+ self.imgformat = 'PNG'
+ self.title = None
+ self.vertical_label = None
+ self.colors = []
+
+ self.__dict__.update ( kwargs )
+ # --- end of __init__ (...) ---
+
+ def get_image ( self ):
+ if self.image_data:
+ return self.image_data ['image_data']
+ else:
+ return None
+ # --- end of get_image (...) ---
+
+ def get_timespan ( self, delta, stop=None, delta_factor=1.0 ):
+ t_stop = time.time() - 1 if stop is None else stop
+ return ( int ( t_stop - ( delta_factor * delta ) ), int ( t_stop ) )
+ # --- end of get_timespan (...) ---
+
+ def set_timespan ( self, delta, stop=None, delta_factor=SECONDS_DAY, recursive=False ):
+ self.start, self.end = self.get_timespan (
+ delta, stop=stop, delta_factor=delta_factor
+ )
+ if recursive:
+ for arg in self.args:
+ if isinstance ( arg, RRDGraphArg ):
+ if hasattr ( arg, 'start' ):
+ arg.start = self.start
+ if hasattr ( arg, 'stop' ):
+ arg.stop = self.end
+ # --- end of set_timespan (...) ---
+
+ def add_arg ( self, arg ):
+ self.args.append ( arg )
+ return arg
+ # --- end of add_arg (...) ---
+
+ def add_args ( self, *args ):
+ self.args.extend ( args )
+ # --- end of add_args (...) ---
+
+ def add_def ( self, name, ds_name, cf, step=None ):
+ return self.add_arg (
+ RRDGraphArg_DEF (
+ name=name, ds_name=ds_name, cf=cf, rrdfile=None, step=step,
+ )
+ )
+ # --- end of add_def (...) ---
+
+ def add_cdef ( self, name, rpn, expression=None ):
+ return self.add_arg (
+ RRDGraphArg_CDEF ( name=name, rpn=rpn, expression=expression )
+ )
+ # --- end of add_cdef (...) ---
+
+ def add_vdef ( self, name, rpn, expression=None ):
+ return self.add_arg (
+ RRDGraphArg_VDEF ( name=name, rpn=rpn, expression=expression )
+ )
+ # --- end of add_vdef (...) ---
+
+ def add_line (
+ self, value, color, width=None, legend=None, extra_kwargs={}
+ ):
+ return self.add_arg (
+ RRDGraphArg_LINE (
+ value=value, color=color, width=width, legend=legend,
+ **extra_kwargs
+ )
+ )
+ # --- end of add_line (...) ---
+
+ def add_area (
+ self, value, color, legend=None, stack=None, skipscale=None
+ ):
+ return self.add_arg (
+ RRDGraphArg_AREA (
+ value=value, color=color, legend=legend,
+ stack=stack, skipscale=skipscale
+ )
+ )
+ # --- end of add_area (...) ---
+
+ def add_print ( self, name, format_str=None, inline=False ):
+ if inline:
+ return self.add_arg ( RRDGraphArg_GPRINT ( name, format_str ) )
+ else:
+ return self.add_arg ( RRDGraphArg_PRINT ( name, format_str ) )
+ # --- end of add_print (...) ---
+
+ def add_def_line (
+ self, name, ds_name, cf, color, step=None,
+ width=None, legend=None, line_kwargs={}
+ ):
+ self.add_def ( name, ds_name, cf, step )
+ self.add_line ( name, color, width, legend, line_kwargs )
+ # --- end of add_def_line (...) ---
+
+ def add_def_area (
+ self, name, ds_name, cf, color, step=None,
+ legend=None, stack=None, skipscale=None
+ ):
+ self.add_def ( name, ds_name, cf, step ),
+ self.add_area ( name, color, legend, stack, skipscale )
+ # --- end of add_def_area (...) ---
+
+ def add_option ( self, option, *values ):
+ if values:
+ for value in values:
+ self.extra_options.append ( option )
+ self.extra_options.append ( value )
+ else:
+ self.extra_options.append ( option )
+ # --- end of add_option (...) ---
+
+ def construct_args ( self ):
+ def gen_args():
+ yield self.GRAPH_COMMAND
+ yield '-'
+
+ for attr_name in self.OPTIONS_ATTR:
+ #attr = getattr ( self, attr_name, None )
+ attr = getattr ( self, attr_name )
+ if attr:
+ yield '--' + attr_name.replace ( '_', '-' )
+ yield str ( attr )
+ # -- end for
+
+ for color_spec in self.colors:
+ yield '--color'
+ yield str ( color_spec )
+
+ for arg in self.extra_options:
+ yield str ( arg )
+
+ for arg in self.args:
+ if isinstance ( arg, RRDGraphArg ):
+ yield arg.get_str ( self.rrd_db )
+ else:
+ yield str ( arg )
+ # --- end of gen_args (...) ---
+
+ return tuple ( gen_args() )
+ # --- end of construct_args (...) ---
+
+ def make ( self ):
+ def parse_output ( data, include_raw_data=False ):
+ # output format:
+ # <key> = <value>\n
+ # special case:
+ # image = BLOB_SIZE:<size>\n<image data>
+ #
+
+ WHITESPACE = ' '
+ EQUAL = '='
+ NEWLINE = '\n'
+
+ # mode / data receive "protocol":
+ # 0 -> read key,
+ # 1 -> have first whitespace,
+ # 2 -> have equal sign,
+ # 3 -> have second whitespace <=> want value,
+ # newline during 3 -> commit key/value, reset to mode 0
+ #
+ mode = 0
+ key = ""
+ value = None
+ get_chr = chr if sys.hexversion >= 0x3000000 else ( lambda b: b )
+
+
+ for index, b in enumerate ( data ):
+ c = get_chr ( b )
+ if mode == 0:
+ if c == WHITESPACE:
+ # done with key
+ mode += 1
+ else:
+ # append to key
+ key += c
+ elif mode == 1:
+ if c == EQUAL:
+ mode += 1
+ else:
+ raise Exception (
+ "expected equal sign after whitespace."
+ )
+ elif mode == 2:
+ if c == WHITESPACE:
+ mode += 1
+ value = ""
+ else:
+ raise Exception (
+ "expected whitesapce after equal sign."
+ )
+ elif c == NEWLINE:
+ mode = 0
+
+ if key == 'image':
+ img_size = int ( value.partition ( 'BLOB_SIZE:' ) [-1] )
+ img_data = data[index+1:index+img_size+1]
+ if len ( img_data ) == img_size:
+ yield ( "image_data", img_data )
+ else:
+ raise Exception (
+ "cannot read image (got {} out of {} bytes).".format (
+ len ( img_data ), img_size
+ )
+ )
+
+ yield ( "image_info", value )
+ yield ( "image_size", img_size )
+
+ # *** BREAK ***
+ break
+ else:
+ try:
+ if key in { 'value_min', 'value_max' }:
+ c_val = float ( value )
+ else:
+ c_val = int ( value )
+ ival = int ( value )
+ except ValueError:
+ yield ( key, value )
+ else:
+ yield ( key, c_val )
+
+ key = ""
+ value = None
+ else:
+ value += c
+ # -- end for
+
+ if include_raw_data:
+ yield ( "raw_data", data )
+
+ # --- end of parse_output (...) ---
+
+ retcode, output = self.rrd_db._call_rrdtool (
+ self.construct_args(), return_success=True, get_output=True,
+ binary_stdout=True,
+ )
+
+ if retcode == os.EX_OK:
+ self.image_data = dict ( parse_output ( output[0] ) )
+ return True
+ else:
+ # discard output
+ return False
+ # --- end of make (...) ---
+
+# --- end of RRDGraph ---
+
+class RRDGraphFactory ( object ):
+
+ def __init__ ( self, rrd_db, graph_kwargs=None ):
+ super ( RRDGraphFactory, self ).__init__()
+ self.rrd_db = rrd_db
+ self.graph_kwargs = dict() if graph_kwargs is None else graph_kwargs
+ self.default_args = list()
+ self.default_options = list()
+ # --- end of __init__ (...) ---
+
+ def get_new ( self ):
+ graph = RRDGraph ( self.rrd_db, **self.graph_kwargs )
+
+ if self.default_args:
+ graph.args.extend ( self.default_args )
+
+ if self.default_options:
+ graph.extra_options.extend ( self.default_options )
+
+ return graph
+ # --- end of get_new (...) ---
+
+# --- end of RRDGraphFactory ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-15 9:18 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-15 9:18 UTC (permalink / raw
To: gentoo-commits
commit: bd68f9287ede316dfdb9a76d23a72249df8ed3cc
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug 15 09:16:01 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug 15 09:16:01 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=bd68f928
roverlay/db/rrdtool: more accurate xml parsing
---
roverlay/db/rrdtool.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/roverlay/db/rrdtool.py b/roverlay/db/rrdtool.py
index 98c3bf5..d3c9d6d 100644
--- a/roverlay/db/rrdtool.py
+++ b/roverlay/db/rrdtool.py
@@ -304,10 +304,10 @@ class RRD ( object ):
self.cache ['values'] = dict()
for ds_node in eroot.findall ( "./ds" ):
- ds_name = ds_node.find ( "name" ).text.strip()
- ds_type = ds_node.find ( "type" ).text.strip()
+ ds_name = ds_node.find ( "./name" ).text.strip()
+ ds_type = ds_node.find ( "./type" ).text.strip()
if ds_type != 'COMPUTE':
- ds_value = ds_node.find ( "last_ds" ).text.strip()
+ ds_value = ds_node.find ( "./last_ds" ).text.strip()
self.cache ['values'] [ds_name] = convert_value (
ds_value, ds_type
)
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-16 12:42 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-16 12:42 UTC (permalink / raw
To: gentoo-commits
commit: cb7c0eddabe5e7211d51efa7779f1fac344288b4
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 16 12:36:10 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 16 12:36:10 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=cb7c0edd
roverlay/db/rrdtool: handle UNKNOWN values
---
roverlay/db/rrdtool.py | 26 ++++++++++++++++----------
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/roverlay/db/rrdtool.py b/roverlay/db/rrdtool.py
index d3c9d6d..bd98492 100644
--- a/roverlay/db/rrdtool.py
+++ b/roverlay/db/rrdtool.py
@@ -20,6 +20,7 @@ from roverlay.tools.runcmd import run_command, run_command_get_output
import roverlay.util
import roverlay.util.objects
+
class RRDVariable ( object ):
DST = frozenset ({ 'GAUGE', 'COUNTER', 'DERIVE', 'ABSOLUTE', })
@@ -131,6 +132,8 @@ class RRDArchive ( object ):
class RRD ( object ):
# should be subclassed 'cause _do_create() is not implemented here
+ KNOWN_UNKNOWN = frozenset ({ 'U', 'UNKNOWN', '-nan', 'nan', })
+
RRDTOOL_CMDV_HEAD = ( 'rrdtool', )
LOGGER = logging.getLogger ( 'RRD' )
@@ -278,17 +281,20 @@ class RRD ( object ):
def make_cache ( self, mask=-1, clear_cache=False ):
def convert_value ( str_value, value_type ):
- try:
- if value_type == 'GAUGE':
- ret = float ( str_value )
- elif value_type in { 'COUNTER', 'DERIVE', 'ABSOLUTE' }:
- ret = int ( str_value )
- else:
- ret = str_value
- except ValueError:
- return str_value
+ if str_value.lower() in self.KNOWN_UNKNOWN:
+ return None
else:
- return ret
+ try:
+ if value_type == 'GAUGE':
+ ret = float ( str_value )
+ elif value_type in { 'COUNTER', 'DERIVE', 'ABSOLUTE' }:
+ ret = int ( str_value )
+ else:
+ ret = str_value
+ except ValueError:
+ return str_value
+ else:
+ return ret
# --- end of convert_value (...) ---
if clear_cache or not self.cache:
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-16 14:26 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-16 14:26 UTC (permalink / raw
To: gentoo-commits
commit: 9a6828ffdf99b117a0c16b17f2b5ab04b2defe32
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 16 14:25:47 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 16 14:25:47 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9a6828ff
roverlay/db/rrdtool: read GAUGE as int by default
---
roverlay/db/rrdtool.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/roverlay/db/rrdtool.py b/roverlay/db/rrdtool.py
index b326e7d..0e3e9b6 100644
--- a/roverlay/db/rrdtool.py
+++ b/roverlay/db/rrdtool.py
@@ -279,14 +279,14 @@ class RRD ( object ):
self.clear()
# --- end of commit (...) ---
- def make_cache ( self, mask=-1, clear_cache=False ):
+ def make_cache ( self, mask=-1, clear_cache=False, gauge_type=int ):
def convert_value ( str_value, value_type ):
if str_value.lower() in self.KNOWN_UNKNOWN:
return None
else:
try:
if value_type == 'GAUGE':
- ret = float ( str_value )
+ ret = gauge_type ( str_value )
elif value_type in { 'COUNTER', 'DERIVE', 'ABSOLUTE' }:
ret = int ( str_value )
else:
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-22 9:01 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-22 9:01 UTC (permalink / raw
To: gentoo-commits
commit: 95b98aaa2a04218edef90fb5da223773f92f35b6
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug 22 08:52:58 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug 22 08:52:58 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=95b98aaa
roverlay/db/distmap: accept precalculated digests
This allows to calculate digests in a hashpool.
---
roverlay/db/distmap.py | 44 ++++++++++++++++++++++++++++++--------------
1 file changed, 30 insertions(+), 14 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 34e5dcb..ca3d236 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -233,6 +233,10 @@ class _DistMapBase ( object ):
return True
# --- end of compare_digest (...) ---
+ def get_hash_type ( self ):
+ return DistMapInfo.DIGEST_TYPE
+ # --- end of get_hash_types (...) ---
+
def get_file_digest ( self, f ):
"""Returns a file checksum for the given file.
@@ -242,6 +246,20 @@ class _DistMapBase ( object ):
return roverlay.digest.dodigest_file ( f, DistMapInfo.DIGEST_TYPE )
# --- end of get_file_digest (...) ---
+ def check_digest_integrity ( self, distfile, digest ):
+ info = self._distmap.get ( distfile, None )
+
+ if info is None:
+ # file not found
+ return 1
+ elif info.digest == digest:
+ # file OK
+ return 0
+ else:
+ # bad checksum
+ return 2
+ # --- end of check_digest_integrity (...) ---
+
def check_integrity ( self, distfile, distfilepath ):
"""Verifies a distfile by comparing its filepath with the distmap entry.
Returns 1 if the file is not in the distmap, 2 if the file's checksum
@@ -251,17 +269,12 @@ class _DistMapBase ( object ):
* distfile -- distfile path relative to the distroot
* distfilepath -- absolute path to the distfile
"""
- info = self._distmap.get ( distfile, None )
-
- if info is None:
- # file not found
+ if self._distmap.get ( distfile, None ) is None:
return 1
- elif info.digest == self.get_file_digest ( distfilepath ):
- # file OK
- return 0
else:
- # bad checksum
- return 2
+ return self.check_digest_integrity (
+ distfile, self.get_file_digest ( distfilepath )
+ )
# --- end of check_integrity (...) ---
def remove ( self, key ):
@@ -364,7 +377,7 @@ class _DistMapBase ( object ):
)
# --- end of add_entry_for (...) ---
- def add_dummy_entry ( self, distfile, distfilepath ):
+ def add_dummy_entry ( self, distfile, distfilepath=None, hashdict=None ):
"""Adds a dummy entry.
Such an entry contains a checksum and a distfile, but no information
about its origin (repo name/file).
@@ -372,12 +385,15 @@ class _DistMapBase ( object ):
arguments:
* distfile -- distfile path relative to the distroot
* distfilepath -- absolute path to the distfile
+ * hashdict -- dict with already calculated hashes
"""
+ if hashdict and DistMapInfo.DIGEST_TYPE in hashdict:
+ digest = hashdict [DistMapInfo.DIGEST_TYPE]
+ else:
+ digest = self.get_file_digest ( distfilepath )
+
return self.add_entry (
- distfile,
- DistMapInfo (
- distfile, None, None, self.get_file_digest ( distfilepath ),
- )
+ distfile, DistMapInfo ( distfile, None, None, digest )
)
# --- end of add_dummy_entry (...) ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-30 14:49 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-30 14:49 UTC (permalink / raw
To: gentoo-commits
commit: bec81d9137f5b68398401e79df094207680eb006
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 30 12:42:18 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 30 12:42:18 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=bec81d91
distmap: add_distfile_owner()
This commit also includes a stub for detecting/handling file collisions.
---
roverlay/db/distmap.py | 97 +++++++++++++++++++++++++++++++++++++++++---------
1 file changed, 80 insertions(+), 17 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index ca3d236..b3b285c 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -4,17 +4,17 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
-import errno
-
import bz2
import gzip
+import errno
import os.path
+import logging
import shutil
-
import roverlay.digest
import roverlay.util
+import roverlay.util.objects
import roverlay.stats.collector
@@ -30,6 +30,14 @@ class DistMapInfo ( object ):
RESTORE_FROM_DISTFILE = '_'
UNSET = 'U'
+ @classmethod
+ def from_package_info ( cls, p_info, allow_digest_create=True ):
+ key, value = p_info.get_distmap_value (
+ allow_digest_create=allow_digest_create
+ )
+ return key, cls ( *value )
+ # --- end of from_package_info (...) ---
+
def __init__ ( self, distfile, repo_name, repo_file, sha256 ):
"""Distmap entry constructor.
@@ -44,12 +52,26 @@ class DistMapInfo ( object ):
self.repo_name = repo_name if repo_name is not None else self.UNSET
self.sha256 = sha256
+ # references to objects that "own" (use, ...) this distfile
+ self.backrefs = set()
+ self.add_backref = self.backrefs.add
+
if repo_file == self.RESTORE_FROM_DISTFILE:
self.repo_file = distfile
else:
self.repo_file = repo_file if repo_file is not None else self.UNSET
# --- end of __init__ (...) ---
+ #def add_backref ( self, ref ): self.backrefs.add ( ref )
+
+ def has_backref_to ( self, obj ):
+ return any ( ( ref.deref_unsafe() is obj ) for ref in self.backrefs )
+ # --- end of has_backref_to (...) ---
+
+ def has_backrefs ( self ):
+ return bool ( self.backrefs )
+ # --- end of has_backrefs (...) ---
+
@property
def digest ( self ):
return self.sha256
@@ -139,16 +161,16 @@ def get_distmap ( distmap_file, distmap_compression, ignore_missing=False ):
class _DistMapBase ( object ):
# { attr[, as_attr] }
- DISTMAP_BIND_ATTR = frozenset ({ 'get', 'keys', 'items', 'values', })
-
- class AbstractMethod ( NotImplementedError ):
- pass
- # --- end of AbstractMethod ---
+ DISTMAP_BIND_ATTR = frozenset ({
+ 'get', 'keys', 'items', 'values', ( 'get', 'get_entry' ),
+ })
def __init__ ( self ):
super ( _DistMapBase, self ).__init__()
+ self.logger = logging.getLogger ( self.__class__.__name__ )
self.dirty = False
self._distmap = dict()
+
self.stats = roverlay.stats.collector.static.distmap
self._rebind_distmap()
@@ -202,9 +224,33 @@ class _DistMapBase ( object ):
if isinstance ( attr, str ):
setattr ( self, attr, getattr ( self._distmap, attr ) )
else:
- setattr ( self, attr [1], getattr ( self._distmap, attr[0] ) )
+ setattr ( self, attr[1], getattr ( self._distmap, attr[0] ) )
# --- end of _rebind_distmap (...) ---
+ def add_distfile_owner ( self, backref, distfile, distfilepath=None ):
+ entry = self.get_entry ( distfile )
+ if entry is not None:
+ entry.add_backref ( backref )
+ else:
+ new_entry = self.add_dummy_entry (
+ distfile, distfilepath=distfilepath, log_level=True
+ )
+ # ^ raises: ? if distfile is missing
+ new_entry.add_backref ( backref )
+ # --- end of add_distfile_owner (...) ---
+
+ def get_distfile_slot ( self, backref, distfile ):
+ entry = self.get_entry ( distfile )
+ if entry is None:
+ raise NotImplementedError ( "backref gets new 'slot'." )
+ return 1
+ elif entry.has_backref_to ( backref.deref_safe() ):
+ return 2
+ else:
+ raise NotImplementedError ( "handle file collision." )
+ return 0
+ # --- end of get_distfile_slot (...) ---
+
def check_revbump_necessary ( self, package_info ):
"""Tries to find package_info's distfile in the distmap and returns
whether a revbump is necessary (True) or not (False).
@@ -351,6 +397,8 @@ class _DistMapBase ( object ):
arguments:
* distfile -- distfile path relative to the distroot
* distmap_info -- distmap entry
+
+ Returns: distmap_info
"""
if self.update_only:
entry = self._distmap.get ( distfile, None )
@@ -362,7 +410,7 @@ class _DistMapBase ( object ):
self._distmap [distfile] = distmap_info
self._file_added ( distfile )
- return True
+ return distmap_info
# --- end of add_entry (...) ---
def add_entry_for ( self, p_info ):
@@ -370,14 +418,16 @@ class _DistMapBase ( object ):
arguments:
* p_info --
+
+ Returns: created entry
"""
- key = p_info.get_distmap_key()
- return self.add_entry (
- key, DistMapInfo ( key, *p_info.get_distmap_value() )
- )
+ key, value = DistMapInfo.from_package_info ( p_info )
+ return self.add_entry ( key, value )
# --- end of add_entry_for (...) ---
- def add_dummy_entry ( self, distfile, distfilepath=None, hashdict=None ):
+ def add_dummy_entry (
+ self, distfile, distfilepath=None, hashdict=None, log_level=None
+ ):
"""Adds a dummy entry.
Such an entry contains a checksum and a distfile, but no information
about its origin (repo name/file).
@@ -386,7 +436,18 @@ class _DistMapBase ( object ):
* distfile -- distfile path relative to the distroot
* distfilepath -- absolute path to the distfile
* hashdict -- dict with already calculated hashes
+ * log_level -- if not None: log entry creation with the given log
+ level (or INFO if log_level is True)
+
+ Returns: created entry
"""
+ if log_level is None or log_level is False:
+ pass
+ elif log_level is True:
+ self.logger.info ( "adding dummy entry for " + distfile )
+ else:
+ self.logger.log ( log_level, "adding dummy entry for " + distfile )
+
if hashdict and DistMapInfo.DIGEST_TYPE in hashdict:
digest = hashdict [DistMapInfo.DIGEST_TYPE]
else:
@@ -523,15 +584,17 @@ class _FileDistMapBase ( _DistMapBase ):
return False
# --- end of _read_header (...) ---
+ @roverlay.util.objects.abstractmethod
def read ( self, filepath=None ):
"""Reads the distmap.
arguments:
* filepath -- path to the distmap file (defaults to self.dbfile)
"""
- raise self.__class__.AbstractMethod()
+ pass
# --- end of read (...) ---
+ @roverlay.util.objects.abstractmethod
def write ( self, filepath=None, force=False ):
"""Writes the distmap.
@@ -539,7 +602,7 @@ class _FileDistMapBase ( _DistMapBase ):
* filepath -- path to the distmap file (defaults to self.dbfile)
* force -- enforce writing even if distmap not modified
"""
- raise self.__class__.AbstractMethod()
+ pass
# --- end of write (...) ---
# --- end of _FileDistMapBase ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-30 14:49 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-30 14:49 UTC (permalink / raw
To: gentoo-commits
commit: adc664ff7740d326a99e43f9de939b69db5648e1
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 30 14:39:21 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 30 14:39:21 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=adc664ff
distmap: get_distfile_slot()
distmap-side support for detecting file collisions.
Note that volatile entries could be handled better (currently, they _should_
be overwritten when writing the overlay).
---
roverlay/db/distmap.py | 40 +++++++++++++++++++++++++++++-----------
1 file changed, 29 insertions(+), 11 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 47cfa27..cddcb5b 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -33,12 +33,22 @@ class DistMapInfo ( object ):
@classmethod
def from_package_info ( cls, p_info, allow_digest_create=True ):
- key, value = p_info.get_distmap_value (
- allow_digest_create=allow_digest_create
+ return cls (
+ *p_info.get_distmap_value (
+ allow_digest_create=allow_digest_create
+ )
)
- return key, cls ( *value )
# --- end of from_package_info (...) ---
+ @classmethod
+ def volatile_from_package_info ( cls, p_info, backref=None ):
+ instance = cls ( *p_info.get_distmap_value ( no_digest=True ) )
+ instance.volatile = True
+ if backref is not None:
+ instance.add_backref ( backref )
+ return instance
+ # --- end of volatile_from_package_info (...) ---
+
def __init__ (
self, distfile, repo_name, repo_file, sha256, volatile=False
):
@@ -233,16 +243,22 @@ class _DistMapBase ( object ):
yield info.to_str ( str ( distfile ), field_delimiter )
# --- end of gen_info_lines (...) ---
- def get_distfile_slot ( self, backref, distfile ):
- entry = self.get_entry ( distfile )
+ def get_distfile_slot ( self, package_dir, p_info ):
+ distfile = p_info ['package_src_destpath']
+ entry = self.get_entry ( distfile )
+
if entry is None:
- raise NotImplementedError ( "backref gets new 'slot'." )
+ # entry does not exist, create a new,volatile one
+ self._distmap [distfile] = DistMapInfo.volatile_from_package_info (
+ p_info, backref=package_dir.get_ref()
+ )
return 1
- elif entry.has_backref_to ( backref.deref_safe() ):
- # revbump check might be necessary
+ elif entry.has_backref_to ( package_dir ):
+ # entry exists and belongs to backref, nothing to do here
+ # a revbump check might be necessary
return 2
else:
- raise NotImplementedError ( "handle file collision." )
+ # collision, should be resolved by the distroot
return 0
# --- end of get_distfile_slot (...) ---
@@ -416,8 +432,10 @@ class _DistMapBase ( object ):
Returns: created entry
"""
- key, value = DistMapInfo.from_package_info ( p_info )
- return self.add_entry ( key, value )
+ return self.add_entry (
+ p_info.get_distmap_key(),
+ DistMapInfo.from_package_info ( p_info )
+ )
# --- end of add_entry_for (...) ---
def add_dummy_entry (
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-30 15:23 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-30 15:23 UTC (permalink / raw
To: gentoo-commits
commit: 372aff47b57d83ac152efdde9d447e5627a6ccff
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 30 15:23:10 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 30 15:23:10 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=372aff47
distmap: __delitem__(), __contains__()
---
roverlay/db/distmap.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 0ae294a..750969f 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -164,6 +164,15 @@ class _DistMapBase ( object ):
self.update_only = True
# --- end of __init__ (...) ---
+ def __contains__ ( self, key ):
+ return key in self._distmap
+ # --- end of __contains__ (...) ---
+
+ def __delitem__ ( self, key ):
+ del self._distmap [key]
+ self._file_removed ( key )
+ # --- end of __delitem__ (...) ---
+
def __getitem__ ( self, key ):
return self._distmap [key]
# --- end of __getitem__ (...) ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-08-30 15:25 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-08-30 15:25 UTC (permalink / raw
To: gentoo-commits
commit: ddb8c1a25befa7218fd82d38fd00777e902759c2
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 30 15:25:25 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 30 15:25:25 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ddb8c1a2
remove debug print()
---
roverlay/db/distmap.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 750969f..68fa6f6 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -656,7 +656,6 @@ class FileDistMap ( _DistMapBase ):
self._nondirty_file_added ( distfile )
# -- end for
self.dirty = self.dirty or filepath is not None
- print( list(self.gen_info_lines()) )
# --- end of read (...) ---
def write ( self, filepath=None, force=False ):
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-02 8:44 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-02 8:44 UTC (permalink / raw
To: gentoo-commits
commit: f41663660bc6676b13aeb2dab409f6eb60f68e7d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Sep 2 08:40:20 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Sep 2 08:40:20 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f4166366
distmap: minor change
* remove get_distmap() from __all__
* remove COULDFIX note as it's implemented now
---
roverlay/db/distmap.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 68fa6f6..16d1b97 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -19,9 +19,7 @@ import roverlay.util.objects
import roverlay.stats.collector
-__all__ = [ 'DistMapInfo', 'get_distmap' ]
-
-#COULDFIX: distmap could use roverlay.util.fileio
+__all__ = [ 'DistMapInfo', 'FileDistMap', ]
class DistMapInfo ( object ):
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-03 12:21 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-03 12:21 UTC (permalink / raw
To: gentoo-commits
commit: 52a42a2e9edebf4b28fa5b354c048ae9e92c9350
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 12:19:42 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 12:19:42 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=52a42a2e
distmap: Dist{m,M}apException, camel case
---
roverlay/db/distmap.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index d400a75..ba0b29b 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -23,7 +23,7 @@ __all__ = [ 'DistMapInfo', 'FileDistMap', ]
-class DistmapException ( Exception ):
+class DistMapException ( Exception ):
pass
@@ -497,7 +497,7 @@ class _DistMapBase ( object ):
self._file_added ( distfile )
return entry
else:
- raise DistmapException (
+ raise DistMapException (
"volatile entry for {} does not exist!".format ( distfile )
)
# --- end of add_entry_for_volatile (...) ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-03 12:21 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-03 12:21 UTC (permalink / raw
To: gentoo-commits
commit: 924bba0dca52503b87a73ce386c01bf33e2fa008
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 12:18:02 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 12:18:02 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=924bba0d
distmap: make "volatile" a ref to a pkg info
DistMapInfo: make_volatile()/make_persistent() for "converting" entries into
volatile/non-volatile form.
---
roverlay/db/distmap.py | 93 +++++++++++++++++++++++++++++++++++++++++---------
1 file changed, 76 insertions(+), 17 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 16d1b97..d400a75 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -22,6 +22,11 @@ import roverlay.stats.collector
__all__ = [ 'DistMapInfo', 'FileDistMap', ]
+
+class DistmapException ( Exception ):
+ pass
+
+
class DistMapInfo ( object ):
"""Distmap entry"""
@@ -41,14 +46,11 @@ class DistMapInfo ( object ):
@classmethod
def volatile_from_package_info ( cls, p_info, backref=None ):
instance = cls ( *p_info.get_distmap_value ( no_digest=True ) )
- instance.volatile = True
- if backref is not None:
- instance.add_backref ( backref )
- return instance
+ return instance.make_volatile ( p_info, backref=backref )
# --- end of volatile_from_package_info (...) ---
def __init__ (
- self, distfile, repo_name, repo_file, sha256, volatile=False
+ self, distfile, repo_name, repo_file, sha256, volatile=None
):
"""Distmap entry constructor.
@@ -57,8 +59,10 @@ class DistMapInfo ( object ):
* repo_name -- name of the repo that owns the package file
* repo_file -- path of the package file relative to the repo
* sha256 -- file checksum
- * volatile -- whether this entry should be persistent (False) or
- not (True). Defaults to False.
+ * volatile -- a reference to a PackageInfo instance or None
+ None indicates that this entry should be persistent,
+ whereas "not None" indicates a "volatile" entry.
+ Defaults to None.
"""
super ( DistMapInfo, self ).__init__()
@@ -76,6 +80,33 @@ class DistMapInfo ( object ):
self.repo_file = repo_file if repo_file is not None else self.UNSET
# --- end of __init__ (...) ---
+ def is_volatile ( self ):
+ return self.volatile is not None
+ # --- end of is_volatile (...) ---
+
+ def is_persistent ( self ):
+ return self.volatile is None
+ # --- end of is_persistent (...) ---
+
+ def make_volatile ( self, p_info, backref=None ):
+ self.volatile = p_info.get_ref()
+ self.sha256 = None
+ if backref is not None:
+ self.add_backref ( backref )
+ return self
+ # --- end of make_volatile (...) ---
+
+ def make_persistent ( self ):
+ p_info = self.volatile.deref_safe()
+ self.sha256 = p_info.make_distmap_hash()
+ self.volatile = None
+ return self
+ # --- end of make_persistent (...) ---
+
+ def deref_volatile ( self ):
+ return None if self.volatile is None else self.volatile.deref_unsafe()
+ # --- end of deref_volatile (...) ---
+
#def add_backref ( self, ref ): self.backrefs.add ( ref )
def has_backref_to ( self, obj ):
@@ -127,7 +158,7 @@ class DistMapInfo ( object ):
* field_delimiter -- char (or char sequence) that is used to separate
values
"""
- assert not self.volatile
+ assert self.volatile is None
return ( field_delimiter.join ((
distfile,
self.repo_name,
@@ -212,13 +243,13 @@ class _DistMapBase ( object ):
def _iter_persistent ( self ):
for distfile, info in self._distmap.items():
- if not info.volatile:
+ if info.is_persistent():
yield ( distfile, info )
# --- end of _iter_persistent (...) ---
def _iter_volatile ( self ):
for distfile, info in self._distmap.items():
- if info.volatile:
+ if info.is_volatile():
yield ( distfile, info )
# --- end of _iter_volatile (...) ---
@@ -244,7 +275,7 @@ class _DistMapBase ( object ):
def gen_info_lines ( self, field_delimiter ):
for distfile, info in self._distmap.items():
- if not info.volatile:
+ if info.is_persistent():
yield info.to_str ( str ( distfile ), field_delimiter )
# --- end of gen_info_lines (...) ---
@@ -262,9 +293,15 @@ class _DistMapBase ( object ):
# entry exists and belongs to backref, nothing to do here
# a revbump check might be necessary
return 2
- else:
+ elif entry.has_backrefs():
# collision, should be resolved by the distroot
return 0
+ else:
+ # distfile has no owner
+ # verify <>.repo, ...
+ #
+ entry.add_backref ( package_dir.get_ref() )
+ return 3
# --- end of get_distfile_slot (...) ---
def check_revbump_necessary ( self, package_info ):
@@ -437,12 +474,34 @@ class _DistMapBase ( object ):
Returns: created entry
"""
- return self.add_entry (
- p_info.get_distmap_key(),
- DistMapInfo.from_package_info ( p_info )
- )
+ distfile = p_info.get_distmap_key()
+ entry = self._distmap.get ( entry, None )
+
+ if entry is None or entry.deref_volatile() is not p_info:
+ return self.add_entry (
+ p_info.get_distmap_key(),
+ DistMapInfo.from_package_info ( p_info )
+ )
+ else:
+ entry.make_persistent()
+ self._file_added ( distfile )
+ return entry
# --- end of add_entry_for (...) ---
+ def add_entry_for_volatile ( self, p_info ):
+ distfile = p_info.get_distmap_key()
+ entry = self._distmap [distfile]
+
+ if entry.deref_volatile() is p_info:
+ entry.make_persistent()
+ self._file_added ( distfile )
+ return entry
+ else:
+ raise DistmapException (
+ "volatile entry for {} does not exist!".format ( distfile )
+ )
+ # --- end of add_entry_for_volatile (...) ---
+
def add_dummy_entry (
self, distfile, distfilepath=None, hashdict=None, log_level=None
):
@@ -599,7 +658,7 @@ class FileDistMap ( _DistMapBase ):
def gen_info_lines ( self ):
for distfile, info in self._distmap.items():
- if not info.volatile:
+ if info.is_persistent():
yield info.to_str ( str ( distfile ), self.FIELD_DELIMITER )
# --- end of gen_info_lines (...) ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-03 13:15 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-03 13:15 UTC (permalink / raw
To: gentoo-commits
commit: 0509ccb1fa18b88e06b5bbb300b191776a2b6642
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 13:08:48 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 13:08:48 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=0509ccb1
distmap entry: get_repo_name()
+ return entry in add_distfile_owner()
---
roverlay/db/distmap.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index ba0b29b..1326e5e 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -123,6 +123,10 @@ class DistMapInfo ( object ):
#return getattr ( self, self.DIGEST_TYPE )
# --- end of digest (...) ---
+ def get_repo_name ( self ):
+ return None if self.repo_name is self.UNSET else self.repo_name
+ # --- end of get_repo_name (...) ---
+
def compare_digest ( self, package_info ):
p_hash = package_info.make_distmap_hash()
return ( bool ( p_hash == self.digest ), p_hash )
@@ -266,11 +270,14 @@ class _DistMapBase ( object ):
if entry is not None:
entry.add_backref ( backref )
else:
- new_entry = self.add_dummy_entry (
+ entry = self.add_dummy_entry (
distfile, distfilepath=distfilepath, log_level=True
)
+ # FIXME:
# ^ raises: ? if distfile is missing
- new_entry.add_backref ( backref )
+ entry.add_backref ( backref )
+ # -- end if
+ return entry
# --- end of add_distfile_owner (...) ---
def gen_info_lines ( self, field_delimiter ):
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-03 15:37 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-03 15:37 UTC (permalink / raw
To: gentoo-commits
commit: 0ba2e90d9988584e6d8e19e04f75615ecc27b13e
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 15:36:21 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 15:36:21 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=0ba2e90d
distmap, get_distfile_slot(): make entry volatile
make existing entries volatile if they're not owned by any package
---
roverlay/db/distmap.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 1326e5e..e297286 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -307,7 +307,7 @@ class _DistMapBase ( object ):
# distfile has no owner
# verify <>.repo, ...
#
- entry.add_backref ( package_dir.get_ref() )
+ entry.make_volatile ( p_info, package_dir.get_ref() )
return 3
# --- end of get_distfile_slot (...) ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-05 9:25 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-05 9:25 UTC (permalink / raw
To: gentoo-commits
commit: 8b1ea2fdaa6d72828f30ae5ec487f8e27977c4ad
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 09:20:48 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 09:20:48 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=8b1ea2fd
distmap, add_distfile_owner(): ignore missing files
create a virtual entry for files that do not exist
---
roverlay/db/distmap.py | 41 +++++++++++++++++++++++++++++++++--------
1 file changed, 33 insertions(+), 8 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 584ae8c..aadb491 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -49,6 +49,16 @@ class DistMapInfo ( object ):
return instance.make_volatile ( p_info, backref=backref )
# --- end of volatile_from_package_info (...) ---
+ @classmethod
+ def volatile_from_distfile ( cls, distfile, backref=None ):
+ # COULDFIX
+ # a static instance should suffice for this entry "type"
+ instance = cls ( distfile, None, None, None, volatile=True )
+ if backref is not None:
+ instance.add_backref ( backref )
+ return instance
+ # --- end of volatile_from_distfile (...) ---
+
def __init__ (
self, distfile, repo_name, repo_file, sha256, volatile=None
):
@@ -269,12 +279,17 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
if entry is not None:
entry.add_backref ( backref )
else:
- entry = self.add_dummy_entry (
- distfile, distfilepath=distfilepath, log_level=True
- )
- # FIXME:
- # ^ raises: ? if distfile is missing
- entry.add_backref ( backref )
+ try:
+ entry = self.add_dummy_entry (
+ distfile, distfilepath=distfilepath, log_level=True
+ )
+ entry.add_backref ( backref )
+ except IOError as ioerr:
+ if ioerr.errno == errno.ENOENT:
+ entry = self.add_virtual_entry ( distfile, backref )
+ else:
+ raise
+
# -- end if
return entry
# --- end of add_distfile_owner (...) ---
@@ -496,11 +511,13 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
entry = self._distmap.get ( distfile, None )
if entry is None or entry != distmap_info:
self._distmap [distfile] = distmap_info
- self._file_added ( distfile )
+ if distmap_info.is_persistent():
+ self._file_added ( distfile )
del entry
else:
self._distmap [distfile] = distmap_info
- self._file_added ( distfile )
+ if distmap_info.is_persistent():
+ self._file_added ( distfile )
return distmap_info
# --- end of add_entry (...) ---
@@ -574,6 +591,14 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
)
# --- end of add_dummy_entry (...) ---
+ def add_virtual_entry ( self, distfile, backref=None ):
+ return self.add_entry (
+ distfile, DistMapInfo.volatile_from_distfile (
+ distfile, backref=backref
+ )
+ )
+ # --- end of add_virtual_entry (...) ---
+
# --- end of _DistMapBase ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-05 9:25 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-05 9:25 UTC (permalink / raw
To: gentoo-commits
commit: 2e90b548ac7177ce8af2c2ff53321db463c9f840
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 09:09:08 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 09:09:08 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2e90b548
distmap: use roverlay.util.fileio.TextFile
---
roverlay/db/distmap.py | 156 +++++++++----------------------------------------
1 file changed, 27 insertions(+), 129 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index d683f24..584ae8c 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -177,17 +177,16 @@ class DistMapInfo ( object ):
# --- end of DistMapInfo ---
-class _DistMapBase ( object ):
+class _DistMapBase ( roverlay.util.objects.PersistentContent ):
# { attr[, as_attr] }
DISTMAP_BIND_ATTR = frozenset ({
'get', 'keys', 'items', 'values', ( 'get', 'get_entry' ),
})
- def __init__ ( self ):
+ def __init__ ( self, *args, **kwargs ):
super ( _DistMapBase, self ).__init__()
self.logger = logging.getLogger ( self.__class__.__name__ )
- self.dirty = False
self._distmap = dict()
self.stats = roverlay.stats.collector.static.distmap
@@ -237,12 +236,12 @@ class _DistMapBase ( object ):
def _file_added ( self, distfile ):
self.stats.file_added()
- self.dirty = True
+ self.set_dirty()
# --- end of _file_added (...) ---
def _file_removed ( self, distfile ):
self.stats.file_removed()
- self.dirty = True
+ self.set_dirty()
# --- end of _file_removed (...) ---
def _iter_persistent ( self ):
@@ -578,7 +577,7 @@ class _DistMapBase ( object ):
# --- end of _DistMapBase ---
-class FileDistMap ( _DistMapBase ):
+class FileDistMap ( roverlay.util.fileio.TextFile, _DistMapBase ):
"""A distmap that is read from / written to a file."""
# the default info field separator
@@ -588,17 +587,6 @@ class FileDistMap ( _DistMapBase ):
# file format (reserved for future usage)
FILE_FORMAT = '0'
- def set_compression ( self, compression ):
- if not compression or compression in { 'default', 'none' }:
- self.compression = None
- elif compression in roverlay.util.fileio.SUPPORTED_COMPRESSION:
- self.compression = compression
- else:
- raise ValueError (
- "unknown distmap compression {!r}".format ( compression )
- )
- # --- end of set_compression (...) ---
-
def __init__ (
self, distmap_file, distmap_compression=None, ignore_missing=False
):
@@ -612,10 +600,9 @@ class FileDistMap ( _DistMapBase ):
raises: ValueError if distmap_compression not supported.
"""
- super ( FileDistMap, self ).__init__ ()
- self.dbfile = distmap_file
- self.compression = None
- self.set_compression ( distmap_compression )
+ super ( FileDistMap, self ).__init__ (
+ filepath=distmap_file, compression=distmap_compression
+ )
if ignore_missing:
self.try_read()
@@ -623,73 +610,6 @@ class FileDistMap ( _DistMapBase ):
self.read()
# --- end of __init__ (...) ---
- def backup_file ( self, destfile=None, move=False, ignore_missing=False ):
- """Creates a backup copy of the distmap file.
-
- arguments:
- * destfile -- backup file path
- Defaults to <distmap file> + '.bak'.
- * move -- move distmap file (instead of copying)
- * ignore_missing -- return False if distmap file does not exist instead
- of raising an exception
- Defaults to False.
- """
- dest = destfile or self.dbfile + '.bak'
- try:
- roverlay.util.dodir ( os.path.dirname ( dest ), mkdir_p=True )
- if move:
- shutil.move ( self.dbfile, dest )
- return True
- else:
- shutil.copyfile ( self.dbfile, dest )
- return True
- except IOError as ioerr:
- if ignore_missing and ioerr.errno == errno.ENOENT:
- return False
- else:
- raise
- # --- end of backup_file (...) ---
-
- def backup_and_write ( self,
- db_file=None, backup_file=None,
- force=False, move=False, ignore_missing=True
- ):
- """Creates a backup copy of the distmap file and writes the modified
- distmap afterwards.
-
- arguments:
- * db_file -- distmap file path (defaults to self.dbfile)
- * backup_file -- backup file path (see backup_file())
- * force -- enforce writing even if distmap not modified
- * move -- move distmap (see backup_file())
- * ignore_missing -- do not fail if distmap file does not exist
- Defaults to True.
- """
- if force or self.dirty:
- self.backup_file (
- destfile=backup_file, move=move, ignore_missing=ignore_missing
- )
- return self.write ( filepath=db_file, force=True )
- else:
- return True
- # --- end of backup_and_write (...) ---
-
- def file_exists ( self ):
- """Returns True if the distmap file exists, else False."""
- return os.path.isfile ( self.dbfile )
- # --- end of file_exists (...) ---
-
- def try_read ( self, *args, **kwargs ):
- """Tries to read the distmap file."""
- try:
- self.read ( *args, **kwargs )
- except IOError as ioerr:
- if ioerr.errno == errno.ENOENT:
- pass
- else:
- raise
- # --- end of try_read (...) ---
-
def get_header ( self ):
return "<{d}<{fmt}".format (
d=self.FIELD_DELIMITER, fmt=self.FILE_FORMAT
@@ -727,51 +647,29 @@ class FileDistMap ( _DistMapBase ):
return False
# --- end of _read_header (...) ---
- def read ( self, filepath=None ):
- """Reads the distmap.
-
- arguments:
- * filepath -- path to the distmap file (defaults to self.dbfile)
- """
- dbfile = self.dbfile if filepath is None else filepath
- first = True
-
- for line in roverlay.util.fileio.read_text_file (
- dbfile, preparse=True, try_harder=True
- ):
- if first:
- first = False
- if self._read_header ( line ):
- continue
- # else no header
- # -- end if
-
- distfile, info = roverlay.util.headtail (
- line.split ( self.FIELD_DELIMITER )
- )
- self._distmap [distfile] = DistMapInfo ( distfile, *info )
- self._nondirty_file_added ( distfile )
- # -- end for
- self.dirty = self.dirty or filepath is not None
- # --- end of read (...) ---
+ def parse_line ( self, line ):
+ distfile, info = roverlay.util.headtail (
+ line.split ( self.FIELD_DELIMITER )
+ )
+ self._distmap [distfile] = DistMapInfo ( distfile, *info )
+ self._nondirty_file_added ( distfile )
+ return True
+ # --- end of parse_line (...) ---
- def write ( self, filepath=None, force=False ):
- """Writes the distmap.
+ def parse_header_line ( self, line ):
+ """Tries to parse a text line as distmap file header, else parses
+ it as normal text line.
arguments:
- * filepath -- path to the distmap file (defaults to self.dbfile)
- * force -- enforce writing even if distmap not modified
+ * line --
"""
- if force or self.dirty or filepath is not None:
- dbfile = self.dbfile if filepath is None else filepath
- roverlay.util.fileio.write_text_file (
- dbfile, self.gen_lines(),
- compression=self.compression, create_dir=True
- )
- self.dirty = self.dirty and filepath is not None
- return True
+ if len ( line ) > 2 and line[0] == line[2]:
+ # instance attr
+ self.FIELD_DELIMITER = line[1]
+ if len ( line ) > 3:
+ self.FILE_FORMAT = line[3:]
else:
- return False
- # --- end of write (...) ---
+ return self.parse_line ( line )
+ # --- end of parse_header_line (...) ---
# --- end of FileDistMap ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-05 9:25 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-05 9:25 UTC (permalink / raw
To: gentoo-commits
commit: 52c528d8310547a11b5c59d35d967751d306ead6
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 09:24:01 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 09:24:01 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=52c528d8
distmap: log creation of virtual entries
---
roverlay/db/distmap.py | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index aadb491..dfc65f1 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -591,7 +591,21 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
)
# --- end of add_dummy_entry (...) ---
- def add_virtual_entry ( self, distfile, backref=None ):
+ def add_virtual_entry ( self, distfile, backref=None, log_level=True ):
+ if log_level is None or log_level is False:
+ pass
+ else:
+ log_msg = (
+ "adding virtual entry for {} - file does not exist!".format (
+ distfile
+ )
+ )
+ if log_level is True:
+ self.logger.warning ( log_msg )
+ else:
+ self.logger.log ( log_level, log_msg )
+ # -- end if <log_level>
+
return self.add_entry (
distfile, DistMapInfo.volatile_from_distfile (
distfile, backref=backref
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-05 10:24 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-05 10:24 UTC (permalink / raw
To: gentoo-commits
commit: d29dfc2d4336696935f51c0ddb2fd839ab75a7b2
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 10:14:16 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 10:14:16 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d29dfc2d
distmap, parse_header_line(): return True
---
roverlay/db/distmap.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index dfc65f1..ae29e14 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -707,6 +707,7 @@ class FileDistMap ( roverlay.util.fileio.TextFile, _DistMapBase ):
self.FIELD_DELIMITER = line[1]
if len ( line ) > 3:
self.FILE_FORMAT = line[3:]
+ return True
else:
return self.parse_line ( line )
# --- end of parse_header_line (...) ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-05 14:43 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-05 14:43 UTC (permalink / raw
To: gentoo-commits
commit: 6a4edc70205f7aaff3dd54f3ec00a44e5a84b89f
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 12:25:18 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 12:25:18 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6a4edc70
distmap: remove unused imports
---
roverlay/db/distmap.py | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index ae29e14..66ed838 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -4,16 +4,11 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
-import bz2
-import gzip
-
import errno
-import os.path
import logging
-import shutil
import roverlay.digest
-import roverlay.util
+import roverlay.util.common
import roverlay.util.fileio
import roverlay.util.objects
import roverlay.stats.collector
@@ -687,7 +682,7 @@ class FileDistMap ( roverlay.util.fileio.TextFile, _DistMapBase ):
# --- end of _read_header (...) ---
def parse_line ( self, line ):
- distfile, info = roverlay.util.headtail (
+ distfile, info = roverlay.util.common.headtail (
line.split ( self.FIELD_DELIMITER )
)
self._distmap [distfile] = DistMapInfo ( distfile, *info )
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-05 14:43 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-05 14:43 UTC (permalink / raw
To: gentoo-commits
commit: c2a13a2f3ec262b0bd0aa2229b9672f4878f8776
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 12:59:57 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 12:59:57 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=c2a13a2f
distmap: use VirtualDistMapInfo
... for virtual entries
---
roverlay/db/distmap.py | 60 +++++++++++++++++++++++++++++++++++---------------
1 file changed, 42 insertions(+), 18 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 0a3ee64..e4f8616 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -22,7 +22,47 @@ class DistMapException ( Exception ):
pass
-class DistMapInfo ( object ):
+
+class VirtualDistMapInfo ( object ):
+
+ def __init__ ( self ):
+ super ( VirtualDistMapInfo, self ).__init__()
+ # references to objects that "own" (use, ...) this distfile
+ self.backrefs = set()
+ self.add_backref = self.backrefs.add
+ # --- end of __init__ (...) ---
+
+ def is_volatile ( self ):
+ return True
+
+ def is_persistent ( self ):
+ return False
+
+ def deref_volatile ( self ):
+ raise NotImplementedError()
+
+ #def add_backref ( self, ref ): self.backrefs.add ( ref )
+
+ def has_backref_to ( self, obj ):
+ return any ( ref.deref_unsafe() is obj for ref in self.backrefs )
+ # --- end of has_backref_to (...) ---
+
+ def has_backrefs ( self ):
+ return bool ( self.backrefs )
+
+ def __eq__ ( self, other ):
+ return self is other
+
+ def __ne__ ( self, other ):
+ return self is not other
+
+ def get_repo_name ( self ):
+ return None
+
+# --- end of VirtualDistMapInfo ---
+
+
+class DistMapInfo ( VirtualDistMapInfo ):
"""Distmap entry"""
DIGEST_TYPE = 'sha256'
@@ -65,10 +105,6 @@ class DistMapInfo ( object ):
self.sha256 = sha256
self.volatile = volatile
- # references to objects that "own" (use, ...) this distfile
- self.backrefs = set()
- self.add_backref = self.backrefs.add
-
if repo_file == self.RESTORE_FROM_DISTFILE:
self.repo_file = distfile
else:
@@ -102,16 +138,6 @@ class DistMapInfo ( object ):
return None if self.volatile is None else self.volatile.deref_unsafe()
# --- end of deref_volatile (...) ---
- #def add_backref ( self, ref ): self.backrefs.add ( ref )
-
- def has_backref_to ( self, obj ):
- return any ( ( ref.deref_unsafe() is obj ) for ref in self.backrefs )
- # --- end of has_backref_to (...) ---
-
- def has_backrefs ( self ):
- return bool ( self.backrefs )
- # --- end of has_backrefs (...) ---
-
@property
def digest ( self ):
return self.sha256
@@ -600,9 +626,7 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
# -- end if <log_level>
if self._VIRTUAL_ENTRY is None:
- self._VIRTUAL_ENTRY = DistMapInfo (
- None, None, None, None, volatile=True
- )
+ self._VIRTUAL_ENTRY = VirtualDistMapInfo()
if backref is not None:
self._VIRTUAL_ENTRY.add_backref ( backref )
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2013-09-05 14:43 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2013-09-05 14:43 UTC (permalink / raw
To: gentoo-commits
commit: 35d3fa6d99e964cddb256813ef786c66becdbe61
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Sep 5 12:47:39 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Sep 5 12:47:39 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=35d3fa6d
distmap: use exactly one virtual entry
use one virtual entry for all distfiles that do not exist
---
roverlay/db/distmap.py | 32 +++++++++++++++++---------------
1 file changed, 17 insertions(+), 15 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 66ed838..0a3ee64 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -44,16 +44,6 @@ class DistMapInfo ( object ):
return instance.make_volatile ( p_info, backref=backref )
# --- end of volatile_from_package_info (...) ---
- @classmethod
- def volatile_from_distfile ( cls, distfile, backref=None ):
- # COULDFIX
- # a static instance should suffice for this entry "type"
- instance = cls ( distfile, None, None, None, volatile=True )
- if backref is not None:
- instance.add_backref ( backref )
- return instance
- # --- end of volatile_from_distfile (...) ---
-
def __init__ (
self, distfile, repo_name, repo_file, sha256, volatile=None
):
@@ -196,6 +186,8 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
self.stats = roverlay.stats.collector.static.distmap
+ self._VIRTUAL_ENTRY = None
+
self._rebind_distmap()
self.update_only = True
@@ -504,16 +496,22 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
"""
if self.update_only:
entry = self._distmap.get ( distfile, None )
- if entry is None or entry != distmap_info:
+
+ if entry is not None and entry == distmap_info:
+ return entry
+
+ else:
self._distmap [distfile] = distmap_info
if distmap_info.is_persistent():
self._file_added ( distfile )
del entry
+
else:
self._distmap [distfile] = distmap_info
if distmap_info.is_persistent():
self._file_added ( distfile )
+
return distmap_info
# --- end of add_entry (...) ---
@@ -601,11 +599,15 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
self.logger.log ( log_level, log_msg )
# -- end if <log_level>
- return self.add_entry (
- distfile, DistMapInfo.volatile_from_distfile (
- distfile, backref=backref
+ if self._VIRTUAL_ENTRY is None:
+ self._VIRTUAL_ENTRY = DistMapInfo (
+ None, None, None, None, volatile=True
)
- )
+
+ if backref is not None:
+ self._VIRTUAL_ENTRY.add_backref ( backref )
+
+ return self.add_entry ( distfile, self._VIRTUAL_ENTRY )
# --- end of add_virtual_entry (...) ---
# --- end of _DistMapBase ---
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2014-07-16 15:14 André Erdmann
0 siblings, 0 replies; 28+ messages in thread
From: André Erdmann @ 2014-07-16 15:14 UTC (permalink / raw
To: gentoo-commits
commit: 884a230b83741b10217767110916cec869be1c08
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sun Jul 13 15:43:22 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sun Jul 13 15:43:22 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=884a230b
roverlay/db/distmap.py: use get_distmap_key()
---
roverlay/db/distmap.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 76196f8..338c012 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -318,7 +318,7 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
# --- end of gen_info_lines (...) ---
def get_distfile_slot ( self, package_dir, p_info ):
- distfile = p_info ['package_src_destpath']
+ distfile = p_info.get_distmap_key()
entry = self.get_entry ( distfile )
if entry is None:
^ permalink raw reply related [flat|nested] 28+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/
@ 2016-07-07 4:19 Benda XU
0 siblings, 0 replies; 28+ messages in thread
From: Benda XU @ 2016-07-07 4:19 UTC (permalink / raw
To: gentoo-commits
commit: 9618ea9eed6e625b9086325f46c80479e82af433
Author: Benda Xu <heroxbd <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 7 04:18:13 2016 +0000
Commit: Benda XU <heroxbd <AT> gentoo <DOT> org>
CommitDate: Thu Jul 7 04:18:13 2016 +0000
URL: https://gitweb.gentoo.org/proj/R_overlay.git/commit/?id=9618ea9e
roverlay/db/distmap.py: call the wrapper for repo_name
info.repo_name might be undefined.
roverlay/db/distmap.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 8562da1..7ca6099 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -404,7 +404,7 @@ class _DistMapBase ( roverlay.util.objects.PersistentContent ):
if info is None:
# new file, no revbump required
return False
- elif info.repo_name != package_info['origin'].name:
+ elif info.get_repo_name() != package_info['origin'].name:
# don't revbump if repo names don't match, this likely results in
# infinite revbumps if a package is available from more than one repo
return False
^ permalink raw reply related [flat|nested] 28+ messages in thread
end of thread, other threads:[~2016-07-07 4:19 UTC | newest]
Thread overview: 28+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-08-16 12:42 [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/ André Erdmann
-- strict thread matches above, loose matches on Subject: below --
2016-07-07 4:19 Benda XU
2014-07-16 15:14 André Erdmann
2013-09-05 14:43 André Erdmann
2013-09-05 14:43 André Erdmann
2013-09-05 14:43 André Erdmann
2013-09-05 10:24 André Erdmann
2013-09-05 9:25 André Erdmann
2013-09-05 9:25 André Erdmann
2013-09-05 9:25 André Erdmann
2013-09-03 15:37 André Erdmann
2013-09-03 13:15 André Erdmann
2013-09-03 12:21 André Erdmann
2013-09-03 12:21 André Erdmann
2013-09-02 8:44 André Erdmann
2013-08-30 15:25 André Erdmann
2013-08-30 15:23 André Erdmann
2013-08-30 14:49 André Erdmann
2013-08-30 14:49 André Erdmann
2013-08-22 9:01 André Erdmann
2013-08-16 14:26 André Erdmann
2013-08-15 9:18 André Erdmann
2013-08-14 14:56 André Erdmann
2013-08-14 14:56 André Erdmann
2013-08-13 8:56 André Erdmann
2013-07-30 18:40 André Erdmann
2013-07-10 15:10 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
2013-07-10 16:16 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
2013-06-22 15:14 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
2013-06-22 15:24 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox