* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/, roverlay/recipe/
2013-06-19 18:58 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
@ 2013-06-22 15:24 ` André Erdmann
0 siblings, 0 replies; 2+ messages in thread
From: André Erdmann @ 2013-06-22 15:24 UTC (permalink / raw
To: gentoo-commits
commit: 468943a11d46a71219286dd323f24bfcb048e22c
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun 19 18:53:10 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun 19 18:53:10 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=468943a1
distmap - track changes in DISTDIR
distmap is a persistent object (written to/read from file) that contains
information about files in distdir, e.g. name of the repo a file belongs to.
It also contains a checksum (sha256 as it's also used in Manifest files) that
can be used to check whether an ebuild has to be regenerated (without scanning
the entire overlay).
The file can (optionally) be bzip2- or gzip-compressed.
---
roverlay/db/__init__.py | 0
roverlay/db/distmap.py | 371 +++++++++++++++++++++++++++++++++++++++++++++
roverlay/recipe/distmap.py | 43 ++++++
3 files changed, 414 insertions(+)
diff --git a/roverlay/db/__init__.py b/roverlay/db/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
new file mode 100644
index 0000000..7bfba59
--- /dev/null
+++ b/roverlay/db/distmap.py
@@ -0,0 +1,371 @@
+# R overlay -- db, ( dist file ) => ( repo, repo file ) map
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import errno
+
+import bz2
+import gzip
+
+import os.path
+import shutil
+
+
+import roverlay.util
+
+
+__all__ = [ 'DistMapInfo', 'get_distmap' ]
+
+
+class DistMapInfo ( object ):
+
+ RESTORE_FROM_DISTFILE = '_'
+
+ def __init__ ( self, distfile, repo_name, repo_file, sha256 ):
+ super ( DistMapInfo, self ).__init__()
+
+ self.repo_name = repo_name
+ self.sha256 = sha256
+
+ if repo_file == self.RESTORE_FROM_DISTFILE:
+ self.repo_file = distfile
+ else:
+ self.repo_file = repo_file
+ # --- end of __init__ (...) ---
+
+ def __eq__ ( self, other ):
+ return not self.__ne__ ( other )
+ # --- end of __eq__ (...) ---
+
+ def __ne__ ( self, other ):
+ if isinstance ( other, DistMapInfo ):
+ return (
+ self.sha256 != other.sha256
+ or self.repo_name != other.repo_name
+ or self.repo_file != other.repo_file
+ )
+ else:
+ return super ( DistMapInfo, self ).__ne__ ( other )
+ # --- end of __ne__ (...) ---
+
+ def to_str ( self, distfile, d ):
+ return ( d.join ((
+ self.repo_name,
+ (
+ self.RESTORE_FROM_DISTFILE if self.repo_file == distfile
+ else self.repo_file
+ ),
+ self.sha256
+ )) )
+ # --- end of to_str (...) ---
+
+# --- end of DistMapInfo ---
+
+
+def get_distmap ( distmap_file, distmap_compression, ignore_missing=False ):
+ if not distmap_compression or (
+ distmap_compression in { 'default', 'none' }
+ ):
+ return FileDistMap (
+ distmap_file, ignore_missing=ignore_missing
+ )
+ elif distmap_compression in { 'bz2', 'bzip2' }:
+ return Bzip2CompressedFileDistMap (
+ distmap_file, ignore_missing=ignore_missing
+ )
+ elif distmap_compression in { 'gz', 'gzip' }:
+ return GzipCompressedFileDistMap (
+ distmap_file, ignore_missing=ignore_missing
+ )
+ else:
+ raise ValueError (
+ "unknown distmap_compression {!r}".format ( distmap_compression )
+ )
+# --- end of get_distmap (...) ---
+
+
+class _DistMapBase ( object ):
+
+ # { attr[, as_attr] }
+ DISTMAP_BIND_ATTR = frozenset ({ 'get', 'keys', 'items', 'values', })
+
+ class AbstractMethod ( NotImplementedError ):
+ pass
+ # --- end of AbstractMethod ---
+
+ def __init__ ( self ):
+ super ( _DistMapBase, self ).__init__()
+ self.dirty = False
+ self._distmap = dict()
+
+ self._rebind_distmap()
+
+ self.update_only = True
+ # --- end of __init__ (...) ---
+
+ def __getitem__ ( self, key ):
+ return self._distmap [key]
+ # --- end of __getitem__ (...) ---
+
+ def __iter__ ( self ):
+ return iter ( self._distmap )
+ # --- end of __iter__ (...) ---
+
+ def __len__ ( self ):
+ return len ( self._distmap )
+ # --- end of __len__ (...) ---
+
+ def __setitem__ ( self, key, value ):
+ if isinstance ( value, DistMapInfo ):
+ self.add_entry ( key, value )
+ elif hasattr ( value, 'get_distmap_value' ):
+ self.add_entry (
+ key, DistMapInfo ( key, *value.get_distmap_value() )
+ )
+ else:
+ self.add_entry ( key, DistMapInfo ( key, *value ) )
+ # --- end of __setitem__ (...) ---
+
+ def _rebind_distmap ( self ):
+ for attr in self.DISTMAP_BIND_ATTR:
+ if isinstance ( attr, str ):
+ setattr ( self, attr, getattr ( self._distmap, attr ) )
+ else:
+ setattr ( self, attr [1], getattr ( self._distmap, attr[0] ) )
+ # --- end of _rebind_distmap (...) ---
+
+ def remove ( self, key ):
+ del self._distmap [key]
+ # --- end of remove (...) ---
+
+ def make_reverse_distmap ( self ):
+ self._reverse_distmap = {
+ ( kv[1].repo_name, kv[1].repo_file ): kv
+ for kv in self._distmap.items()
+ }
+ # --- end of make_reverse_distmap (...) ---
+
+ def release_reverse_distmap ( self ):
+ try:
+ del self._reverse_distmap
+ except AttributeError:
+ pass
+ # --- end of release_reverse_distmap (...) ---
+
+ def lookup ( self, repo_name, repo_file ):
+ if hasattr ( self, '_reverse_distmap' ):
+ return self._reverse_distmap.get ( ( repo_name, repo_file ), None )
+ else:
+ for distfile, info in self._distmap.items():
+ if info.repo_name == repo_name and info.repo_file == repo_file:
+ return ( distfile, info )
+ # -- end if
+ else:
+ return None
+ # --- end of lookup (...) ---
+
+ def add_entry ( self, distfile, distmap_info ):
+ if self.update_only:
+ entry = self._distmap.get ( distfile, None )
+ if entry is None or entry != distmap_info:
+ self._distmap [distfile] = distmap_info
+ self.dirty = True
+ del entry
+ else:
+ self._distmap [distfile] = distmap_info
+ self.dirty = True
+
+ return True
+ # --- end of add_entry (...) ---
+
+ def add_entry_for ( self, p_info ):
+ key = p_info.get_distmap_key()
+ return self.add_entry (
+ key, DistMapInfo ( key, *p_info.get_distmap_value() )
+ )
+ # --- end of add_entry_for (...) ---
+
+ def read ( self, *args, **kwargs ):
+ raise self.__class__.AbstractMethod()
+ # --- end of read (...) ---
+
+ def write ( self, *args, **kwargs ):
+ raise self.__class__.AbstractMethod()
+ # --- end of write (...) ---
+
+
+# --- end of _DistMapBase ---
+
+
+class _FileDistMapBase ( _DistMapBase ):
+
+ FIELD_DELIMITER = '|'
+ #FIELD_DELIMITER = ' '
+ FILE_FORMAT = '0'
+
+ def __init__ ( self, filepath, ignore_missing=False ):
+ super ( _FileDistMapBase, self ).__init__ ()
+ self.dbfile = filepath
+ if ignore_missing:
+ self.try_read()
+ else:
+ self.read()
+ # --- end of __init__ (...) ---
+
+ def backup_file ( self, destfile=None, move=False, ignore_missing=False ):
+ dest = destfile or self.dbfile + '.bak'
+ try:
+ roverlay.util.dodir ( os.path.dirname ( dest ), mkdir_p=True )
+ if move:
+ shutil.move ( self.dbfile, dest )
+ return True
+ else:
+ shutil.copyfile ( self.dbfile, dest )
+ return True
+ except IOError as ioerr:
+ if ignore_missing and ioerr.errno == errno.ENOENT:
+ return False
+ else:
+ raise
+ # --- end of backup_file (...) ---
+
+ def file_exists ( self ):
+ return os.path.isfile ( self.dbfile )
+ # --- end of file_exists (...) ---
+
+ def try_read ( self, *args, **kwargs ):
+ try:
+ self.read()
+ except IOError as ioerr:
+ if ioerr.errno == errno.ENOENT:
+ pass
+ else:
+ raise
+ # --- end of try_read (...) ---
+
+ def _file_written ( self, filepath ):
+ self.dirty = self.dirty and ( filepath is not self.dbfile )
+ # --- end of _file_written (...) ---
+
+ def _file_read ( self, filepath ):
+ self.dirty = self.dirty or ( filepath is not self.dbfile )
+ # --- end of _file_read (...) ---
+
+ def gen_lines ( self ):
+ # header
+ yield "<{d}<{fmt}".format (
+ d=self.FIELD_DELIMITER, fmt=self.FILE_FORMAT
+ )
+ for distfile, info in self._distmap.items():
+ yield (
+ str ( distfile ) + self.FIELD_DELIMITER
+ + info.to_str ( distfile, self.FIELD_DELIMITER )
+ )
+ # --- end of gen_lines (...) ---
+
+ def _read_header ( self, line ):
+ if len ( line ) > 2 and line[0] == line[2]:
+ # instance attr
+ self.FIELD_DELIMITER = line[1]
+ if len ( line ) > 3:
+ self.FILE_FORMAT = line[3:]
+ return True
+ else:
+ return False
+ # --- end of _read_header (...) ---
+
+# --- end of _FileDistMapBase ---
+
+
+class FileDistMap ( _FileDistMapBase ):
+
+ def read ( self, filepath=None ):
+ f = filepath or self.dbfile
+ first = True
+ with open ( f, 'rt') as FH:
+ for line in FH.readlines():
+ rsline = line.rstrip('\n')
+
+ if first:
+ first = False
+ if self._read_header ( rsline ):
+ continue
+ # else no header
+ # -- end if
+
+ distfile, info = roverlay.util.headtail (
+ rsline.split ( self.FIELD_DELIMITER )
+ )
+ self._distmap [distfile] = DistMapInfo ( distfile, *info )
+ self._file_read ( f )
+ # --- end of read_file (...) ---
+
+ def write ( self, filepath=None, force=False ):
+ if force or self.dirty:
+ f = filepath or self.dbfile
+ roverlay.util.dodir ( os.path.dirname ( f ), mkdir_p=True )
+ with open ( f, 'wt' ) as FH:
+ for line in self.gen_lines():
+ FH.write ( line )
+ FH.write ( '\n' )
+ self._file_written ( f )
+ return True
+ else:
+ return False
+ # --- end of write (...) ---
+
+# --- end of FileDistMap ---
+
+class _CompressedFileDistMap ( _FileDistMapBase ):
+
+ _OPEN_COMPRESSED = None
+
+ def read ( self, filepath=None ):
+ f = filepath or self.dbfile
+ first = True
+ with self._OPEN_COMPRESSED ( f, mode='r' ) as FH:
+ for compressed in FH.readlines():
+ rsline = compressed.decode().rstrip('\n')
+
+ if first:
+ first = False
+ if self._read_header ( rsline ):
+ continue
+ # else no header
+ # -- end if
+
+ distfile, info = roverlay.util.headtail (
+ rsline.split ( self.FIELD_DELIMITER )
+ )
+ self._distmap [distfile] = DistMapInfo ( distfile, *info )
+ self._file_read ( f )
+ # --- end of read (...) ---
+
+ def write ( self, filepath=None, force=False ):
+ if force or self.dirty:
+ f = filepath or self.dbfile
+ nl = '\n'.encode()
+ roverlay.util.dodir ( os.path.dirname ( f ), mkdir_p=True )
+ with self._OPEN_COMPRESSED ( f, mode='w' ) as FH:
+ for line in self.gen_lines():
+ FH.write ( line.encode() )
+ FH.write ( nl )
+ self._file_written ( f )
+ return True
+ else:
+ return False
+ # --- end of write (...) ---
+
+# --- end of _CompressedFileDistMap ---
+
+def create_CompressedFileDistMap ( open_compressed ):
+ class CompressedFileDistMap ( _CompressedFileDistMap ):
+ _OPEN_COMPRESSED = open_compressed
+ # --- end of CompressedFileDistMap ---
+ return CompressedFileDistMap
+# --- end of create_CompressedFileDistMap (...) ---
+
+Bzip2CompressedFileDistMap = create_CompressedFileDistMap ( bz2.BZ2File )
+GzipCompressedFileDistMap = create_CompressedFileDistMap ( gzip.GzipFile )
diff --git a/roverlay/recipe/distmap.py b/roverlay/recipe/distmap.py
new file mode 100644
index 0000000..00f3348
--- /dev/null
+++ b/roverlay/recipe/distmap.py
@@ -0,0 +1,43 @@
+# R overlay -- recipe, distmap
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import os.path
+
+import roverlay.config
+import roverlay.db.distmap
+
+__all__ = [ 'access', ]
+
+def setup():
+ """Creates the static distmap instance."""
+ global DISTMAP
+
+ distmap_file = (
+ roverlay.config.get ( 'OVERLAY.DISTMAP.dbfile', None )
+ or (
+ roverlay.config.get_or_fail ( 'CACHEDIR.root' )
+ + os.path.sep + "distmap.db"
+ )
+ )
+
+ if distmap_file:
+ DISTMAP = roverlay.db.distmap.get_distmap (
+ distmap_file = distmap_file,
+ distmap_compression = roverlay.config.get (
+ 'OVERLAY.DISTMAP.compression', 'bz2'
+ ),
+ ignore_missing=True
+ )
+ else:
+ DISTMAP = None
+
+ return DISTMAP
+# --- end of setup (...) ---
+
+def access():
+ """Returns the static distmap instance."""
+ return DISTMAP
+# --- end of access (...) ---
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/, roverlay/recipe/
@ 2013-08-30 14:49 André Erdmann
0 siblings, 0 replies; 2+ messages in thread
From: André Erdmann @ 2013-08-30 14:49 UTC (permalink / raw
To: gentoo-commits
commit: d824fed46faebd39b81983a8f8938bd69970c307
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 30 13:47:49 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 30 13:49:58 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d824fed4
distmap: volatile entries, use roverlay.util.fileio
* 'volatile' status indicator for distmap info entries that are not (yet)
persistent
* use common (un)compressed file read/write code from roverlay.util.fileio
---
roverlay/db/distmap.py | 253 ++++++++++++++++++---------------------------
roverlay/recipe/distmap.py | 2 +-
2 files changed, 99 insertions(+), 156 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index b3b285c..47cfa27 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -14,6 +14,7 @@ import shutil
import roverlay.digest
import roverlay.util
+import roverlay.util.fileio
import roverlay.util.objects
import roverlay.stats.collector
@@ -38,7 +39,9 @@ class DistMapInfo ( object ):
return key, cls ( *value )
# --- end of from_package_info (...) ---
- def __init__ ( self, distfile, repo_name, repo_file, sha256 ):
+ def __init__ (
+ self, distfile, repo_name, repo_file, sha256, volatile=False
+ ):
"""Distmap entry constructor.
arguments:
@@ -46,11 +49,14 @@ class DistMapInfo ( object ):
* repo_name -- name of the repo that owns the package file
* repo_file -- path of the package file relative to the repo
* sha256 -- file checksum
+ * volatile -- whether this entry should be persistent (False) or
+ not (True). Defaults to False.
"""
super ( DistMapInfo, self ).__init__()
self.repo_name = repo_name if repo_name is not None else self.UNSET
self.sha256 = sha256
+ self.volatile = volatile
# references to objects that "own" (use, ...) this distfile
self.backrefs = set()
@@ -113,6 +119,7 @@ class DistMapInfo ( object ):
* field_delimiter -- char (or char sequence) that is used to separate
values
"""
+ assert not self.volatile
return ( field_delimiter.join ((
distfile,
self.repo_name,
@@ -127,37 +134,6 @@ class DistMapInfo ( object ):
# --- end of DistMapInfo ---
-def get_distmap ( distmap_file, distmap_compression, ignore_missing=False ):
- """Returns a new distmap instance.
-
- arguments:
- * distmap_file -- file with distmap info entries
- * distmap_compression -- distmap file compression format (None: disable)
- * ignore_missing -- do not fail if distmap file does not exist?
-
- raises: ValueError if distmap_compression not supported.
- """
- if not distmap_compression or (
- distmap_compression in { 'default', 'none' }
- ):
- return FileDistMap (
- distmap_file, ignore_missing=ignore_missing
- )
- elif distmap_compression in { 'bz2', 'bzip2' }:
- return Bzip2CompressedFileDistMap (
- distmap_file, ignore_missing=ignore_missing
- )
- elif distmap_compression in { 'gz', 'gzip' }:
- return GzipCompressedFileDistMap (
- distmap_file, ignore_missing=ignore_missing
- )
- else:
- raise ValueError (
- "unknown distmap_compression {!r}".format ( distmap_compression )
- )
-# --- end of get_distmap (...) ---
-
-
class _DistMapBase ( object ):
# { attr[, as_attr] }
@@ -219,6 +195,18 @@ class _DistMapBase ( object ):
self.dirty = True
# --- end of _file_removed (...) ---
+ def _iter_persistent ( self ):
+ for distfile, info in self._distmap.items():
+ if not info.volatile:
+ yield ( distfile, info )
+ # --- end of _iter_persistent (...) ---
+
+ def _iter_volatile ( self ):
+ for distfile, info in self._distmap.items():
+ if info.volatile:
+ yield ( distfile, info )
+ # --- end of _iter_volatile (...) ---
+
def _rebind_distmap ( self ):
for attr in self.DISTMAP_BIND_ATTR:
if isinstance ( attr, str ):
@@ -239,12 +227,19 @@ class _DistMapBase ( object ):
new_entry.add_backref ( backref )
# --- end of add_distfile_owner (...) ---
+ def gen_info_lines ( self, field_delimiter ):
+ for distfile, info in self._distmap.items():
+ if not info.volatile:
+ yield info.to_str ( str ( distfile ), field_delimiter )
+ # --- end of gen_info_lines (...) ---
+
def get_distfile_slot ( self, backref, distfile ):
entry = self.get_entry ( distfile )
if entry is None:
raise NotImplementedError ( "backref gets new 'slot'." )
return 1
elif entry.has_backref_to ( backref.deref_safe() ):
+ # revbump check might be necessary
return 2
else:
raise NotImplementedError ( "handle file collision." )
@@ -461,7 +456,7 @@ class _DistMapBase ( object ):
# --- end of _DistMapBase ---
-class _FileDistMapBase ( _DistMapBase ):
+class FileDistMap ( _DistMapBase ):
"""A distmap that is read from / written to a file."""
# the default info field separator
@@ -471,9 +466,35 @@ class _FileDistMapBase ( _DistMapBase ):
# file format (reserved for future usage)
FILE_FORMAT = '0'
- def __init__ ( self, filepath, ignore_missing=False ):
- super ( _FileDistMapBase, self ).__init__ ()
- self.dbfile = filepath
+ def set_compression ( self, compression ):
+ if not compression or compression in { 'default', 'none' }:
+ self.compression = None
+ elif compression in roverlay.util.fileio.SUPPORTED_COMPRESSION:
+ self.compression = compression
+ else:
+ raise ValueError (
+ "unknown distmap compression {!r}".format ( compression )
+ )
+ # --- end of set_compression (...) ---
+
+ def __init__ (
+ self, distmap_file, distmap_compression=None, ignore_missing=False
+ ):
+ """Constructor for a distmap that stores its information to a file,
+ optionally compressed.
+
+ arguments:
+ * distmap_file -- file with distmap info entries
+ * distmap_compression -- distmap file compression format (None: disable)
+ * ignore_missing -- do not fail if distmap file does not exist?
+
+ raises: ValueError if distmap_compression not supported.
+ """
+ super ( FileDistMap, self ).__init__ ()
+ self.dbfile = distmap_file
+ self.compression = None
+ self.set_compression ( distmap_compression )
+
if ignore_missing:
self.try_read()
else:
@@ -547,24 +568,24 @@ class _FileDistMapBase ( _DistMapBase ):
raise
# --- end of try_read (...) ---
- def _file_written ( self, filepath ):
- """Method that should be called after writing a distmap file."""
- self.dirty = self.dirty and ( filepath is not self.dbfile )
- # --- end of _file_written (...) ---
+ def get_header ( self ):
+ return "<{d}<{fmt}".format (
+ d=self.FIELD_DELIMITER, fmt=self.FILE_FORMAT
+ )
+ # --- end of get_header (...) ---
- def _file_read ( self, filepath ):
- """Method that should be called after reading a distmap file."""
- self.dirty = self.dirty or ( filepath is not self.dbfile )
- # --- end of _file_read (...) ---
+ def gen_info_lines ( self ):
+ for distfile, info in self._distmap.items():
+ if not info.volatile:
+ yield info.to_str ( str ( distfile ), self.FIELD_DELIMITER )
+ # --- end of gen_info_lines (...) ---
def gen_lines ( self ):
"""Generator that creates distmap file text lines."""
# header
- yield "<{d}<{fmt}".format (
- d=self.FIELD_DELIMITER, fmt=self.FILE_FORMAT
- )
- for distfile, info in self._distmap.items():
- yield info.to_str ( str ( distfile ), self.FIELD_DELIMITER )
+ yield self.get_header()
+ for line in self.gen_info_lines():
+ yield line
# --- end of gen_lines (...) ---
def _read_header ( self, line ):
@@ -584,17 +605,35 @@ class _FileDistMapBase ( _DistMapBase ):
return False
# --- end of _read_header (...) ---
- @roverlay.util.objects.abstractmethod
def read ( self, filepath=None ):
"""Reads the distmap.
arguments:
* filepath -- path to the distmap file (defaults to self.dbfile)
"""
- pass
+ dbfile = self.dbfile if filepath is None else filepath
+ first = True
+
+ for line in roverlay.util.fileio.read_text_file (
+ dbfile, preparse=True, try_harder=True
+ ):
+ if first:
+ first = False
+ if self._read_header ( line ):
+ continue
+ # else no header
+ # -- end if
+
+ distfile, info = roverlay.util.headtail (
+ line.split ( self.FIELD_DELIMITER )
+ )
+ self._distmap [distfile] = DistMapInfo ( distfile, *info )
+ self._nondirty_file_added ( distfile )
+ # -- end for
+ self.dirty = self.dirty or filepath is not None
+ print( list(self.gen_info_lines()) )
# --- end of read (...) ---
- @roverlay.util.objects.abstractmethod
def write ( self, filepath=None, force=False ):
"""Writes the distmap.
@@ -602,112 +641,16 @@ class _FileDistMapBase ( _DistMapBase ):
* filepath -- path to the distmap file (defaults to self.dbfile)
* force -- enforce writing even if distmap not modified
"""
- pass
- # --- end of write (...) ---
-
-# --- end of _FileDistMapBase ---
-
-
-class FileDistMap ( _FileDistMapBase ):
-
- def read ( self, filepath=None ):
- f = filepath or self.dbfile
- first = True
- with open ( f, 'rt') as FH:
- for line in FH.readlines():
- rsline = line.rstrip('\n')
-
- if first:
- first = False
- if self._read_header ( rsline ):
- continue
- # else no header
- # -- end if
-
- distfile, info = roverlay.util.headtail (
- rsline.split ( self.FIELD_DELIMITER )
- )
- self._distmap [distfile] = DistMapInfo ( distfile, *info )
- self._nondirty_file_added ( distfile )
- # -- end for
- self._file_read ( f )
- # --- end of read_file (...) ---
-
- def write ( self, filepath=None, force=False ):
- if force or self.dirty:
- f = filepath or self.dbfile
- roverlay.util.dodir ( os.path.dirname ( f ), mkdir_p=True )
- with open ( f, 'wt' ) as FH:
- for line in self.gen_lines():
- FH.write ( line )
- FH.write ( '\n' )
- self._file_written ( f )
+ if force or self.dirty or filepath is not None:
+ dbfile = self.dbfile if filepath is None else filepath
+ roverlay.util.fileio.write_text_file (
+ dbfile, self.gen_lines(),
+ compression=self.compression, create_dir=True
+ )
+ self.dirty = self.dirty and filepath is not None
return True
else:
return False
# --- end of write (...) ---
# --- end of FileDistMap ---
-
-class _CompressedFileDistMap ( _FileDistMapBase ):
-
- # _OPEN_COMPRESSED:
- # callable that returns a file handle for reading compressed files
- #
- _OPEN_COMPRESSED = None
-
- def read ( self, filepath=None ):
- f = filepath or self.dbfile
- first = True
- with self._OPEN_COMPRESSED ( f, mode='r' ) as FH:
- for compressed in FH.readlines():
- rsline = compressed.decode().rstrip('\n')
-
- if first:
- first = False
- if self._read_header ( rsline ):
- continue
- # else no header
- # -- end if
-
- distfile, info = roverlay.util.headtail (
- rsline.split ( self.FIELD_DELIMITER )
- )
- self._distmap [distfile] = DistMapInfo ( distfile, *info )
- self._nondirty_file_added ( distfile )
- # -- end for
- self._file_read ( f )
- # --- end of read (...) ---
-
- def write ( self, filepath=None, force=False ):
- if force or self.dirty:
- f = filepath or self.dbfile
- nl = '\n'.encode()
- roverlay.util.dodir ( os.path.dirname ( f ), mkdir_p=True )
- with self._OPEN_COMPRESSED ( f, mode='w' ) as FH:
- for line in self.gen_lines():
- FH.write ( line.encode() )
- FH.write ( nl )
- self._file_written ( f )
- return True
- else:
- return False
- # --- end of write (...) ---
-
-# --- end of _CompressedFileDistMap ---
-
-def create_CompressedFileDistMap ( open_compressed ):
- """Creates a CompressedFileDistMap class.
-
- arguments:
- * open_compressed -- function that returns a file handle for reading
- """
- class CompressedFileDistMap ( _CompressedFileDistMap ):
- _OPEN_COMPRESSED = open_compressed
- # --- end of CompressedFileDistMap ---
- return CompressedFileDistMap
-# --- end of create_CompressedFileDistMap (...) ---
-
-# bzip2, gzip
-Bzip2CompressedFileDistMap = create_CompressedFileDistMap ( bz2.BZ2File )
-GzipCompressedFileDistMap = create_CompressedFileDistMap ( gzip.GzipFile )
diff --git a/roverlay/recipe/distmap.py b/roverlay/recipe/distmap.py
index 00f3348..27a67bf 100644
--- a/roverlay/recipe/distmap.py
+++ b/roverlay/recipe/distmap.py
@@ -24,7 +24,7 @@ def setup():
)
if distmap_file:
- DISTMAP = roverlay.db.distmap.get_distmap (
+ DISTMAP = roverlay.db.distmap.FileDistMap (
distmap_file = distmap_file,
distmap_compression = roverlay.config.get (
'OVERLAY.DISTMAP.compression', 'bz2'
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2013-08-30 14:49 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-08-30 14:49 [gentoo-commits] proj/R_overlay:master commit in: roverlay/db/, roverlay/recipe/ André Erdmann
-- strict thread matches above, loose matches on Subject: below --
2013-06-19 18:58 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
2013-06-22 15:24 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox