* [gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/distroot/, roverlay/db/, roverlay/, ...
2013-06-22 15:24 [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/distroot/, roverlay/db/, roverlay/, André Erdmann
@ 2013-06-20 23:40 ` André Erdmann
0 siblings, 0 replies; 2+ messages in thread
From: André Erdmann @ 2013-06-20 23:40 UTC (permalink / raw
To: gentoo-commits
commit: 6a34fb49dd27014154427e95de9f1f7917cd90b9
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jun 20 23:34:54 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jun 20 23:34:54 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6a34fb49
roverlay: use distmap
Add distmap support to distroot/overlay.
revbump functionality in PackageInfo is missing and some debug output has to be
removed.
---
roverlay/db/distmap.py | 81 ++++++++++-
roverlay/overlay/creator.py | 11 +-
roverlay/overlay/pkgdir/distroot/distdir.py | 10 +-
roverlay/overlay/pkgdir/distroot/distroot.py | 155 ++++++++++++++++++---
roverlay/overlay/pkgdir/distroot/static.py | 10 +-
roverlay/overlay/pkgdir/packagedir_base.py | 48 +++++--
.../overlay/pkgdir/packagedir_ebuildmanifest.py | 2 +-
.../overlay/pkgdir/packagedir_portagemanifest.py | 4 +-
roverlay/overlay/root.py | 7 +-
roverlay/packageinfo.py | 48 ++++++-
10 files changed, 335 insertions(+), 41 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 7bfba59..dfdcd01 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -13,6 +13,7 @@ import os.path
import shutil
+import roverlay.digest
import roverlay.util
@@ -21,20 +22,33 @@ __all__ = [ 'DistMapInfo', 'get_distmap' ]
class DistMapInfo ( object ):
+ DIGEST_TYPE = 'sha256'
RESTORE_FROM_DISTFILE = '_'
+ UNSET = 'U'
def __init__ ( self, distfile, repo_name, repo_file, sha256 ):
super ( DistMapInfo, self ).__init__()
- self.repo_name = repo_name
+ self.repo_name = repo_name if repo_name is not None else self.UNSET
self.sha256 = sha256
if repo_file == self.RESTORE_FROM_DISTFILE:
self.repo_file = distfile
else:
- self.repo_file = repo_file
+ self.repo_file = repo_file if repo_file is not None else self.UNSET
# --- end of __init__ (...) ---
+ @property
+ def digest ( self ):
+ return self.sha256
+ #return getattr ( self, self.DIGEST_TYPE )
+ # --- end of digest (...) ---
+
+ def compare_digest ( self, package_info ):
+ p_hash = package_info.make_distmap_hash()
+ return ( bool ( p_hash == self.digest ), p_hash )
+ # --- end of compare_digest (...) ---
+
def __eq__ ( self, other ):
return not self.__ne__ ( other )
# --- end of __eq__ (...) ---
@@ -136,10 +150,61 @@ class _DistMapBase ( object ):
setattr ( self, attr [1], getattr ( self._distmap, attr[0] ) )
# --- end of _rebind_distmap (...) ---
+ def check_revbump_necessary ( self, package_info ):
+ """Tries to find package_info's distfile in the distmap and returns
+ whether a revbump is necessary (True) or not (False).
+
+ Compares checksums if distfile already exists.
+
+ arguments:
+ * package_info --
+ """
+ distfile = package_info.get_distmap_key()
+
+ info = self._distmap.get ( distfile, None )
+ if info is None:
+ # new file, no revbump required
+ return False
+ elif info.compare_digest ( package_info ) [0] is True:
+ # old digest == new digest, no revbump
+ # (package_info should be filtered out)
+ return False
+ else:
+ # digest mismatch => diff
+ return True
+ # --- end of compare_digest (...) ---
+
+ def get_file_digest ( self, f ):
+ return roverlay.digest.dodigest_file ( f, DistMapInfo.DIGEST_TYPE )
+ # --- end of get_file_digest (...) ---
+
+ def check_integrity ( self, distfile, distfilepath ):
+ info = self._distmap.get ( distfile, None )
+
+ if info is None:
+ # file not found
+ return 1
+ elif info.digest == self.get_file_digest ( distfilepath ):
+ # file OK
+ return 0
+ else:
+ # bad checksum
+ return 2
+ # --- end of check_integrity (...) ---
+
def remove ( self, key ):
del self._distmap [key]
+ self.dirty = True
# --- end of remove (...) ---
+ def try_remove ( self, key ):
+ try:
+ del self._distmap [key]
+ self.dirty = True
+ except KeyError:
+ pass
+ # --- end of try_remove (...) ---
+
def make_reverse_distmap ( self ):
self._reverse_distmap = {
( kv[1].repo_name, kv[1].repo_file ): kv
@@ -187,6 +252,16 @@ class _DistMapBase ( object ):
)
# --- end of add_entry_for (...) ---
+ def add_dummy_entry ( self, distfile, distfilepath ):
+ print ( "DUMMY", distfile )
+ return self.add_entry (
+ distfile,
+ DistMapInfo (
+ distfile, None, None, self.get_file_digest ( distfilepath ),
+ )
+ )
+ # --- end of add_dummy_entry (...) ---
+
def read ( self, *args, **kwargs ):
raise self.__class__.AbstractMethod()
# --- end of read (...) ---
@@ -304,6 +379,7 @@ class FileDistMap ( _FileDistMapBase ):
def write ( self, filepath=None, force=False ):
if force or self.dirty:
+ print ( "DBFILE WILL BE WRITTEN", force, dirty, list(self.keys()) )
f = filepath or self.dbfile
roverlay.util.dodir ( os.path.dirname ( f ), mkdir_p=True )
with open ( f, 'wt' ) as FH:
@@ -313,6 +389,7 @@ class FileDistMap ( _FileDistMapBase ):
self._file_written ( f )
return True
else:
+ print ( "DBFILE WILL NOT BE WRITTEN", force, dirty, list(self.keys()) )
return False
# --- end of write (...) ---
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 4c29ffa..79a70bf 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -34,6 +34,9 @@ from roverlay.packagerules.rules import PackageRules
from roverlay.recipe import easyresolver
+import roverlay.recipe.distmap
+import roverlay.overlay.pkgdir.distroot.static
+
class PseudoAtomicCounter ( object ):
@@ -97,6 +100,10 @@ class OverlayCreator ( object ):
self.rsuggests_flags = set()
+ # create distmap and distroot here
+ self.distmap = roverlay.recipe.distmap.setup()
+ self.distroot = roverlay.overlay.pkgdir.distroot.static.get_configured()
+
time_scan_overlay = time.time()
# init overlay using config values
self.overlay = Overlay.new_configured (
@@ -462,10 +469,10 @@ class OverlayCreator ( object ):
"""
if package_info ['ebuild'] is not None:
self.create_success.inc()
- if package_info.overlay_package_ref.new_ebuild():
+ if package_info.overlay_package_ref().new_ebuild():
self.overlay_added.inc()
else:
- package_info.overlay_package_ref.ebuild_uncreateable ( package_info )
+ package_info.overlay_package_ref().ebuild_uncreateable ( package_info )
self.create_fail.inc()
# --- end of _add_to_overlay (...) ---
diff --git a/roverlay/overlay/pkgdir/distroot/distdir.py b/roverlay/overlay/pkgdir/distroot/distdir.py
index 0f76e30..62cb210 100644
--- a/roverlay/overlay/pkgdir/distroot/distdir.py
+++ b/roverlay/overlay/pkgdir/distroot/distdir.py
@@ -22,11 +22,15 @@ class Distdir ( object ):
self._distroot = distroot
# --- end of __init__ (...) ---
- def add ( self, fpath, fname=None ):
- return self._distroot._add (
+ def add ( self, fpath, fname, p_info ):
+ if self._distroot._add (
fpath,
self.get_root() + os.sep + ( fname or os.path.basename ( fpath ) )
- )
+ ):
+ self._distroot.distmap_register ( p_info )
+ return True
+ else:
+ return False
# --- end of add (...) ---
def get_root ( self ):
diff --git a/roverlay/overlay/pkgdir/distroot/distroot.py b/roverlay/overlay/pkgdir/distroot/distroot.py
index 9f56742..c59d813 100644
--- a/roverlay/overlay/pkgdir/distroot/distroot.py
+++ b/roverlay/overlay/pkgdir/distroot/distroot.py
@@ -15,8 +15,11 @@ import os
import shutil
import tempfile
+
+import roverlay.db.distmap
import roverlay.overlay.pkgdir.distroot.distdir
+
class DistrootBase ( object ):
"""Base class for distroots."""
@@ -31,20 +34,24 @@ class DistrootBase ( object ):
return self.get_root()
# --- end of __str__ (...) ---
- def __init__ ( self, root, flat ):
+ def __init__ ( self, root, flat, logger=None, distmap=None ):
"""DistrootBase constructor.
arguments:
- * root -- root directory
- * flat -- whether to use a flat structure (all packages in a single
- directory, True) or per-package sub directories (False)
+ * root -- root directory
+ * flat -- whether to use a flat structure (all packages in a single
+ directory, True) or per-package sub directories (False)
+ * logger --
+ * distmap --
"""
super ( DistrootBase, self ).__init__()
- self.logger = logging.getLogger ( self.__class__.__name__ )
+ self.logger = logger or logging.getLogger ( self.__class__.__name__ )
self._root = root
# or use hasattr ( self, '_default_distdir' )
self._flat = flat
+ self.distmap = distmap
+
if flat:
self._default_distdir = (
roverlay.overlay.pkgdir.distroot.distdir.Distdir ( self )
@@ -166,29 +173,66 @@ class DistrootBase ( object ):
pass
# --- end of _prepare (...) ---
+ def iter_distfiles ( self, distfile_only ):
+ def recursive_iter ( root_abspath, root_relpath ):
+ for item in os.listdir ( root_abspath ):
+ abspath = root_abspath + os.sep + item
+ relpath = (
+ item if root_relpath is None
+ else root_relpath + os.sep + item
+ )
+ if os.path.isdir ( abspath ):
+ for result in iter_distfiles ( abspath, relpath ):
+ yield result
+ else:
+ yield ( abspath, relpath )
+ # --- end of recursive_iter (...) ---
+
+ if distfile_only:
+ for pkgfile, distfile in recursive_iter ( self.get_root(), None ):
+ yield distfile
+ else:
+ for pkgfile, distfile in recursive_iter ( self.get_root(), None ):
+ yield ( pkgfile, distfile )
+ # --- end of iter_distfiles (...) ---
+
def _remove_broken_symlinks ( self ):
"""Recursively remove broken/dead symlinks."""
- def recursive_remove ( root, rmdir ):
- for item in os.listdir ( root ):
- fpath = root + os.sep + item
+
+ def recursive_remove ( dirpath, rel_dirpath, rmdir ):
+ for item in os.listdir ( dirpath ):
+ fpath = dirpath + os.sep + item
+ relpath = (
+ item if rel_dirpath is None else rel_dirpath + os.sep + item
+ )
if not os.path.exists ( fpath ):
+ # drop broken symlink
+ self.logger.debug (
+ "Removing broken symlink {!r}".format ( fpath )
+ )
os.unlink ( fpath )
-
+ if self.distmap is not None:
+ self.distmap.try_remove ( relpath )
elif os.path.isdir ( fpath ):
- recursive_remove ( fpath, True )
+ recursive_remove ( fpath, relpath, True )
if rmdir:
try:
os.rmdir ( fpath )
except OSError:
pass
+ # -- end for
# --- end of recursive_remove (...) ---
- recursive_remove ( self.get_root(), False )
+
+ return recursive_remove ( self.get_root(), None, False )
# --- end of _remove_broken_symlinks (...) ---
def _try_remove ( self, dest ):
try:
os.unlink ( dest )
+ if self.distmap is not None:
+ relpath = os.path.relpath ( dest, self.get_root() )
+
except OSError as e:
if e.errno == errno.ENOENT:
pass
@@ -219,16 +263,72 @@ class DistrootBase ( object ):
return str ( self._root )
# --- end of get_root (...) ---
+ def distmap_register ( self, p_info ):
+ return self.distmap.add_entry_for ( p_info )
+ # --- end of distmap_register (...) ---
+
+ def check_integrity ( self ):
+ if self.distmap is not None:
+ root = self.get_root()
+ distfiles = set()
+ checkfile = self.distmap.check_integrity
+
+ for abspath, relpath in self.iter_distfiles ( False ):
+ status = checkfile ( relpath, abspath )
+
+ if status == 0:
+ self.logger.debug (
+ "file has been verified: {!r}".format ( relpath )
+ )
+ distfiles.add ( relpath )
+ elif status == 1:
+ # file not in distmap
+ self.logger.info (
+ "file not in distmap, creating dummy entry: {!r}".format ( relpath )
+ )
+ self.distmap.add_dummy_entry ( relpath, abspath )
+ distfiles.add ( relpath )
+ elif status == 2:
+ # file in distmap, but not valid - remove it from distmap
+ self.logger.warning (
+ "digest mismatch: {!r}".format ( relpath )
+ )
+ self.distmap.remove ( relpath )
+ # -- end for
+
+ distmap_keys = frozenset ( self.distmap.keys() )
+
+ if distfiles:
+ # reverse compare
+ for distfile in distmap_keys:
+ if distfile not in distfiles:
+ self.logger.warning (
+ "distmap file does not exist: {!r}".format ( distfile )
+ )
+ self.distmap.remove ( distfile )
+
+ else:
+ # no files from distroot in distmap -- invalidate distmap
+ for distfile in distmap_keys:
+ self.logger.warning (
+ "distmap file does not exist: {!r}".format ( distfile )
+ )
+ self.distmap.remove ( distfile )
+ else:
+ raise Exception ( "check_integrity() needs a distmap." )
+ # --- end of check_integrity (...) ---
+
# --- end of DistrootBase ---
class TemporaryDistroot ( DistrootBase ):
- def __init__ ( self ):
+ def __init__ ( self, logger=None ):
# temporary distroots always use the non-flat distdir layout
super ( TemporaryDistroot, self ).__init__ (
- root = tempfile.mkdtemp ( prefix='tmp_roverlay_distroot_' ),
- flat = False
+ root = tempfile.mkdtemp ( prefix='tmp_roverlay_distroot_' ),
+ flat = False,
+ logger = logger,
)
# --- end of __init__ (...) ---
@@ -238,6 +338,7 @@ class TemporaryDistroot ( DistrootBase ):
def _cleanup ( self ):
"""Cleans up the temporary distroot by simply wiping it."""
+ super ( TemporaryDistroot, self )._cleanup()
shutil.rmtree ( self._root )
# --- end of _cleanup (...) ---
@@ -265,7 +366,9 @@ class PersistentDistroot ( DistrootBase ):
)
# --- end of __repr__ (...) ---
- def __init__ ( self, root, flat, strategy ):
+ def __init__ ( self,
+ root, flat, strategy, distmap, verify=False, logger=None
+ ):
"""Initializes a non-temporary distroot.
arguments:
@@ -274,8 +377,13 @@ class PersistentDistroot ( DistrootBase ):
* strategy -- the distroot 'strategy' that determines what mode (sym-
link, hardlink, copy) will be tried in which order
This has to be an iterable with valid items.
+ * distmap --
+ * verify --
+ * logger --
"""
- super ( PersistentDistroot, self ).__init__ ( root=root, flat=flat )
+ super ( PersistentDistroot, self ).__init__ (
+ root=root, flat=flat, logger=logger, distmap=distmap
+ )
self._strategy = self._get_int_strategy ( strategy )
@@ -294,6 +402,15 @@ class PersistentDistroot ( DistrootBase ):
self.USE_HARDLINK : self._add_hardlink,
self.USE_COPY : self._add_file,
}
+
+
+ if verify and self.distmap is not None:
+ # FIXME debug print
+ print (
+ "Checking distroot file integrity, this may take some time ... "
+ )
+ self.check_integrity()
+ print ( "Done!" )
# --- end of __init__ (...) ---
def _add ( self, src, dest ):
@@ -328,10 +445,14 @@ class PersistentDistroot ( DistrootBase ):
# --- end of _add (...) ---
def _cleanup ( self ):
+ super ( PersistentDistroot, self )._cleanup()
if hasattr ( self, '_supported_modes_initial' ):
if self._supported_modes_initial & self.USE_SYMLINK:
self._remove_broken_symlinks()
- # --- end of _prepare (...) ---
+
+ if self.distmap is not None:
+ self.distmap.write ( force=False )
+ # --- end of _cleanup (...) ---
def _get_int_strategy ( self, strategy ):
"""Converts the given strategy into its integer tuple representation.
diff --git a/roverlay/overlay/pkgdir/distroot/static.py b/roverlay/overlay/pkgdir/distroot/static.py
index b4ad055..0054140 100644
--- a/roverlay/overlay/pkgdir/distroot/static.py
+++ b/roverlay/overlay/pkgdir/distroot/static.py
@@ -8,6 +8,8 @@ __all__ = [ 'get_configured', 'get_distdir', ]
import threading
+import roverlay.recipe.distmap
+
import roverlay.config
import roverlay.overlay.pkgdir.distroot.distroot
@@ -34,7 +36,11 @@ def get_configured ( static=True ):
# generally, the "flat" distroot/distdir layout is desired as it
# can serve as package mirror directory, so default to True here
flat = roverlay.config.get ( 'OVERLAY.DISTDIR.flat', True ),
- strategy = distdir_strategy
+ strategy = distdir_strategy,
+ distmap = roverlay.recipe.distmap.access(),
+ # FIXME/TODO: 'verify' config key does not exist
+ verify = roverlay.config.get ( 'OVERLAY.DISTDIR.verify', False ),
+
)
# --- end of get_new (...) ---
@@ -49,7 +55,7 @@ def get_configured ( static=True ):
return _distroot_instance
else:
- return get_new()
+ raise Exception ( "static keyword arg is deprecated" )
# --- end of get_configured (...) ---
def get_distdir ( ebuild_name ):
diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index e67114e..a68ecbd 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -20,12 +20,14 @@ import os
import shutil
import sys
import threading
-
+import weakref
import roverlay.config
import roverlay.packageinfo
import roverlay.util
+import roverlay.recipe.distmap
+
import roverlay.tools.ebuild
import roverlay.tools.ebuildenv
import roverlay.tools.patch
@@ -40,8 +42,10 @@ class PackageDirBase ( object ):
for Manifest file creation."""
#DISTROOT =
+ #DISTMAP =
EBUILD_SUFFIX = '.ebuild'
#FETCHENV =
+ #HASHES =
SUPPRESS_EXCEPTIONS = True
# MANIFEST_THREADSAFE (tri-state)
@@ -59,9 +63,11 @@ class PackageDirBase ( object ):
roverlay.config.get_or_fail ( 'OVERLAY.dir' )
)
- cls.DISTROOT = (
- roverlay.overlay.pkgdir.distroot.static.get_configured ( static=True )
- )
+ cls.DISTROOT = roverlay.overlay.pkgdir.distroot.static.get_configured()
+ cls.DISTMAP = roverlay.recipe.distmap.access()
+ # FIXME/TODO: config
+ #cls.HASHES = frozenset ({ 'sha256', 'sha512', 'whirlpool', })
+ cls.HASHES = frozenset ({ 'sha256', })
cls.FETCHENV = fetch_env
# --- end of init_cls (...) ---
@@ -142,11 +148,12 @@ class PackageDirBase ( object ):
p = roverlay.packageinfo.PackageInfo (
physical_only=True, pvr=pvr, ebuild_file=efile
)
+ # TODO/FIXME: parse SRC_URI, knowledge of distfile path would be good...
self._packages [ p ['ebuild_verstr'] ] = p
return p
# --- end of _scan_add_package (...) ---
- def add ( self, package_info, add_if_physical=False ):
+ def add ( self, package_info, add_if_physical=False, _lock=True ):
"""Adds a package to this PackageDir.
arguments:
@@ -161,7 +168,8 @@ class PackageDirBase ( object ):
shortver = package_info ['ebuild_verstr']
added = False
try:
- self._lock.acquire()
+ if _lock: self._lock.acquire()
+
if shortver in self._packages:
# package exists, check if it existed before script invocation
if self._packages [shortver] ['physical_only']:
@@ -170,6 +178,12 @@ class PackageDirBase ( object ):
self._packages [shortver] = package_info
added = True
+ elif self.DISTMAP.check_revbump_necessary ( package_info ):
+ # resolve by recursion
+ added = self.add (
+ package_info.revbump(), add_if_physical=False, _lock=False
+ )
+
else:
self.logger.debug (
"'{PN}-{PVR}.ebuild' exists as file, skipping.".format (
@@ -190,12 +204,12 @@ class PackageDirBase ( object ):
added = True
finally:
- self._lock.release()
+ if _lock: self._lock.release()
if added:
# add a link to this PackageDir into the package info,
# !! package_info <-> self (double-linked)
- package_info.overlay_package_ref = self
+ package_info.overlay_package_ref = weakref.ref ( self )
return True
else:
return False
@@ -786,6 +800,10 @@ class PackageDirBase ( object ):
or patch_ebuild ( efile, pvr, patchview.get_patches ( pvr ) )
):
+ # generate hashes here (benefit from threading)
+ # FIXME/TODO: ^ actually faster?
+ p_info.make_hashes ( self.HASHES )
+
self._need_manifest = True
# update metadata for each successfully written ebuild
@@ -848,6 +866,20 @@ class PackageDirBase ( object ):
if p.has ( 'package_file', 'ebuild_file' )
]
+ print ( "--- DEBUG PRINT from packagedir_base :: {} :: {} ---".format (
+ self.__class__.__name__, self.name
+ )
+ )
+ print ( "Calculating hashes ..." )
+ for p in pkgs_for_manifest:
+ p.make_hashes ( self.HASHES )
+ print ( p.hashdict )
+ print ( "Done!" )
+ print ( "--- END DEBUG PRINT from packagedir_base :: {} :: {} ---".format (
+ self.__class__.__name__, self.name
+ )
+ )
+
if pkgs_for_manifest:
self.logger.debug ( "Writing Manifest" )
if self._write_manifest ( pkgs_for_manifest ):
diff --git a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
index 5217f6a..9766606 100644
--- a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
@@ -102,7 +102,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
# TODO: optimize this further?
# -> "not has physical_only?"
# (should be covered by "has package_file")
- distdir.add ( p ['package_file'], p ['package_src_destpath'] )
+ distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
# -- end for;
return self._do_ebuildmanifest ( ebuild_file, distdir )
diff --git a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
index 6596dec..f4527a1 100644
--- a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
@@ -151,7 +151,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
manifest.addFile ( 'EBUILD', ebuild_filename )
if not manifest.hasFile ( 'DIST', package_filename ):
- distdir.add ( p ['package_file'], package_filename )
+ distdir.add ( p ['package_file'], package_filename, p )
manifest.addFile (
'DIST',
package_filename,
@@ -165,7 +165,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
# doing the check above (if hasFile())
# be "fully compatible" and do that here, too
try:
- distdir.add ( p ['package_file'], package_filename )
+ distdir.add ( p ['package_file'], package_filename, p )
except Exception as err:
self.logger.exception ( err )
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index c2522b7..a3b02f1 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -23,14 +23,15 @@ import os
import shutil
import threading
+
import roverlay.config
import roverlay.util
+import roverlay.recipe.distmap
import roverlay.overlay.additionsdir
import roverlay.overlay.category
import roverlay.overlay.header
-
class Overlay ( object ):
DEFAULT_USE_DESC = (
'byte-compile - enable byte compiling\n'
@@ -730,6 +731,10 @@ class Overlay ( object ):
write_manifest = not self.skip_manifest,
additions_dir = self.additions_dir.get_obj_subdir ( cat ),
)
+
+ distmap = roverlay.recipe.distmap.access()
+ distmap.backup_file ( ignore_missing=True )
+ distmap.write()
else:
# FIXME debug print
print (
diff --git a/roverlay/packageinfo.py b/roverlay/packageinfo.py
index e8004d0..83a05ed 100644
--- a/roverlay/packageinfo.py
+++ b/roverlay/packageinfo.py
@@ -19,6 +19,7 @@ import logging
import threading
import roverlay.digest
+import roverlay.db.distmap
from roverlay import config, strutil
from roverlay.rpackage import descriptionreader
@@ -116,6 +117,9 @@ class PackageInfo ( object ):
'ebuild'
))
+ # bind DIGEST_TYPE to this class
+ DISTMAP_DIGEST_TYPE = roverlay.db.distmap.DistMapInfo.DIGEST_TYPE
+
def __init__ ( self, **initial_info ):
"""Initializes a PackageInfo.
@@ -451,25 +455,45 @@ class PackageInfo ( object ):
# --- end of get_desc_data (...) ---
def get_distmap_item ( self ):
+ """Returns a 2-tuple ( key, info ) for the distmap."""
return ( self.get_distmap_key(), self.get_distmap_value() )
# --- end of get_distmap_item (...) ---
def get_distmap_key ( self ):
+ """Returns a key for the distmap."""
return self.get ( "package_src_destpath" )
# --- end of get_distmap_key (...) ---
- def get_distmap_value ( self ):
- assert 'sha256' in self.hashdict
+ def get_distmap_value ( self, allow_digest_create=False ):
+ """Returns a data tuple for creating DistMapInfo instances.
+ arguments:
+ * allow_digest_create --
+ """
repo = self.get ( "origin" )
return (
repo.name,
os.path.relpath ( self.get ( "package_file" ), repo.distdir ),
- self.hashdict ['sha256']
+ (
+ self.make_distmap_hash() if allow_digest_create
+ else self.hashdict [self.DISTMAP_DIGEST_TYPE]
+ )
)
# --- end of get_distmap_value (...) ---
+ def make_distmap_hash ( self ):
+ """Creates (and returns) the distmap package file hash."""
+ return self.make_hashes ( { self.DISTMAP_DIGEST_TYPE, } ) [self.DISTMAP_DIGEST_TYPE]
+ # --- end of make_distmap_hash (...) ---
+
def make_hashes ( self, hashlist ):
+ """Creates zero or more hashes and returns the hashdict.
+
+ Note: the hashdict can also be accessed directly via <this>.hashdict.
+
+ arguments:
+ * hashlist -- list of hash names, e.g. "sha256", "md5"
+ """
pkgfile = self.get ( "package_file" )
if hasattr ( self, 'hashdict' ) and self.hashdict:
@@ -483,8 +507,26 @@ class PackageInfo ( object ):
)
else:
self.hashdict = roverlay.digest.multihash_file ( pkgfile, hashlist )
+
+ return self.hashdict
# --- end of make_hashes (...) ---
+ def revbump ( self, newrev=None ):
+ """Do whatever necessary to revbump this pakages, that is set/update
+ all data like src_uri_destfile.
+
+ arguments:
+ * newrev -- new revision, (current rev + 1) is used if this is None
+ """
+ raise NotImplementedError ( "revbump code" )
+ if newrev is None:
+ # get old rev and increment it
+ pass
+ else:
+ pass
+ return self
+ # --- end of revbump (...) ---
+
def __getitem__ ( self, key ):
"""Returns an item."""
return self.get ( key, do_fallback=False )
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/distroot/, roverlay/db/, roverlay/, ...
@ 2013-06-22 15:24 André Erdmann
2013-06-20 23:40 ` [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
0 siblings, 1 reply; 2+ messages in thread
From: André Erdmann @ 2013-06-22 15:24 UTC (permalink / raw
To: gentoo-commits
commit: 6a34fb49dd27014154427e95de9f1f7917cd90b9
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jun 20 23:34:54 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jun 20 23:34:54 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6a34fb49
roverlay: use distmap
Add distmap support to distroot/overlay.
revbump functionality in PackageInfo is missing and some debug output has to be
removed.
---
roverlay/db/distmap.py | 81 ++++++++++-
roverlay/overlay/creator.py | 11 +-
roverlay/overlay/pkgdir/distroot/distdir.py | 10 +-
roverlay/overlay/pkgdir/distroot/distroot.py | 155 ++++++++++++++++++---
roverlay/overlay/pkgdir/distroot/static.py | 10 +-
roverlay/overlay/pkgdir/packagedir_base.py | 48 +++++--
.../overlay/pkgdir/packagedir_ebuildmanifest.py | 2 +-
.../overlay/pkgdir/packagedir_portagemanifest.py | 4 +-
roverlay/overlay/root.py | 7 +-
roverlay/packageinfo.py | 48 ++++++-
10 files changed, 335 insertions(+), 41 deletions(-)
diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 7bfba59..dfdcd01 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -13,6 +13,7 @@ import os.path
import shutil
+import roverlay.digest
import roverlay.util
@@ -21,20 +22,33 @@ __all__ = [ 'DistMapInfo', 'get_distmap' ]
class DistMapInfo ( object ):
+ DIGEST_TYPE = 'sha256'
RESTORE_FROM_DISTFILE = '_'
+ UNSET = 'U'
def __init__ ( self, distfile, repo_name, repo_file, sha256 ):
super ( DistMapInfo, self ).__init__()
- self.repo_name = repo_name
+ self.repo_name = repo_name if repo_name is not None else self.UNSET
self.sha256 = sha256
if repo_file == self.RESTORE_FROM_DISTFILE:
self.repo_file = distfile
else:
- self.repo_file = repo_file
+ self.repo_file = repo_file if repo_file is not None else self.UNSET
# --- end of __init__ (...) ---
+ @property
+ def digest ( self ):
+ return self.sha256
+ #return getattr ( self, self.DIGEST_TYPE )
+ # --- end of digest (...) ---
+
+ def compare_digest ( self, package_info ):
+ p_hash = package_info.make_distmap_hash()
+ return ( bool ( p_hash == self.digest ), p_hash )
+ # --- end of compare_digest (...) ---
+
def __eq__ ( self, other ):
return not self.__ne__ ( other )
# --- end of __eq__ (...) ---
@@ -136,10 +150,61 @@ class _DistMapBase ( object ):
setattr ( self, attr [1], getattr ( self._distmap, attr[0] ) )
# --- end of _rebind_distmap (...) ---
+ def check_revbump_necessary ( self, package_info ):
+ """Tries to find package_info's distfile in the distmap and returns
+ whether a revbump is necessary (True) or not (False).
+
+ Compares checksums if distfile already exists.
+
+ arguments:
+ * package_info --
+ """
+ distfile = package_info.get_distmap_key()
+
+ info = self._distmap.get ( distfile, None )
+ if info is None:
+ # new file, no revbump required
+ return False
+ elif info.compare_digest ( package_info ) [0] is True:
+ # old digest == new digest, no revbump
+ # (package_info should be filtered out)
+ return False
+ else:
+ # digest mismatch => diff
+ return True
+ # --- end of compare_digest (...) ---
+
+ def get_file_digest ( self, f ):
+ return roverlay.digest.dodigest_file ( f, DistMapInfo.DIGEST_TYPE )
+ # --- end of get_file_digest (...) ---
+
+ def check_integrity ( self, distfile, distfilepath ):
+ info = self._distmap.get ( distfile, None )
+
+ if info is None:
+ # file not found
+ return 1
+ elif info.digest == self.get_file_digest ( distfilepath ):
+ # file OK
+ return 0
+ else:
+ # bad checksum
+ return 2
+ # --- end of check_integrity (...) ---
+
def remove ( self, key ):
del self._distmap [key]
+ self.dirty = True
# --- end of remove (...) ---
+ def try_remove ( self, key ):
+ try:
+ del self._distmap [key]
+ self.dirty = True
+ except KeyError:
+ pass
+ # --- end of try_remove (...) ---
+
def make_reverse_distmap ( self ):
self._reverse_distmap = {
( kv[1].repo_name, kv[1].repo_file ): kv
@@ -187,6 +252,16 @@ class _DistMapBase ( object ):
)
# --- end of add_entry_for (...) ---
+ def add_dummy_entry ( self, distfile, distfilepath ):
+ print ( "DUMMY", distfile )
+ return self.add_entry (
+ distfile,
+ DistMapInfo (
+ distfile, None, None, self.get_file_digest ( distfilepath ),
+ )
+ )
+ # --- end of add_dummy_entry (...) ---
+
def read ( self, *args, **kwargs ):
raise self.__class__.AbstractMethod()
# --- end of read (...) ---
@@ -304,6 +379,7 @@ class FileDistMap ( _FileDistMapBase ):
def write ( self, filepath=None, force=False ):
if force or self.dirty:
+ print ( "DBFILE WILL BE WRITTEN", force, dirty, list(self.keys()) )
f = filepath or self.dbfile
roverlay.util.dodir ( os.path.dirname ( f ), mkdir_p=True )
with open ( f, 'wt' ) as FH:
@@ -313,6 +389,7 @@ class FileDistMap ( _FileDistMapBase ):
self._file_written ( f )
return True
else:
+ print ( "DBFILE WILL NOT BE WRITTEN", force, dirty, list(self.keys()) )
return False
# --- end of write (...) ---
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 4c29ffa..79a70bf 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -34,6 +34,9 @@ from roverlay.packagerules.rules import PackageRules
from roverlay.recipe import easyresolver
+import roverlay.recipe.distmap
+import roverlay.overlay.pkgdir.distroot.static
+
class PseudoAtomicCounter ( object ):
@@ -97,6 +100,10 @@ class OverlayCreator ( object ):
self.rsuggests_flags = set()
+ # create distmap and distroot here
+ self.distmap = roverlay.recipe.distmap.setup()
+ self.distroot = roverlay.overlay.pkgdir.distroot.static.get_configured()
+
time_scan_overlay = time.time()
# init overlay using config values
self.overlay = Overlay.new_configured (
@@ -462,10 +469,10 @@ class OverlayCreator ( object ):
"""
if package_info ['ebuild'] is not None:
self.create_success.inc()
- if package_info.overlay_package_ref.new_ebuild():
+ if package_info.overlay_package_ref().new_ebuild():
self.overlay_added.inc()
else:
- package_info.overlay_package_ref.ebuild_uncreateable ( package_info )
+ package_info.overlay_package_ref().ebuild_uncreateable ( package_info )
self.create_fail.inc()
# --- end of _add_to_overlay (...) ---
diff --git a/roverlay/overlay/pkgdir/distroot/distdir.py b/roverlay/overlay/pkgdir/distroot/distdir.py
index 0f76e30..62cb210 100644
--- a/roverlay/overlay/pkgdir/distroot/distdir.py
+++ b/roverlay/overlay/pkgdir/distroot/distdir.py
@@ -22,11 +22,15 @@ class Distdir ( object ):
self._distroot = distroot
# --- end of __init__ (...) ---
- def add ( self, fpath, fname=None ):
- return self._distroot._add (
+ def add ( self, fpath, fname, p_info ):
+ if self._distroot._add (
fpath,
self.get_root() + os.sep + ( fname or os.path.basename ( fpath ) )
- )
+ ):
+ self._distroot.distmap_register ( p_info )
+ return True
+ else:
+ return False
# --- end of add (...) ---
def get_root ( self ):
diff --git a/roverlay/overlay/pkgdir/distroot/distroot.py b/roverlay/overlay/pkgdir/distroot/distroot.py
index 9f56742..c59d813 100644
--- a/roverlay/overlay/pkgdir/distroot/distroot.py
+++ b/roverlay/overlay/pkgdir/distroot/distroot.py
@@ -15,8 +15,11 @@ import os
import shutil
import tempfile
+
+import roverlay.db.distmap
import roverlay.overlay.pkgdir.distroot.distdir
+
class DistrootBase ( object ):
"""Base class for distroots."""
@@ -31,20 +34,24 @@ class DistrootBase ( object ):
return self.get_root()
# --- end of __str__ (...) ---
- def __init__ ( self, root, flat ):
+ def __init__ ( self, root, flat, logger=None, distmap=None ):
"""DistrootBase constructor.
arguments:
- * root -- root directory
- * flat -- whether to use a flat structure (all packages in a single
- directory, True) or per-package sub directories (False)
+ * root -- root directory
+ * flat -- whether to use a flat structure (all packages in a single
+ directory, True) or per-package sub directories (False)
+ * logger --
+ * distmap --
"""
super ( DistrootBase, self ).__init__()
- self.logger = logging.getLogger ( self.__class__.__name__ )
+ self.logger = logger or logging.getLogger ( self.__class__.__name__ )
self._root = root
# or use hasattr ( self, '_default_distdir' )
self._flat = flat
+ self.distmap = distmap
+
if flat:
self._default_distdir = (
roverlay.overlay.pkgdir.distroot.distdir.Distdir ( self )
@@ -166,29 +173,66 @@ class DistrootBase ( object ):
pass
# --- end of _prepare (...) ---
+ def iter_distfiles ( self, distfile_only ):
+ def recursive_iter ( root_abspath, root_relpath ):
+ for item in os.listdir ( root_abspath ):
+ abspath = root_abspath + os.sep + item
+ relpath = (
+ item if root_relpath is None
+ else root_relpath + os.sep + item
+ )
+ if os.path.isdir ( abspath ):
+ for result in iter_distfiles ( abspath, relpath ):
+ yield result
+ else:
+ yield ( abspath, relpath )
+ # --- end of recursive_iter (...) ---
+
+ if distfile_only:
+ for pkgfile, distfile in recursive_iter ( self.get_root(), None ):
+ yield distfile
+ else:
+ for pkgfile, distfile in recursive_iter ( self.get_root(), None ):
+ yield ( pkgfile, distfile )
+ # --- end of iter_distfiles (...) ---
+
def _remove_broken_symlinks ( self ):
"""Recursively remove broken/dead symlinks."""
- def recursive_remove ( root, rmdir ):
- for item in os.listdir ( root ):
- fpath = root + os.sep + item
+
+ def recursive_remove ( dirpath, rel_dirpath, rmdir ):
+ for item in os.listdir ( dirpath ):
+ fpath = dirpath + os.sep + item
+ relpath = (
+ item if rel_dirpath is None else rel_dirpath + os.sep + item
+ )
if not os.path.exists ( fpath ):
+ # drop broken symlink
+ self.logger.debug (
+ "Removing broken symlink {!r}".format ( fpath )
+ )
os.unlink ( fpath )
-
+ if self.distmap is not None:
+ self.distmap.try_remove ( relpath )
elif os.path.isdir ( fpath ):
- recursive_remove ( fpath, True )
+ recursive_remove ( fpath, relpath, True )
if rmdir:
try:
os.rmdir ( fpath )
except OSError:
pass
+ # -- end for
# --- end of recursive_remove (...) ---
- recursive_remove ( self.get_root(), False )
+
+ return recursive_remove ( self.get_root(), None, False )
# --- end of _remove_broken_symlinks (...) ---
def _try_remove ( self, dest ):
try:
os.unlink ( dest )
+ if self.distmap is not None:
+ relpath = os.path.relpath ( dest, self.get_root() )
+
except OSError as e:
if e.errno == errno.ENOENT:
pass
@@ -219,16 +263,72 @@ class DistrootBase ( object ):
return str ( self._root )
# --- end of get_root (...) ---
+ def distmap_register ( self, p_info ):
+ return self.distmap.add_entry_for ( p_info )
+ # --- end of distmap_register (...) ---
+
+ def check_integrity ( self ):
+ if self.distmap is not None:
+ root = self.get_root()
+ distfiles = set()
+ checkfile = self.distmap.check_integrity
+
+ for abspath, relpath in self.iter_distfiles ( False ):
+ status = checkfile ( relpath, abspath )
+
+ if status == 0:
+ self.logger.debug (
+ "file has been verified: {!r}".format ( relpath )
+ )
+ distfiles.add ( relpath )
+ elif status == 1:
+ # file not in distmap
+ self.logger.info (
+ "file not in distmap, creating dummy entry: {!r}".format ( relpath )
+ )
+ self.distmap.add_dummy_entry ( relpath, abspath )
+ distfiles.add ( relpath )
+ elif status == 2:
+ # file in distmap, but not valid - remove it from distmap
+ self.logger.warning (
+ "digest mismatch: {!r}".format ( relpath )
+ )
+ self.distmap.remove ( relpath )
+ # -- end for
+
+ distmap_keys = frozenset ( self.distmap.keys() )
+
+ if distfiles:
+ # reverse compare
+ for distfile in distmap_keys:
+ if distfile not in distfiles:
+ self.logger.warning (
+ "distmap file does not exist: {!r}".format ( distfile )
+ )
+ self.distmap.remove ( distfile )
+
+ else:
+ # no files from distroot in distmap -- invalidate distmap
+ for distfile in distmap_keys:
+ self.logger.warning (
+ "distmap file does not exist: {!r}".format ( distfile )
+ )
+ self.distmap.remove ( distfile )
+ else:
+ raise Exception ( "check_integrity() needs a distmap." )
+ # --- end of check_integrity (...) ---
+
# --- end of DistrootBase ---
class TemporaryDistroot ( DistrootBase ):
- def __init__ ( self ):
+ def __init__ ( self, logger=None ):
# temporary distroots always use the non-flat distdir layout
super ( TemporaryDistroot, self ).__init__ (
- root = tempfile.mkdtemp ( prefix='tmp_roverlay_distroot_' ),
- flat = False
+ root = tempfile.mkdtemp ( prefix='tmp_roverlay_distroot_' ),
+ flat = False,
+ logger = logger,
)
# --- end of __init__ (...) ---
@@ -238,6 +338,7 @@ class TemporaryDistroot ( DistrootBase ):
def _cleanup ( self ):
"""Cleans up the temporary distroot by simply wiping it."""
+ super ( TemporaryDistroot, self )._cleanup()
shutil.rmtree ( self._root )
# --- end of _cleanup (...) ---
@@ -265,7 +366,9 @@ class PersistentDistroot ( DistrootBase ):
)
# --- end of __repr__ (...) ---
- def __init__ ( self, root, flat, strategy ):
+ def __init__ ( self,
+ root, flat, strategy, distmap, verify=False, logger=None
+ ):
"""Initializes a non-temporary distroot.
arguments:
@@ -274,8 +377,13 @@ class PersistentDistroot ( DistrootBase ):
* strategy -- the distroot 'strategy' that determines what mode (sym-
link, hardlink, copy) will be tried in which order
This has to be an iterable with valid items.
+ * distmap --
+ * verify --
+ * logger --
"""
- super ( PersistentDistroot, self ).__init__ ( root=root, flat=flat )
+ super ( PersistentDistroot, self ).__init__ (
+ root=root, flat=flat, logger=logger, distmap=distmap
+ )
self._strategy = self._get_int_strategy ( strategy )
@@ -294,6 +402,15 @@ class PersistentDistroot ( DistrootBase ):
self.USE_HARDLINK : self._add_hardlink,
self.USE_COPY : self._add_file,
}
+
+
+ if verify and self.distmap is not None:
+ # FIXME debug print
+ print (
+ "Checking distroot file integrity, this may take some time ... "
+ )
+ self.check_integrity()
+ print ( "Done!" )
# --- end of __init__ (...) ---
def _add ( self, src, dest ):
@@ -328,10 +445,14 @@ class PersistentDistroot ( DistrootBase ):
# --- end of _add (...) ---
def _cleanup ( self ):
+ super ( PersistentDistroot, self )._cleanup()
if hasattr ( self, '_supported_modes_initial' ):
if self._supported_modes_initial & self.USE_SYMLINK:
self._remove_broken_symlinks()
- # --- end of _prepare (...) ---
+
+ if self.distmap is not None:
+ self.distmap.write ( force=False )
+ # --- end of _cleanup (...) ---
def _get_int_strategy ( self, strategy ):
"""Converts the given strategy into its integer tuple representation.
diff --git a/roverlay/overlay/pkgdir/distroot/static.py b/roverlay/overlay/pkgdir/distroot/static.py
index b4ad055..0054140 100644
--- a/roverlay/overlay/pkgdir/distroot/static.py
+++ b/roverlay/overlay/pkgdir/distroot/static.py
@@ -8,6 +8,8 @@ __all__ = [ 'get_configured', 'get_distdir', ]
import threading
+import roverlay.recipe.distmap
+
import roverlay.config
import roverlay.overlay.pkgdir.distroot.distroot
@@ -34,7 +36,11 @@ def get_configured ( static=True ):
# generally, the "flat" distroot/distdir layout is desired as it
# can serve as package mirror directory, so default to True here
flat = roverlay.config.get ( 'OVERLAY.DISTDIR.flat', True ),
- strategy = distdir_strategy
+ strategy = distdir_strategy,
+ distmap = roverlay.recipe.distmap.access(),
+ # FIXME/TODO: 'verify' config key does not exist
+ verify = roverlay.config.get ( 'OVERLAY.DISTDIR.verify', False ),
+
)
# --- end of get_new (...) ---
@@ -49,7 +55,7 @@ def get_configured ( static=True ):
return _distroot_instance
else:
- return get_new()
+ raise Exception ( "static keyword arg is deprecated" )
# --- end of get_configured (...) ---
def get_distdir ( ebuild_name ):
diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index e67114e..a68ecbd 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -20,12 +20,14 @@ import os
import shutil
import sys
import threading
-
+import weakref
import roverlay.config
import roverlay.packageinfo
import roverlay.util
+import roverlay.recipe.distmap
+
import roverlay.tools.ebuild
import roverlay.tools.ebuildenv
import roverlay.tools.patch
@@ -40,8 +42,10 @@ class PackageDirBase ( object ):
for Manifest file creation."""
#DISTROOT =
+ #DISTMAP =
EBUILD_SUFFIX = '.ebuild'
#FETCHENV =
+ #HASHES =
SUPPRESS_EXCEPTIONS = True
# MANIFEST_THREADSAFE (tri-state)
@@ -59,9 +63,11 @@ class PackageDirBase ( object ):
roverlay.config.get_or_fail ( 'OVERLAY.dir' )
)
- cls.DISTROOT = (
- roverlay.overlay.pkgdir.distroot.static.get_configured ( static=True )
- )
+ cls.DISTROOT = roverlay.overlay.pkgdir.distroot.static.get_configured()
+ cls.DISTMAP = roverlay.recipe.distmap.access()
+ # FIXME/TODO: config
+ #cls.HASHES = frozenset ({ 'sha256', 'sha512', 'whirlpool', })
+ cls.HASHES = frozenset ({ 'sha256', })
cls.FETCHENV = fetch_env
# --- end of init_cls (...) ---
@@ -142,11 +148,12 @@ class PackageDirBase ( object ):
p = roverlay.packageinfo.PackageInfo (
physical_only=True, pvr=pvr, ebuild_file=efile
)
+ # TODO/FIXME: parse SRC_URI, knowledge of distfile path would be good...
self._packages [ p ['ebuild_verstr'] ] = p
return p
# --- end of _scan_add_package (...) ---
- def add ( self, package_info, add_if_physical=False ):
+ def add ( self, package_info, add_if_physical=False, _lock=True ):
"""Adds a package to this PackageDir.
arguments:
@@ -161,7 +168,8 @@ class PackageDirBase ( object ):
shortver = package_info ['ebuild_verstr']
added = False
try:
- self._lock.acquire()
+ if _lock: self._lock.acquire()
+
if shortver in self._packages:
# package exists, check if it existed before script invocation
if self._packages [shortver] ['physical_only']:
@@ -170,6 +178,12 @@ class PackageDirBase ( object ):
self._packages [shortver] = package_info
added = True
+ elif self.DISTMAP.check_revbump_necessary ( package_info ):
+ # resolve by recursion
+ added = self.add (
+ package_info.revbump(), add_if_physical=False, _lock=False
+ )
+
else:
self.logger.debug (
"'{PN}-{PVR}.ebuild' exists as file, skipping.".format (
@@ -190,12 +204,12 @@ class PackageDirBase ( object ):
added = True
finally:
- self._lock.release()
+ if _lock: self._lock.release()
if added:
# add a link to this PackageDir into the package info,
# !! package_info <-> self (double-linked)
- package_info.overlay_package_ref = self
+ package_info.overlay_package_ref = weakref.ref ( self )
return True
else:
return False
@@ -786,6 +800,10 @@ class PackageDirBase ( object ):
or patch_ebuild ( efile, pvr, patchview.get_patches ( pvr ) )
):
+ # generate hashes here (benefit from threading)
+ # FIXME/TODO: ^ actually faster?
+ p_info.make_hashes ( self.HASHES )
+
self._need_manifest = True
# update metadata for each successfully written ebuild
@@ -848,6 +866,20 @@ class PackageDirBase ( object ):
if p.has ( 'package_file', 'ebuild_file' )
]
+ print ( "--- DEBUG PRINT from packagedir_base :: {} :: {} ---".format (
+ self.__class__.__name__, self.name
+ )
+ )
+ print ( "Calculating hashes ..." )
+ for p in pkgs_for_manifest:
+ p.make_hashes ( self.HASHES )
+ print ( p.hashdict )
+ print ( "Done!" )
+ print ( "--- END DEBUG PRINT from packagedir_base :: {} :: {} ---".format (
+ self.__class__.__name__, self.name
+ )
+ )
+
if pkgs_for_manifest:
self.logger.debug ( "Writing Manifest" )
if self._write_manifest ( pkgs_for_manifest ):
diff --git a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
index 5217f6a..9766606 100644
--- a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
@@ -102,7 +102,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
# TODO: optimize this further?
# -> "not has physical_only?"
# (should be covered by "has package_file")
- distdir.add ( p ['package_file'], p ['package_src_destpath'] )
+ distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
# -- end for;
return self._do_ebuildmanifest ( ebuild_file, distdir )
diff --git a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
index 6596dec..f4527a1 100644
--- a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
@@ -151,7 +151,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
manifest.addFile ( 'EBUILD', ebuild_filename )
if not manifest.hasFile ( 'DIST', package_filename ):
- distdir.add ( p ['package_file'], package_filename )
+ distdir.add ( p ['package_file'], package_filename, p )
manifest.addFile (
'DIST',
package_filename,
@@ -165,7 +165,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
# doing the check above (if hasFile())
# be "fully compatible" and do that here, too
try:
- distdir.add ( p ['package_file'], package_filename )
+ distdir.add ( p ['package_file'], package_filename, p )
except Exception as err:
self.logger.exception ( err )
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index c2522b7..a3b02f1 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -23,14 +23,15 @@ import os
import shutil
import threading
+
import roverlay.config
import roverlay.util
+import roverlay.recipe.distmap
import roverlay.overlay.additionsdir
import roverlay.overlay.category
import roverlay.overlay.header
-
class Overlay ( object ):
DEFAULT_USE_DESC = (
'byte-compile - enable byte compiling\n'
@@ -730,6 +731,10 @@ class Overlay ( object ):
write_manifest = not self.skip_manifest,
additions_dir = self.additions_dir.get_obj_subdir ( cat ),
)
+
+ distmap = roverlay.recipe.distmap.access()
+ distmap.backup_file ( ignore_missing=True )
+ distmap.write()
else:
# FIXME debug print
print (
diff --git a/roverlay/packageinfo.py b/roverlay/packageinfo.py
index e8004d0..83a05ed 100644
--- a/roverlay/packageinfo.py
+++ b/roverlay/packageinfo.py
@@ -19,6 +19,7 @@ import logging
import threading
import roverlay.digest
+import roverlay.db.distmap
from roverlay import config, strutil
from roverlay.rpackage import descriptionreader
@@ -116,6 +117,9 @@ class PackageInfo ( object ):
'ebuild'
))
+ # bind DIGEST_TYPE to this class
+ DISTMAP_DIGEST_TYPE = roverlay.db.distmap.DistMapInfo.DIGEST_TYPE
+
def __init__ ( self, **initial_info ):
"""Initializes a PackageInfo.
@@ -451,25 +455,45 @@ class PackageInfo ( object ):
# --- end of get_desc_data (...) ---
def get_distmap_item ( self ):
+ """Returns a 2-tuple ( key, info ) for the distmap."""
return ( self.get_distmap_key(), self.get_distmap_value() )
# --- end of get_distmap_item (...) ---
def get_distmap_key ( self ):
+ """Returns a key for the distmap."""
return self.get ( "package_src_destpath" )
# --- end of get_distmap_key (...) ---
- def get_distmap_value ( self ):
- assert 'sha256' in self.hashdict
+ def get_distmap_value ( self, allow_digest_create=False ):
+ """Returns a data tuple for creating DistMapInfo instances.
+ arguments:
+ * allow_digest_create --
+ """
repo = self.get ( "origin" )
return (
repo.name,
os.path.relpath ( self.get ( "package_file" ), repo.distdir ),
- self.hashdict ['sha256']
+ (
+ self.make_distmap_hash() if allow_digest_create
+ else self.hashdict [self.DISTMAP_DIGEST_TYPE]
+ )
)
# --- end of get_distmap_value (...) ---
+ def make_distmap_hash ( self ):
+ """Creates (and returns) the distmap package file hash."""
+ return self.make_hashes ( { self.DISTMAP_DIGEST_TYPE, } ) [self.DISTMAP_DIGEST_TYPE]
+ # --- end of make_distmap_hash (...) ---
+
def make_hashes ( self, hashlist ):
+ """Creates zero or more hashes and returns the hashdict.
+
+ Note: the hashdict can also be accessed directly via <this>.hashdict.
+
+ arguments:
+ * hashlist -- list of hash names, e.g. "sha256", "md5"
+ """
pkgfile = self.get ( "package_file" )
if hasattr ( self, 'hashdict' ) and self.hashdict:
@@ -483,8 +507,26 @@ class PackageInfo ( object ):
)
else:
self.hashdict = roverlay.digest.multihash_file ( pkgfile, hashlist )
+
+ return self.hashdict
# --- end of make_hashes (...) ---
+ def revbump ( self, newrev=None ):
+ """Do whatever necessary to revbump this pakages, that is set/update
+ all data like src_uri_destfile.
+
+ arguments:
+ * newrev -- new revision, (current rev + 1) is used if this is None
+ """
+ raise NotImplementedError ( "revbump code" )
+ if newrev is None:
+ # get old rev and increment it
+ pass
+ else:
+ pass
+ return self
+ # --- end of revbump (...) ---
+
def __getitem__ ( self, key ):
"""Returns an item."""
return self.get ( key, do_fallback=False )
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2013-06-22 15:24 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-06-22 15:24 [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/distroot/, roverlay/db/, roverlay/, André Erdmann
2013-06-20 23:40 ` [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox