public inbox for gentoo-commits@lists.gentoo.org
 help / color / mirror / Atom feed
* [gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/distroot/, roverlay/db/, roverlay/, ...
  2013-06-22 15:24 [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/distroot/, roverlay/db/, roverlay/, André Erdmann
@ 2013-06-20 23:40 ` André Erdmann
  0 siblings, 0 replies; 2+ messages in thread
From: André Erdmann @ 2013-06-20 23:40 UTC (permalink / raw
  To: gentoo-commits

commit:     6a34fb49dd27014154427e95de9f1f7917cd90b9
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jun 20 23:34:54 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jun 20 23:34:54 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6a34fb49

roverlay: use distmap

Add distmap support to distroot/overlay.

revbump functionality in PackageInfo is missing and some debug output has to be
removed.

---
 roverlay/db/distmap.py                             |  81 ++++++++++-
 roverlay/overlay/creator.py                        |  11 +-
 roverlay/overlay/pkgdir/distroot/distdir.py        |  10 +-
 roverlay/overlay/pkgdir/distroot/distroot.py       | 155 ++++++++++++++++++---
 roverlay/overlay/pkgdir/distroot/static.py         |  10 +-
 roverlay/overlay/pkgdir/packagedir_base.py         |  48 +++++--
 .../overlay/pkgdir/packagedir_ebuildmanifest.py    |   2 +-
 .../overlay/pkgdir/packagedir_portagemanifest.py   |   4 +-
 roverlay/overlay/root.py                           |   7 +-
 roverlay/packageinfo.py                            |  48 ++++++-
 10 files changed, 335 insertions(+), 41 deletions(-)

diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 7bfba59..dfdcd01 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -13,6 +13,7 @@ import os.path
 import shutil
 
 
+import roverlay.digest
 import roverlay.util
 
 
@@ -21,20 +22,33 @@ __all__ = [ 'DistMapInfo', 'get_distmap' ]
 
 class DistMapInfo ( object ):
 
+   DIGEST_TYPE           = 'sha256'
    RESTORE_FROM_DISTFILE = '_'
+   UNSET                 = 'U'
 
    def __init__ ( self, distfile, repo_name, repo_file, sha256 ):
       super ( DistMapInfo, self ).__init__()
 
-      self.repo_name = repo_name
+      self.repo_name = repo_name if repo_name is not None else self.UNSET
       self.sha256    = sha256
 
       if repo_file == self.RESTORE_FROM_DISTFILE:
          self.repo_file = distfile
       else:
-         self.repo_file = repo_file
+         self.repo_file = repo_file if repo_file is not None else self.UNSET
    # --- end of __init__ (...) ---
 
+   @property
+   def digest ( self ):
+      return self.sha256
+      #return getattr ( self, self.DIGEST_TYPE )
+   # --- end of digest (...) ---
+
+   def compare_digest ( self, package_info ):
+      p_hash = package_info.make_distmap_hash()
+      return ( bool ( p_hash == self.digest ), p_hash )
+   # --- end of compare_digest (...) ---
+
    def __eq__ ( self, other ):
       return not self.__ne__ ( other )
    # --- end of __eq__ (...) ---
@@ -136,10 +150,61 @@ class _DistMapBase ( object ):
             setattr ( self, attr [1], getattr ( self._distmap, attr[0] ) )
    # --- end of _rebind_distmap (...) ---
 
+   def check_revbump_necessary ( self, package_info ):
+      """Tries to find package_info's distfile in the distmap and returns
+      whether a revbump is necessary (True) or not (False).
+
+      Compares checksums if distfile already exists.
+
+      arguments:
+      * package_info --
+      """
+      distfile = package_info.get_distmap_key()
+
+      info = self._distmap.get ( distfile, None )
+      if info is None:
+         # new file, no revbump required
+         return False
+      elif info.compare_digest ( package_info ) [0] is True:
+         # old digest == new digest, no revbump
+         #  (package_info should be filtered out)
+         return False
+      else:
+         # digest mismatch => diff
+         return True
+   # --- end of compare_digest (...) ---
+
+   def get_file_digest ( self, f ):
+      return roverlay.digest.dodigest_file ( f, DistMapInfo.DIGEST_TYPE )
+   # --- end of get_file_digest (...) ---
+
+   def check_integrity ( self, distfile, distfilepath ):
+      info = self._distmap.get ( distfile, None )
+
+      if info is None:
+         # file not found
+         return 1
+      elif info.digest == self.get_file_digest ( distfilepath ):
+         # file OK
+         return 0
+      else:
+         # bad checksum
+         return 2
+   # --- end of check_integrity (...) ---
+
    def remove ( self, key ):
       del self._distmap [key]
+      self.dirty = True
    # --- end of remove (...) ---
 
+   def try_remove ( self, key ):
+      try:
+         del self._distmap [key]
+         self.dirty = True
+      except KeyError:
+         pass
+   # --- end of try_remove (...) ---
+
    def make_reverse_distmap ( self ):
       self._reverse_distmap = {
          ( kv[1].repo_name, kv[1].repo_file ): kv
@@ -187,6 +252,16 @@ class _DistMapBase ( object ):
       )
    # --- end of add_entry_for (...) ---
 
+   def add_dummy_entry ( self, distfile, distfilepath ):
+      print ( "DUMMY", distfile )
+      return self.add_entry (
+         distfile,
+         DistMapInfo (
+            distfile, None, None, self.get_file_digest ( distfilepath ),
+         )
+      )
+   # --- end of add_dummy_entry (...) ---
+
    def read ( self, *args, **kwargs ):
       raise self.__class__.AbstractMethod()
    # --- end of read (...) ---
@@ -304,6 +379,7 @@ class FileDistMap ( _FileDistMapBase ):
 
    def write ( self, filepath=None, force=False ):
       if force or self.dirty:
+         print ( "DBFILE WILL BE WRITTEN", force, dirty, list(self.keys()) )
          f  = filepath or self.dbfile
          roverlay.util.dodir ( os.path.dirname ( f ), mkdir_p=True )
          with open ( f, 'wt' ) as FH:
@@ -313,6 +389,7 @@ class FileDistMap ( _FileDistMapBase ):
             self._file_written ( f )
          return True
       else:
+         print ( "DBFILE WILL NOT BE WRITTEN", force, dirty, list(self.keys()) )
          return False
    # --- end of write (...) ---
 

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 4c29ffa..79a70bf 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -34,6 +34,9 @@ from roverlay.packagerules.rules import PackageRules
 
 from roverlay.recipe             import easyresolver
 
+import roverlay.recipe.distmap
+import roverlay.overlay.pkgdir.distroot.static
+
 
 class PseudoAtomicCounter ( object ):
 
@@ -97,6 +100,10 @@ class OverlayCreator ( object ):
 
       self.rsuggests_flags = set()
 
+      # create distmap and distroot here
+      self.distmap  = roverlay.recipe.distmap.setup()
+      self.distroot = roverlay.overlay.pkgdir.distroot.static.get_configured()
+
       time_scan_overlay = time.time()
       # init overlay using config values
       self.overlay = Overlay.new_configured (
@@ -462,10 +469,10 @@ class OverlayCreator ( object ):
       """
       if package_info ['ebuild'] is not None:
          self.create_success.inc()
-         if package_info.overlay_package_ref.new_ebuild():
+         if package_info.overlay_package_ref().new_ebuild():
             self.overlay_added.inc()
       else:
-         package_info.overlay_package_ref.ebuild_uncreateable ( package_info )
+         package_info.overlay_package_ref().ebuild_uncreateable ( package_info )
          self.create_fail.inc()
 
    # --- end of _add_to_overlay (...) ---

diff --git a/roverlay/overlay/pkgdir/distroot/distdir.py b/roverlay/overlay/pkgdir/distroot/distdir.py
index 0f76e30..62cb210 100644
--- a/roverlay/overlay/pkgdir/distroot/distdir.py
+++ b/roverlay/overlay/pkgdir/distroot/distdir.py
@@ -22,11 +22,15 @@ class Distdir ( object ):
       self._distroot = distroot
    # --- end of __init__ (...) ---
 
-   def add ( self, fpath, fname=None ):
-      return self._distroot._add (
+   def add ( self, fpath, fname, p_info ):
+      if self._distroot._add (
          fpath,
          self.get_root() + os.sep + ( fname or os.path.basename ( fpath ) )
-      )
+      ):
+         self._distroot.distmap_register ( p_info )
+         return True
+      else:
+         return False
    # --- end of add (...) ---
 
    def get_root ( self ):

diff --git a/roverlay/overlay/pkgdir/distroot/distroot.py b/roverlay/overlay/pkgdir/distroot/distroot.py
index 9f56742..c59d813 100644
--- a/roverlay/overlay/pkgdir/distroot/distroot.py
+++ b/roverlay/overlay/pkgdir/distroot/distroot.py
@@ -15,8 +15,11 @@ import os
 import shutil
 import tempfile
 
+
+import roverlay.db.distmap
 import roverlay.overlay.pkgdir.distroot.distdir
 
+
 class DistrootBase ( object ):
    """Base class for distroots."""
 
@@ -31,20 +34,24 @@ class DistrootBase ( object ):
       return self.get_root()
    # --- end of __str__ (...) ---
 
-   def __init__ ( self, root, flat ):
+   def __init__ ( self, root, flat, logger=None, distmap=None ):
       """DistrootBase constructor.
 
       arguments:
-      * root -- root directory
-      * flat -- whether to use a flat structure (all packages in a single
-                 directory, True) or per-package sub directories (False)
+      * root    -- root directory
+      * flat    -- whether to use a flat structure (all packages in a single
+                    directory, True) or per-package sub directories (False)
+      * logger  --
+      * distmap --
       """
       super ( DistrootBase, self ).__init__()
-      self.logger = logging.getLogger ( self.__class__.__name__ )
+      self.logger = logger or logging.getLogger ( self.__class__.__name__ )
       self._root  = root
       # or use hasattr ( self, '_default_distdir' )
       self._flat  = flat
 
+      self.distmap = distmap
+
       if flat:
          self._default_distdir = (
             roverlay.overlay.pkgdir.distroot.distdir.Distdir ( self )
@@ -166,29 +173,66 @@ class DistrootBase ( object ):
       pass
    # --- end of _prepare (...) ---
 
+   def iter_distfiles ( self, distfile_only ):
+      def recursive_iter ( root_abspath, root_relpath ):
+         for item in os.listdir ( root_abspath ):
+            abspath = root_abspath + os.sep + item
+            relpath = (
+               item if root_relpath is None
+               else root_relpath + os.sep + item
+            )
+            if os.path.isdir ( abspath ):
+               for result in iter_distfiles ( abspath, relpath ):
+                  yield result
+            else:
+               yield ( abspath, relpath )
+      # --- end of recursive_iter (...) ---
+
+      if distfile_only:
+         for pkgfile, distfile in recursive_iter ( self.get_root(), None ):
+            yield distfile
+      else:
+         for pkgfile, distfile in recursive_iter ( self.get_root(), None ):
+            yield ( pkgfile, distfile )
+   # --- end of iter_distfiles (...) ---
+
    def _remove_broken_symlinks ( self ):
       """Recursively remove broken/dead symlinks."""
-      def recursive_remove ( root, rmdir ):
-         for item in os.listdir ( root ):
-            fpath = root + os.sep + item
+
+      def recursive_remove ( dirpath, rel_dirpath, rmdir ):
+         for item in os.listdir ( dirpath ):
+            fpath   = dirpath + os.sep + item
+            relpath = (
+               item if rel_dirpath is None else rel_dirpath + os.sep + item
+            )
 
             if not os.path.exists ( fpath ):
+               # drop broken symlink
+               self.logger.debug (
+                  "Removing broken symlink {!r}".format ( fpath )
+               )
                os.unlink ( fpath )
-
+               if self.distmap is not None:
+                  self.distmap.try_remove ( relpath )
             elif os.path.isdir ( fpath ):
-               recursive_remove ( fpath, True )
+               recursive_remove ( fpath, relpath, True )
                if rmdir:
                   try:
                      os.rmdir ( fpath )
                   except OSError:
                      pass
+         # -- end for
       # --- end of recursive_remove (...) ---
-      recursive_remove ( self.get_root(), False )
+
+      return recursive_remove ( self.get_root(), None, False )
    # --- end of _remove_broken_symlinks (...) ---
 
    def _try_remove ( self, dest ):
       try:
          os.unlink ( dest )
+         if self.distmap is not None:
+            relpath = os.path.relpath ( dest, self.get_root() )
+
       except OSError as e:
          if e.errno == errno.ENOENT:
             pass
@@ -219,16 +263,72 @@ class DistrootBase ( object ):
       return str ( self._root )
    # --- end of get_root (...) ---
 
+   def distmap_register ( self, p_info ):
+      return self.distmap.add_entry_for ( p_info )
+   # --- end of distmap_register (...) ---
+
+   def check_integrity ( self ):
+      if self.distmap is not None:
+         root      = self.get_root()
+         distfiles = set()
+         checkfile = self.distmap.check_integrity
+
+         for abspath, relpath in self.iter_distfiles ( False ):
+            status = checkfile ( relpath, abspath )
+
+            if status == 0:
+               self.logger.debug (
+                  "file has been verified: {!r}".format ( relpath )
+               )
+               distfiles.add ( relpath )
+            elif status == 1:
+               # file not in distmap
+               self.logger.info (
+                  "file not in distmap, creating dummy entry: {!r}".format ( relpath )
+               )
+               self.distmap.add_dummy_entry ( relpath, abspath )
+               distfiles.add ( relpath )
+            elif status == 2:
+               # file in distmap, but not valid - remove it from distmap
+               self.logger.warning (
+                  "digest mismatch: {!r}".format ( relpath )
+               )
+               self.distmap.remove ( relpath )
+         # -- end for
+
+         distmap_keys = frozenset ( self.distmap.keys() )
+
+         if distfiles:
+            # reverse compare
+            for distfile in distmap_keys:
+               if distfile not in distfiles:
+                  self.logger.warning (
+                     "distmap file does not exist: {!r}".format ( distfile )
+                  )
+                  self.distmap.remove ( distfile )
+
+         else:
+            # no files from distroot in distmap -- invalidate distmap
+            for distfile in distmap_keys:
+               self.logger.warning (
+                  "distmap file does not exist: {!r}".format ( distfile )
+               )
+               self.distmap.remove ( distfile )
+      else:
+         raise Exception ( "check_integrity() needs a distmap." )
+   # --- end of check_integrity (...) ---
+
 # --- end of DistrootBase ---
 
 
 class TemporaryDistroot ( DistrootBase ):
 
-   def __init__ ( self ):
+   def __init__ ( self, logger=None ):
       # temporary distroots always use the non-flat distdir layout
       super ( TemporaryDistroot, self ).__init__ (
-         root = tempfile.mkdtemp ( prefix='tmp_roverlay_distroot_' ),
-         flat = False
+         root   = tempfile.mkdtemp ( prefix='tmp_roverlay_distroot_' ),
+         flat   = False,
+         logger = logger,
       )
    # --- end of __init__ (...) ---
 
@@ -238,6 +338,7 @@ class TemporaryDistroot ( DistrootBase ):
 
    def _cleanup ( self ):
       """Cleans up the temporary distroot by simply wiping it."""
+      super ( TemporaryDistroot, self )._cleanup()
       shutil.rmtree ( self._root )
    # --- end of _cleanup (...) ---
 
@@ -265,7 +366,9 @@ class PersistentDistroot ( DistrootBase ):
       )
    # --- end of __repr__ (...) ---
 
-   def __init__ ( self, root, flat, strategy ):
+   def __init__ ( self,
+      root, flat, strategy, distmap, verify=False, logger=None
+   ):
       """Initializes a non-temporary distroot.
 
       arguments:
@@ -274,8 +377,13 @@ class PersistentDistroot ( DistrootBase ):
       * strategy -- the distroot 'strategy' that determines what mode (sym-
                     link, hardlink, copy) will be tried in which order
                     This has to be an iterable with valid items.
+      * distmap  --
+      * verify   --
+      * logger   --
       """
-      super ( PersistentDistroot, self ).__init__ ( root=root, flat=flat )
+      super ( PersistentDistroot, self ).__init__ (
+         root=root, flat=flat, logger=logger, distmap=distmap
+      )
 
       self._strategy = self._get_int_strategy ( strategy )
 
@@ -294,6 +402,15 @@ class PersistentDistroot ( DistrootBase ):
          self.USE_HARDLINK : self._add_hardlink,
          self.USE_COPY     : self._add_file,
       }
+
+
+      if verify and self.distmap is not None:
+         # FIXME debug print
+         print (
+            "Checking distroot file integrity, this may take some time ... "
+         )
+         self.check_integrity()
+         print ( "Done!" )
    # --- end of __init__ (...) ---
 
    def _add ( self, src, dest ):
@@ -328,10 +445,14 @@ class PersistentDistroot ( DistrootBase ):
    # --- end of _add (...) ---
 
    def _cleanup ( self ):
+      super ( PersistentDistroot, self )._cleanup()
       if hasattr ( self, '_supported_modes_initial' ):
          if self._supported_modes_initial & self.USE_SYMLINK:
             self._remove_broken_symlinks()
-   # --- end of _prepare (...) ---
+
+      if self.distmap is not None:
+         self.distmap.write ( force=False )
+   # --- end of _cleanup (...) ---
 
    def _get_int_strategy ( self, strategy ):
       """Converts the given strategy into its integer tuple representation.

diff --git a/roverlay/overlay/pkgdir/distroot/static.py b/roverlay/overlay/pkgdir/distroot/static.py
index b4ad055..0054140 100644
--- a/roverlay/overlay/pkgdir/distroot/static.py
+++ b/roverlay/overlay/pkgdir/distroot/static.py
@@ -8,6 +8,8 @@ __all__ = [ 'get_configured', 'get_distdir', ]
 
 import threading
 
+import roverlay.recipe.distmap
+
 import roverlay.config
 
 import roverlay.overlay.pkgdir.distroot.distroot
@@ -34,7 +36,11 @@ def get_configured ( static=True ):
             # generally, the "flat" distroot/distdir layout is desired as it
             # can serve as package mirror directory, so default to True here
             flat     = roverlay.config.get ( 'OVERLAY.DISTDIR.flat', True ),
-            strategy = distdir_strategy
+            strategy = distdir_strategy,
+            distmap  = roverlay.recipe.distmap.access(),
+            # FIXME/TODO: 'verify' config key does not exist
+            verify   = roverlay.config.get ( 'OVERLAY.DISTDIR.verify', False ),
+
          )
    # --- end of get_new (...) ---
 
@@ -49,7 +55,7 @@ def get_configured ( static=True ):
 
       return _distroot_instance
    else:
-      return get_new()
+      raise Exception ( "static keyword arg is deprecated" )
 # --- end of get_configured (...) ---
 
 def get_distdir ( ebuild_name ):

diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index e67114e..a68ecbd 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -20,12 +20,14 @@ import os
 import shutil
 import sys
 import threading
-
+import weakref
 
 import roverlay.config
 import roverlay.packageinfo
 import roverlay.util
 
+import roverlay.recipe.distmap
+
 import roverlay.tools.ebuild
 import roverlay.tools.ebuildenv
 import roverlay.tools.patch
@@ -40,8 +42,10 @@ class PackageDirBase ( object ):
    for Manifest file creation."""
 
    #DISTROOT =
+   #DISTMAP  =
    EBUILD_SUFFIX       = '.ebuild'
    #FETCHENV =
+   #HASHES   =
    SUPPRESS_EXCEPTIONS = True
 
    # MANIFEST_THREADSAFE (tri-state)
@@ -59,9 +63,11 @@ class PackageDirBase ( object ):
          roverlay.config.get_or_fail ( 'OVERLAY.dir' )
       )
 
-      cls.DISTROOT = (
-         roverlay.overlay.pkgdir.distroot.static.get_configured ( static=True )
-      )
+      cls.DISTROOT = roverlay.overlay.pkgdir.distroot.static.get_configured()
+      cls.DISTMAP  = roverlay.recipe.distmap.access()
+      # FIXME/TODO: config
+      #cls.HASHES = frozenset ({ 'sha256', 'sha512', 'whirlpool', })
+      cls.HASHES = frozenset ({ 'sha256', })
       cls.FETCHENV = fetch_env
    # --- end of init_cls (...) ---
 
@@ -142,11 +148,12 @@ class PackageDirBase ( object ):
       p = roverlay.packageinfo.PackageInfo (
          physical_only=True, pvr=pvr, ebuild_file=efile
       )
+      # TODO/FIXME: parse SRC_URI, knowledge of distfile path would be good...
       self._packages [ p ['ebuild_verstr'] ] = p
       return p
    # --- end of _scan_add_package (...) ---
 
-   def add ( self, package_info, add_if_physical=False ):
+   def add ( self, package_info, add_if_physical=False, _lock=True ):
       """Adds a package to this PackageDir.
 
       arguments:
@@ -161,7 +168,8 @@ class PackageDirBase ( object ):
       shortver = package_info ['ebuild_verstr']
       added = False
       try:
-         self._lock.acquire()
+         if _lock: self._lock.acquire()
+
          if shortver in self._packages:
             # package exists, check if it existed before script invocation
             if self._packages [shortver] ['physical_only']:
@@ -170,6 +178,12 @@ class PackageDirBase ( object ):
                   self._packages [shortver] = package_info
                   added = True
 
+               elif self.DISTMAP.check_revbump_necessary ( package_info ):
+                  # resolve by recursion
+                  added = self.add (
+                     package_info.revbump(), add_if_physical=False, _lock=False
+                  )
+
                else:
                   self.logger.debug (
                      "'{PN}-{PVR}.ebuild' exists as file, skipping.".format (
@@ -190,12 +204,12 @@ class PackageDirBase ( object ):
             added = True
 
       finally:
-         self._lock.release()
+         if _lock: self._lock.release()
 
       if added:
          # add a link to this PackageDir into the package info,
          # !! package_info <-> self (double-linked)
-         package_info.overlay_package_ref = self
+         package_info.overlay_package_ref = weakref.ref ( self )
          return True
       else:
          return False
@@ -786,6 +800,10 @@ class PackageDirBase ( object ):
             or patch_ebuild ( efile, pvr, patchview.get_patches ( pvr ) )
          ):
 
+            # generate hashes here (benefit from threading)
+            # FIXME/TODO: ^ actually faster?
+            p_info.make_hashes ( self.HASHES )
+
             self._need_manifest = True
 
             # update metadata for each successfully written ebuild
@@ -848,6 +866,20 @@ class PackageDirBase ( object ):
          if p.has ( 'package_file', 'ebuild_file' )
       ]
 
+      print ( "--- DEBUG PRINT from packagedir_base :: {} :: {} ---".format (
+            self.__class__.__name__, self.name
+         )
+      )
+      print ( "Calculating hashes ..." )
+      for p in pkgs_for_manifest:
+         p.make_hashes ( self.HASHES )
+         print ( p.hashdict )
+      print ( "Done!" )
+      print ( "--- END DEBUG PRINT from packagedir_base :: {} :: {} ---".format (
+            self.__class__.__name__, self.name
+         )
+      )
+
       if pkgs_for_manifest:
          self.logger.debug ( "Writing Manifest" )
          if self._write_manifest ( pkgs_for_manifest ):

diff --git a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
index 5217f6a..9766606 100644
--- a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
@@ -102,7 +102,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
          # TODO: optimize this further?
          # -> "not has physical_only?"
          #     (should be covered by "has package_file")
-         distdir.add ( p ['package_file'], p ['package_src_destpath'] )
+         distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
       # -- end for;
 
       return self._do_ebuildmanifest ( ebuild_file, distdir )

diff --git a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
index 6596dec..f4527a1 100644
--- a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
@@ -151,7 +151,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
             manifest.addFile ( 'EBUILD', ebuild_filename )
 
          if not manifest.hasFile ( 'DIST', package_filename ):
-            distdir.add ( p ['package_file'], package_filename )
+            distdir.add ( p ['package_file'], package_filename, p )
             manifest.addFile (
                'DIST',
                package_filename,
@@ -165,7 +165,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
             # doing the check above (if hasFile())
             # be "fully compatible" and do that here, too
             try:
-               distdir.add ( p ['package_file'], package_filename )
+               distdir.add ( p ['package_file'], package_filename, p )
             except Exception as err:
                self.logger.exception ( err )
 

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index c2522b7..a3b02f1 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -23,14 +23,15 @@ import os
 import shutil
 import threading
 
+
 import roverlay.config
 import roverlay.util
+import roverlay.recipe.distmap
 import roverlay.overlay.additionsdir
 import roverlay.overlay.category
 import roverlay.overlay.header
 
 
-
 class Overlay ( object ):
    DEFAULT_USE_DESC = (
       'byte-compile - enable byte compiling\n'
@@ -730,6 +731,10 @@ class Overlay ( object ):
                write_manifest    = not self.skip_manifest,
                additions_dir     = self.additions_dir.get_obj_subdir ( cat ),
             )
+
+         distmap = roverlay.recipe.distmap.access()
+         distmap.backup_file ( ignore_missing=True )
+         distmap.write()
       else:
          # FIXME debug print
          print (

diff --git a/roverlay/packageinfo.py b/roverlay/packageinfo.py
index e8004d0..83a05ed 100644
--- a/roverlay/packageinfo.py
+++ b/roverlay/packageinfo.py
@@ -19,6 +19,7 @@ import logging
 import threading
 
 import roverlay.digest
+import roverlay.db.distmap
 
 from roverlay          import config, strutil
 from roverlay.rpackage import descriptionreader
@@ -116,6 +117,9 @@ class PackageInfo ( object ):
       'ebuild'
    ))
 
+   # bind DIGEST_TYPE to this class
+   DISTMAP_DIGEST_TYPE = roverlay.db.distmap.DistMapInfo.DIGEST_TYPE
+
    def __init__ ( self, **initial_info ):
       """Initializes a PackageInfo.
 
@@ -451,25 +455,45 @@ class PackageInfo ( object ):
    # --- end of get_desc_data (...) ---
 
    def get_distmap_item ( self ):
+      """Returns a 2-tuple ( key, info ) for the distmap."""
       return ( self.get_distmap_key(), self.get_distmap_value() )
    # --- end of get_distmap_item (...) ---
 
    def get_distmap_key ( self ):
+      """Returns a key for the distmap."""
       return self.get ( "package_src_destpath" )
    # --- end of get_distmap_key (...) ---
 
-   def get_distmap_value ( self ):
-      assert 'sha256' in self.hashdict
+   def get_distmap_value ( self, allow_digest_create=False ):
+      """Returns a data tuple for creating DistMapInfo instances.
 
+      arguments:
+      * allow_digest_create --
+      """
       repo = self.get ( "origin" )
       return (
          repo.name,
          os.path.relpath ( self.get ( "package_file" ), repo.distdir ),
-         self.hashdict ['sha256']
+         (
+            self.make_distmap_hash() if allow_digest_create
+            else self.hashdict [self.DISTMAP_DIGEST_TYPE]
+         )
       )
    # --- end of get_distmap_value (...) ---
 
+   def make_distmap_hash ( self ):
+      """Creates (and returns) the distmap package file hash."""
+      return self.make_hashes ( { self.DISTMAP_DIGEST_TYPE, } ) [self.DISTMAP_DIGEST_TYPE]
+   # --- end of make_distmap_hash (...) ---
+
    def make_hashes ( self, hashlist ):
+      """Creates zero or more hashes and returns the hashdict.
+
+      Note: the hashdict can also be accessed directly via <this>.hashdict.
+
+      arguments:
+      * hashlist -- list of hash names, e.g. "sha256", "md5"
+      """
       pkgfile = self.get ( "package_file" )
 
       if hasattr ( self, 'hashdict' ) and self.hashdict:
@@ -483,8 +507,26 @@ class PackageInfo ( object ):
             )
       else:
          self.hashdict = roverlay.digest.multihash_file ( pkgfile, hashlist )
+
+      return self.hashdict
    # --- end of make_hashes (...) ---
 
+   def revbump ( self, newrev=None ):
+      """Do whatever necessary to revbump this pakages, that is set/update
+      all data like src_uri_destfile.
+
+      arguments:
+      * newrev -- new revision, (current rev + 1) is used if this is None
+      """
+      raise NotImplementedError ( "revbump code" )
+      if newrev is None:
+         # get old rev and increment it
+         pass
+      else:
+         pass
+      return self
+   # --- end of revbump (...) ---
+
    def __getitem__ ( self, key ):
       """Returns an item."""
       return self.get ( key, do_fallback=False )


^ permalink raw reply related	[flat|nested] 2+ messages in thread

* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/distroot/, roverlay/db/, roverlay/, ...
@ 2013-06-22 15:24 André Erdmann
  2013-06-20 23:40 ` [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
  0 siblings, 1 reply; 2+ messages in thread
From: André Erdmann @ 2013-06-22 15:24 UTC (permalink / raw
  To: gentoo-commits

commit:     6a34fb49dd27014154427e95de9f1f7917cd90b9
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jun 20 23:34:54 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jun 20 23:34:54 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6a34fb49

roverlay: use distmap

Add distmap support to distroot/overlay.

revbump functionality in PackageInfo is missing and some debug output has to be
removed.

---
 roverlay/db/distmap.py                             |  81 ++++++++++-
 roverlay/overlay/creator.py                        |  11 +-
 roverlay/overlay/pkgdir/distroot/distdir.py        |  10 +-
 roverlay/overlay/pkgdir/distroot/distroot.py       | 155 ++++++++++++++++++---
 roverlay/overlay/pkgdir/distroot/static.py         |  10 +-
 roverlay/overlay/pkgdir/packagedir_base.py         |  48 +++++--
 .../overlay/pkgdir/packagedir_ebuildmanifest.py    |   2 +-
 .../overlay/pkgdir/packagedir_portagemanifest.py   |   4 +-
 roverlay/overlay/root.py                           |   7 +-
 roverlay/packageinfo.py                            |  48 ++++++-
 10 files changed, 335 insertions(+), 41 deletions(-)

diff --git a/roverlay/db/distmap.py b/roverlay/db/distmap.py
index 7bfba59..dfdcd01 100644
--- a/roverlay/db/distmap.py
+++ b/roverlay/db/distmap.py
@@ -13,6 +13,7 @@ import os.path
 import shutil
 
 
+import roverlay.digest
 import roverlay.util
 
 
@@ -21,20 +22,33 @@ __all__ = [ 'DistMapInfo', 'get_distmap' ]
 
 class DistMapInfo ( object ):
 
+   DIGEST_TYPE           = 'sha256'
    RESTORE_FROM_DISTFILE = '_'
+   UNSET                 = 'U'
 
    def __init__ ( self, distfile, repo_name, repo_file, sha256 ):
       super ( DistMapInfo, self ).__init__()
 
-      self.repo_name = repo_name
+      self.repo_name = repo_name if repo_name is not None else self.UNSET
       self.sha256    = sha256
 
       if repo_file == self.RESTORE_FROM_DISTFILE:
          self.repo_file = distfile
       else:
-         self.repo_file = repo_file
+         self.repo_file = repo_file if repo_file is not None else self.UNSET
    # --- end of __init__ (...) ---
 
+   @property
+   def digest ( self ):
+      return self.sha256
+      #return getattr ( self, self.DIGEST_TYPE )
+   # --- end of digest (...) ---
+
+   def compare_digest ( self, package_info ):
+      p_hash = package_info.make_distmap_hash()
+      return ( bool ( p_hash == self.digest ), p_hash )
+   # --- end of compare_digest (...) ---
+
    def __eq__ ( self, other ):
       return not self.__ne__ ( other )
    # --- end of __eq__ (...) ---
@@ -136,10 +150,61 @@ class _DistMapBase ( object ):
             setattr ( self, attr [1], getattr ( self._distmap, attr[0] ) )
    # --- end of _rebind_distmap (...) ---
 
+   def check_revbump_necessary ( self, package_info ):
+      """Tries to find package_info's distfile in the distmap and returns
+      whether a revbump is necessary (True) or not (False).
+
+      Compares checksums if distfile already exists.
+
+      arguments:
+      * package_info --
+      """
+      distfile = package_info.get_distmap_key()
+
+      info = self._distmap.get ( distfile, None )
+      if info is None:
+         # new file, no revbump required
+         return False
+      elif info.compare_digest ( package_info ) [0] is True:
+         # old digest == new digest, no revbump
+         #  (package_info should be filtered out)
+         return False
+      else:
+         # digest mismatch => diff
+         return True
+   # --- end of compare_digest (...) ---
+
+   def get_file_digest ( self, f ):
+      return roverlay.digest.dodigest_file ( f, DistMapInfo.DIGEST_TYPE )
+   # --- end of get_file_digest (...) ---
+
+   def check_integrity ( self, distfile, distfilepath ):
+      info = self._distmap.get ( distfile, None )
+
+      if info is None:
+         # file not found
+         return 1
+      elif info.digest == self.get_file_digest ( distfilepath ):
+         # file OK
+         return 0
+      else:
+         # bad checksum
+         return 2
+   # --- end of check_integrity (...) ---
+
    def remove ( self, key ):
       del self._distmap [key]
+      self.dirty = True
    # --- end of remove (...) ---
 
+   def try_remove ( self, key ):
+      try:
+         del self._distmap [key]
+         self.dirty = True
+      except KeyError:
+         pass
+   # --- end of try_remove (...) ---
+
    def make_reverse_distmap ( self ):
       self._reverse_distmap = {
          ( kv[1].repo_name, kv[1].repo_file ): kv
@@ -187,6 +252,16 @@ class _DistMapBase ( object ):
       )
    # --- end of add_entry_for (...) ---
 
+   def add_dummy_entry ( self, distfile, distfilepath ):
+      print ( "DUMMY", distfile )
+      return self.add_entry (
+         distfile,
+         DistMapInfo (
+            distfile, None, None, self.get_file_digest ( distfilepath ),
+         )
+      )
+   # --- end of add_dummy_entry (...) ---
+
    def read ( self, *args, **kwargs ):
       raise self.__class__.AbstractMethod()
    # --- end of read (...) ---
@@ -304,6 +379,7 @@ class FileDistMap ( _FileDistMapBase ):
 
    def write ( self, filepath=None, force=False ):
       if force or self.dirty:
+         print ( "DBFILE WILL BE WRITTEN", force, dirty, list(self.keys()) )
          f  = filepath or self.dbfile
          roverlay.util.dodir ( os.path.dirname ( f ), mkdir_p=True )
          with open ( f, 'wt' ) as FH:
@@ -313,6 +389,7 @@ class FileDistMap ( _FileDistMapBase ):
             self._file_written ( f )
          return True
       else:
+         print ( "DBFILE WILL NOT BE WRITTEN", force, dirty, list(self.keys()) )
          return False
    # --- end of write (...) ---
 

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 4c29ffa..79a70bf 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -34,6 +34,9 @@ from roverlay.packagerules.rules import PackageRules
 
 from roverlay.recipe             import easyresolver
 
+import roverlay.recipe.distmap
+import roverlay.overlay.pkgdir.distroot.static
+
 
 class PseudoAtomicCounter ( object ):
 
@@ -97,6 +100,10 @@ class OverlayCreator ( object ):
 
       self.rsuggests_flags = set()
 
+      # create distmap and distroot here
+      self.distmap  = roverlay.recipe.distmap.setup()
+      self.distroot = roverlay.overlay.pkgdir.distroot.static.get_configured()
+
       time_scan_overlay = time.time()
       # init overlay using config values
       self.overlay = Overlay.new_configured (
@@ -462,10 +469,10 @@ class OverlayCreator ( object ):
       """
       if package_info ['ebuild'] is not None:
          self.create_success.inc()
-         if package_info.overlay_package_ref.new_ebuild():
+         if package_info.overlay_package_ref().new_ebuild():
             self.overlay_added.inc()
       else:
-         package_info.overlay_package_ref.ebuild_uncreateable ( package_info )
+         package_info.overlay_package_ref().ebuild_uncreateable ( package_info )
          self.create_fail.inc()
 
    # --- end of _add_to_overlay (...) ---

diff --git a/roverlay/overlay/pkgdir/distroot/distdir.py b/roverlay/overlay/pkgdir/distroot/distdir.py
index 0f76e30..62cb210 100644
--- a/roverlay/overlay/pkgdir/distroot/distdir.py
+++ b/roverlay/overlay/pkgdir/distroot/distdir.py
@@ -22,11 +22,15 @@ class Distdir ( object ):
       self._distroot = distroot
    # --- end of __init__ (...) ---
 
-   def add ( self, fpath, fname=None ):
-      return self._distroot._add (
+   def add ( self, fpath, fname, p_info ):
+      if self._distroot._add (
          fpath,
          self.get_root() + os.sep + ( fname or os.path.basename ( fpath ) )
-      )
+      ):
+         self._distroot.distmap_register ( p_info )
+         return True
+      else:
+         return False
    # --- end of add (...) ---
 
    def get_root ( self ):

diff --git a/roverlay/overlay/pkgdir/distroot/distroot.py b/roverlay/overlay/pkgdir/distroot/distroot.py
index 9f56742..c59d813 100644
--- a/roverlay/overlay/pkgdir/distroot/distroot.py
+++ b/roverlay/overlay/pkgdir/distroot/distroot.py
@@ -15,8 +15,11 @@ import os
 import shutil
 import tempfile
 
+
+import roverlay.db.distmap
 import roverlay.overlay.pkgdir.distroot.distdir
 
+
 class DistrootBase ( object ):
    """Base class for distroots."""
 
@@ -31,20 +34,24 @@ class DistrootBase ( object ):
       return self.get_root()
    # --- end of __str__ (...) ---
 
-   def __init__ ( self, root, flat ):
+   def __init__ ( self, root, flat, logger=None, distmap=None ):
       """DistrootBase constructor.
 
       arguments:
-      * root -- root directory
-      * flat -- whether to use a flat structure (all packages in a single
-                 directory, True) or per-package sub directories (False)
+      * root    -- root directory
+      * flat    -- whether to use a flat structure (all packages in a single
+                    directory, True) or per-package sub directories (False)
+      * logger  --
+      * distmap --
       """
       super ( DistrootBase, self ).__init__()
-      self.logger = logging.getLogger ( self.__class__.__name__ )
+      self.logger = logger or logging.getLogger ( self.__class__.__name__ )
       self._root  = root
       # or use hasattr ( self, '_default_distdir' )
       self._flat  = flat
 
+      self.distmap = distmap
+
       if flat:
          self._default_distdir = (
             roverlay.overlay.pkgdir.distroot.distdir.Distdir ( self )
@@ -166,29 +173,66 @@ class DistrootBase ( object ):
       pass
    # --- end of _prepare (...) ---
 
+   def iter_distfiles ( self, distfile_only ):
+      def recursive_iter ( root_abspath, root_relpath ):
+         for item in os.listdir ( root_abspath ):
+            abspath = root_abspath + os.sep + item
+            relpath = (
+               item if root_relpath is None
+               else root_relpath + os.sep + item
+            )
+            if os.path.isdir ( abspath ):
+               for result in iter_distfiles ( abspath, relpath ):
+                  yield result
+            else:
+               yield ( abspath, relpath )
+      # --- end of recursive_iter (...) ---
+
+      if distfile_only:
+         for pkgfile, distfile in recursive_iter ( self.get_root(), None ):
+            yield distfile
+      else:
+         for pkgfile, distfile in recursive_iter ( self.get_root(), None ):
+            yield ( pkgfile, distfile )
+   # --- end of iter_distfiles (...) ---
+
    def _remove_broken_symlinks ( self ):
       """Recursively remove broken/dead symlinks."""
-      def recursive_remove ( root, rmdir ):
-         for item in os.listdir ( root ):
-            fpath = root + os.sep + item
+
+      def recursive_remove ( dirpath, rel_dirpath, rmdir ):
+         for item in os.listdir ( dirpath ):
+            fpath   = dirpath + os.sep + item
+            relpath = (
+               item if rel_dirpath is None else rel_dirpath + os.sep + item
+            )
 
             if not os.path.exists ( fpath ):
+               # drop broken symlink
+               self.logger.debug (
+                  "Removing broken symlink {!r}".format ( fpath )
+               )
                os.unlink ( fpath )
-
+               if self.distmap is not None:
+                  self.distmap.try_remove ( relpath )
             elif os.path.isdir ( fpath ):
-               recursive_remove ( fpath, True )
+               recursive_remove ( fpath, relpath, True )
                if rmdir:
                   try:
                      os.rmdir ( fpath )
                   except OSError:
                      pass
+         # -- end for
       # --- end of recursive_remove (...) ---
-      recursive_remove ( self.get_root(), False )
+
+      return recursive_remove ( self.get_root(), None, False )
    # --- end of _remove_broken_symlinks (...) ---
 
    def _try_remove ( self, dest ):
       try:
          os.unlink ( dest )
+         if self.distmap is not None:
+            relpath = os.path.relpath ( dest, self.get_root() )
+
       except OSError as e:
          if e.errno == errno.ENOENT:
             pass
@@ -219,16 +263,72 @@ class DistrootBase ( object ):
       return str ( self._root )
    # --- end of get_root (...) ---
 
+   def distmap_register ( self, p_info ):
+      return self.distmap.add_entry_for ( p_info )
+   # --- end of distmap_register (...) ---
+
+   def check_integrity ( self ):
+      if self.distmap is not None:
+         root      = self.get_root()
+         distfiles = set()
+         checkfile = self.distmap.check_integrity
+
+         for abspath, relpath in self.iter_distfiles ( False ):
+            status = checkfile ( relpath, abspath )
+
+            if status == 0:
+               self.logger.debug (
+                  "file has been verified: {!r}".format ( relpath )
+               )
+               distfiles.add ( relpath )
+            elif status == 1:
+               # file not in distmap
+               self.logger.info (
+                  "file not in distmap, creating dummy entry: {!r}".format ( relpath )
+               )
+               self.distmap.add_dummy_entry ( relpath, abspath )
+               distfiles.add ( relpath )
+            elif status == 2:
+               # file in distmap, but not valid - remove it from distmap
+               self.logger.warning (
+                  "digest mismatch: {!r}".format ( relpath )
+               )
+               self.distmap.remove ( relpath )
+         # -- end for
+
+         distmap_keys = frozenset ( self.distmap.keys() )
+
+         if distfiles:
+            # reverse compare
+            for distfile in distmap_keys:
+               if distfile not in distfiles:
+                  self.logger.warning (
+                     "distmap file does not exist: {!r}".format ( distfile )
+                  )
+                  self.distmap.remove ( distfile )
+
+         else:
+            # no files from distroot in distmap -- invalidate distmap
+            for distfile in distmap_keys:
+               self.logger.warning (
+                  "distmap file does not exist: {!r}".format ( distfile )
+               )
+               self.distmap.remove ( distfile )
+      else:
+         raise Exception ( "check_integrity() needs a distmap." )
+   # --- end of check_integrity (...) ---
+
 # --- end of DistrootBase ---
 
 
 class TemporaryDistroot ( DistrootBase ):
 
-   def __init__ ( self ):
+   def __init__ ( self, logger=None ):
       # temporary distroots always use the non-flat distdir layout
       super ( TemporaryDistroot, self ).__init__ (
-         root = tempfile.mkdtemp ( prefix='tmp_roverlay_distroot_' ),
-         flat = False
+         root   = tempfile.mkdtemp ( prefix='tmp_roverlay_distroot_' ),
+         flat   = False,
+         logger = logger,
       )
    # --- end of __init__ (...) ---
 
@@ -238,6 +338,7 @@ class TemporaryDistroot ( DistrootBase ):
 
    def _cleanup ( self ):
       """Cleans up the temporary distroot by simply wiping it."""
+      super ( TemporaryDistroot, self )._cleanup()
       shutil.rmtree ( self._root )
    # --- end of _cleanup (...) ---
 
@@ -265,7 +366,9 @@ class PersistentDistroot ( DistrootBase ):
       )
    # --- end of __repr__ (...) ---
 
-   def __init__ ( self, root, flat, strategy ):
+   def __init__ ( self,
+      root, flat, strategy, distmap, verify=False, logger=None
+   ):
       """Initializes a non-temporary distroot.
 
       arguments:
@@ -274,8 +377,13 @@ class PersistentDistroot ( DistrootBase ):
       * strategy -- the distroot 'strategy' that determines what mode (sym-
                     link, hardlink, copy) will be tried in which order
                     This has to be an iterable with valid items.
+      * distmap  --
+      * verify   --
+      * logger   --
       """
-      super ( PersistentDistroot, self ).__init__ ( root=root, flat=flat )
+      super ( PersistentDistroot, self ).__init__ (
+         root=root, flat=flat, logger=logger, distmap=distmap
+      )
 
       self._strategy = self._get_int_strategy ( strategy )
 
@@ -294,6 +402,15 @@ class PersistentDistroot ( DistrootBase ):
          self.USE_HARDLINK : self._add_hardlink,
          self.USE_COPY     : self._add_file,
       }
+
+
+      if verify and self.distmap is not None:
+         # FIXME debug print
+         print (
+            "Checking distroot file integrity, this may take some time ... "
+         )
+         self.check_integrity()
+         print ( "Done!" )
    # --- end of __init__ (...) ---
 
    def _add ( self, src, dest ):
@@ -328,10 +445,14 @@ class PersistentDistroot ( DistrootBase ):
    # --- end of _add (...) ---
 
    def _cleanup ( self ):
+      super ( PersistentDistroot, self )._cleanup()
       if hasattr ( self, '_supported_modes_initial' ):
          if self._supported_modes_initial & self.USE_SYMLINK:
             self._remove_broken_symlinks()
-   # --- end of _prepare (...) ---
+
+      if self.distmap is not None:
+         self.distmap.write ( force=False )
+   # --- end of _cleanup (...) ---
 
    def _get_int_strategy ( self, strategy ):
       """Converts the given strategy into its integer tuple representation.

diff --git a/roverlay/overlay/pkgdir/distroot/static.py b/roverlay/overlay/pkgdir/distroot/static.py
index b4ad055..0054140 100644
--- a/roverlay/overlay/pkgdir/distroot/static.py
+++ b/roverlay/overlay/pkgdir/distroot/static.py
@@ -8,6 +8,8 @@ __all__ = [ 'get_configured', 'get_distdir', ]
 
 import threading
 
+import roverlay.recipe.distmap
+
 import roverlay.config
 
 import roverlay.overlay.pkgdir.distroot.distroot
@@ -34,7 +36,11 @@ def get_configured ( static=True ):
             # generally, the "flat" distroot/distdir layout is desired as it
             # can serve as package mirror directory, so default to True here
             flat     = roverlay.config.get ( 'OVERLAY.DISTDIR.flat', True ),
-            strategy = distdir_strategy
+            strategy = distdir_strategy,
+            distmap  = roverlay.recipe.distmap.access(),
+            # FIXME/TODO: 'verify' config key does not exist
+            verify   = roverlay.config.get ( 'OVERLAY.DISTDIR.verify', False ),
+
          )
    # --- end of get_new (...) ---
 
@@ -49,7 +55,7 @@ def get_configured ( static=True ):
 
       return _distroot_instance
    else:
-      return get_new()
+      raise Exception ( "static keyword arg is deprecated" )
 # --- end of get_configured (...) ---
 
 def get_distdir ( ebuild_name ):

diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index e67114e..a68ecbd 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -20,12 +20,14 @@ import os
 import shutil
 import sys
 import threading
-
+import weakref
 
 import roverlay.config
 import roverlay.packageinfo
 import roverlay.util
 
+import roverlay.recipe.distmap
+
 import roverlay.tools.ebuild
 import roverlay.tools.ebuildenv
 import roverlay.tools.patch
@@ -40,8 +42,10 @@ class PackageDirBase ( object ):
    for Manifest file creation."""
 
    #DISTROOT =
+   #DISTMAP  =
    EBUILD_SUFFIX       = '.ebuild'
    #FETCHENV =
+   #HASHES   =
    SUPPRESS_EXCEPTIONS = True
 
    # MANIFEST_THREADSAFE (tri-state)
@@ -59,9 +63,11 @@ class PackageDirBase ( object ):
          roverlay.config.get_or_fail ( 'OVERLAY.dir' )
       )
 
-      cls.DISTROOT = (
-         roverlay.overlay.pkgdir.distroot.static.get_configured ( static=True )
-      )
+      cls.DISTROOT = roverlay.overlay.pkgdir.distroot.static.get_configured()
+      cls.DISTMAP  = roverlay.recipe.distmap.access()
+      # FIXME/TODO: config
+      #cls.HASHES = frozenset ({ 'sha256', 'sha512', 'whirlpool', })
+      cls.HASHES = frozenset ({ 'sha256', })
       cls.FETCHENV = fetch_env
    # --- end of init_cls (...) ---
 
@@ -142,11 +148,12 @@ class PackageDirBase ( object ):
       p = roverlay.packageinfo.PackageInfo (
          physical_only=True, pvr=pvr, ebuild_file=efile
       )
+      # TODO/FIXME: parse SRC_URI, knowledge of distfile path would be good...
       self._packages [ p ['ebuild_verstr'] ] = p
       return p
    # --- end of _scan_add_package (...) ---
 
-   def add ( self, package_info, add_if_physical=False ):
+   def add ( self, package_info, add_if_physical=False, _lock=True ):
       """Adds a package to this PackageDir.
 
       arguments:
@@ -161,7 +168,8 @@ class PackageDirBase ( object ):
       shortver = package_info ['ebuild_verstr']
       added = False
       try:
-         self._lock.acquire()
+         if _lock: self._lock.acquire()
+
          if shortver in self._packages:
             # package exists, check if it existed before script invocation
             if self._packages [shortver] ['physical_only']:
@@ -170,6 +178,12 @@ class PackageDirBase ( object ):
                   self._packages [shortver] = package_info
                   added = True
 
+               elif self.DISTMAP.check_revbump_necessary ( package_info ):
+                  # resolve by recursion
+                  added = self.add (
+                     package_info.revbump(), add_if_physical=False, _lock=False
+                  )
+
                else:
                   self.logger.debug (
                      "'{PN}-{PVR}.ebuild' exists as file, skipping.".format (
@@ -190,12 +204,12 @@ class PackageDirBase ( object ):
             added = True
 
       finally:
-         self._lock.release()
+         if _lock: self._lock.release()
 
       if added:
          # add a link to this PackageDir into the package info,
          # !! package_info <-> self (double-linked)
-         package_info.overlay_package_ref = self
+         package_info.overlay_package_ref = weakref.ref ( self )
          return True
       else:
          return False
@@ -786,6 +800,10 @@ class PackageDirBase ( object ):
             or patch_ebuild ( efile, pvr, patchview.get_patches ( pvr ) )
          ):
 
+            # generate hashes here (benefit from threading)
+            # FIXME/TODO: ^ actually faster?
+            p_info.make_hashes ( self.HASHES )
+
             self._need_manifest = True
 
             # update metadata for each successfully written ebuild
@@ -848,6 +866,20 @@ class PackageDirBase ( object ):
          if p.has ( 'package_file', 'ebuild_file' )
       ]
 
+      print ( "--- DEBUG PRINT from packagedir_base :: {} :: {} ---".format (
+            self.__class__.__name__, self.name
+         )
+      )
+      print ( "Calculating hashes ..." )
+      for p in pkgs_for_manifest:
+         p.make_hashes ( self.HASHES )
+         print ( p.hashdict )
+      print ( "Done!" )
+      print ( "--- END DEBUG PRINT from packagedir_base :: {} :: {} ---".format (
+            self.__class__.__name__, self.name
+         )
+      )
+
       if pkgs_for_manifest:
          self.logger.debug ( "Writing Manifest" )
          if self._write_manifest ( pkgs_for_manifest ):

diff --git a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
index 5217f6a..9766606 100644
--- a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
@@ -102,7 +102,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
          # TODO: optimize this further?
          # -> "not has physical_only?"
          #     (should be covered by "has package_file")
-         distdir.add ( p ['package_file'], p ['package_src_destpath'] )
+         distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
       # -- end for;
 
       return self._do_ebuildmanifest ( ebuild_file, distdir )

diff --git a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
index 6596dec..f4527a1 100644
--- a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
@@ -151,7 +151,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
             manifest.addFile ( 'EBUILD', ebuild_filename )
 
          if not manifest.hasFile ( 'DIST', package_filename ):
-            distdir.add ( p ['package_file'], package_filename )
+            distdir.add ( p ['package_file'], package_filename, p )
             manifest.addFile (
                'DIST',
                package_filename,
@@ -165,7 +165,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
             # doing the check above (if hasFile())
             # be "fully compatible" and do that here, too
             try:
-               distdir.add ( p ['package_file'], package_filename )
+               distdir.add ( p ['package_file'], package_filename, p )
             except Exception as err:
                self.logger.exception ( err )
 

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index c2522b7..a3b02f1 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -23,14 +23,15 @@ import os
 import shutil
 import threading
 
+
 import roverlay.config
 import roverlay.util
+import roverlay.recipe.distmap
 import roverlay.overlay.additionsdir
 import roverlay.overlay.category
 import roverlay.overlay.header
 
 
-
 class Overlay ( object ):
    DEFAULT_USE_DESC = (
       'byte-compile - enable byte compiling\n'
@@ -730,6 +731,10 @@ class Overlay ( object ):
                write_manifest    = not self.skip_manifest,
                additions_dir     = self.additions_dir.get_obj_subdir ( cat ),
             )
+
+         distmap = roverlay.recipe.distmap.access()
+         distmap.backup_file ( ignore_missing=True )
+         distmap.write()
       else:
          # FIXME debug print
          print (

diff --git a/roverlay/packageinfo.py b/roverlay/packageinfo.py
index e8004d0..83a05ed 100644
--- a/roverlay/packageinfo.py
+++ b/roverlay/packageinfo.py
@@ -19,6 +19,7 @@ import logging
 import threading
 
 import roverlay.digest
+import roverlay.db.distmap
 
 from roverlay          import config, strutil
 from roverlay.rpackage import descriptionreader
@@ -116,6 +117,9 @@ class PackageInfo ( object ):
       'ebuild'
    ))
 
+   # bind DIGEST_TYPE to this class
+   DISTMAP_DIGEST_TYPE = roverlay.db.distmap.DistMapInfo.DIGEST_TYPE
+
    def __init__ ( self, **initial_info ):
       """Initializes a PackageInfo.
 
@@ -451,25 +455,45 @@ class PackageInfo ( object ):
    # --- end of get_desc_data (...) ---
 
    def get_distmap_item ( self ):
+      """Returns a 2-tuple ( key, info ) for the distmap."""
       return ( self.get_distmap_key(), self.get_distmap_value() )
    # --- end of get_distmap_item (...) ---
 
    def get_distmap_key ( self ):
+      """Returns a key for the distmap."""
       return self.get ( "package_src_destpath" )
    # --- end of get_distmap_key (...) ---
 
-   def get_distmap_value ( self ):
-      assert 'sha256' in self.hashdict
+   def get_distmap_value ( self, allow_digest_create=False ):
+      """Returns a data tuple for creating DistMapInfo instances.
 
+      arguments:
+      * allow_digest_create --
+      """
       repo = self.get ( "origin" )
       return (
          repo.name,
          os.path.relpath ( self.get ( "package_file" ), repo.distdir ),
-         self.hashdict ['sha256']
+         (
+            self.make_distmap_hash() if allow_digest_create
+            else self.hashdict [self.DISTMAP_DIGEST_TYPE]
+         )
       )
    # --- end of get_distmap_value (...) ---
 
+   def make_distmap_hash ( self ):
+      """Creates (and returns) the distmap package file hash."""
+      return self.make_hashes ( { self.DISTMAP_DIGEST_TYPE, } ) [self.DISTMAP_DIGEST_TYPE]
+   # --- end of make_distmap_hash (...) ---
+
    def make_hashes ( self, hashlist ):
+      """Creates zero or more hashes and returns the hashdict.
+
+      Note: the hashdict can also be accessed directly via <this>.hashdict.
+
+      arguments:
+      * hashlist -- list of hash names, e.g. "sha256", "md5"
+      """
       pkgfile = self.get ( "package_file" )
 
       if hasattr ( self, 'hashdict' ) and self.hashdict:
@@ -483,8 +507,26 @@ class PackageInfo ( object ):
             )
       else:
          self.hashdict = roverlay.digest.multihash_file ( pkgfile, hashlist )
+
+      return self.hashdict
    # --- end of make_hashes (...) ---
 
+   def revbump ( self, newrev=None ):
+      """Do whatever necessary to revbump this pakages, that is set/update
+      all data like src_uri_destfile.
+
+      arguments:
+      * newrev -- new revision, (current rev + 1) is used if this is None
+      """
+      raise NotImplementedError ( "revbump code" )
+      if newrev is None:
+         # get old rev and increment it
+         pass
+      else:
+         pass
+      return self
+   # --- end of revbump (...) ---
+
    def __getitem__ ( self, key ):
       """Returns an item."""
       return self.get ( key, do_fallback=False )


^ permalink raw reply related	[flat|nested] 2+ messages in thread

end of thread, other threads:[~2013-06-22 15:24 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-06-22 15:24 [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/distroot/, roverlay/db/, roverlay/, André Erdmann
2013-06-20 23:40 ` [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox