[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/

[André Erdmann]

commit:     4d105910093b10ca2aa721e1209f75f4fb2b1649
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun  5 18:00:19 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun  5 18:00:19 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=4d105910

overlay/pkgdir: fs_destroy()

This function removes all ebuilds of a package from the overlay.

---
 roverlay/overlay/pkgdir/packagedir_base.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 8fb6ca4..62dd7c7 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -105,6 +105,8 @@ class PackageDirBase ( object ):
       """Called for each ebuild that is found during scan().
       Creates a PackageInfo for the ebuild and adds it to self._packages.
 
+      PackageInfo objects added this way are not affected by package rules.
+
       arguments:
       * efile -- full path to the ebuild file
       * pvr   -- version ($PVR) of the ebuild
@@ -353,6 +355,15 @@ class PackageDirBase ( object ):
          return None
    # --- end of purge_package (...) ---
 
+   def fs_destroy ( self ):
+      pvr_list = list ( self._packages.keys() )
+      for pvr in pvr_list:
+         self.purge_package ( pvr )
+
+      assert not self.empty()
+      self.fs_cleanup()
+   # --- end of fs_destroy (...) ---
+
    def scan ( self, **kw ):
       """Scans the filesystem location of this package for existing
       ebuilds and adds them.

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/

[André Erdmann]

commit:     4d105910093b10ca2aa721e1209f75f4fb2b1649
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun  5 18:00:19 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun  5 18:00:19 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=4d105910

overlay/pkgdir: fs_destroy()

This function removes all ebuilds of a package from the overlay.

---
 roverlay/overlay/pkgdir/packagedir_base.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 8fb6ca4..62dd7c7 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -105,6 +105,8 @@ class PackageDirBase ( object ):
       """Called for each ebuild that is found during scan().
       Creates a PackageInfo for the ebuild and adds it to self._packages.
 
+      PackageInfo objects added this way are not affected by package rules.
+
       arguments:
       * efile -- full path to the ebuild file
       * pvr   -- version ($PVR) of the ebuild
@@ -353,6 +355,15 @@ class PackageDirBase ( object ):
          return None
    # --- end of purge_package (...) ---
 
+   def fs_destroy ( self ):
+      pvr_list = list ( self._packages.keys() )
+      for pvr in pvr_list:
+         self.purge_package ( pvr )
+
+      assert not self.empty()
+      self.fs_cleanup()
+   # --- end of fs_destroy (...) ---
+
    def scan ( self, **kw ):
       """Scans the filesystem location of this package for existing
       ebuilds and adds them.

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/

[André Erdmann]

commit:     5e0459d05a23a44110fdc0342d7e5c8fd566ee80
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul  8 08:12:43 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul  8 08:12:43 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=5e0459d0

reenable --write-immediate

As ebuild creation is now complete _after_ validating selfdeps,
--write-immediate is no longer blocked by it.

---
 roverlay/overlay/pkgdir/packagedir_base.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 3c28703..63adfb4 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -384,10 +384,6 @@ class PackageDirBase ( object ):
       self._need_metadata = True
       self.modified       = True
       if self.runtime_incremental:
-         raise Exception (
-            '--write-immediate is currently not available '
-            '(blocked by selfdep reduction).'
-         )
          with self._lock:
             return self.write_ebuilds ( overwrite=False )
       else:

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/

[André Erdmann]

commit:     45b5e9f8e6aac6d6729e96b46849f7bf4e9c9f56
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 14:09:38 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 14:09:38 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=45b5e9f8

pkgdir_base: fixup

* unused variables removed: HASHES, SUPPRESS_EXCEPTIONS
* added DOEBUILD_FETCH to set the doebuild fetch function (class-wide)
* get_distdir(), fetch_src_for_ebuild() added

---
 roverlay/overlay/pkgdir/packagedir_base.py | 39 +++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 63adfb4..53ebc42 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -1,6 +1,6 @@
 # R overlay -- overlay package, package directory
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
 # Distributed under the terms of the GNU General Public License;
 # either version 2 of the License, or (at your option) any later version.
 
@@ -37,6 +37,8 @@ import roverlay.overlay.additionsdir
 import roverlay.overlay.pkgdir.distroot.static
 import roverlay.overlay.pkgdir.metadata
 
+# TODO: proper reading of $SRC_URI when importing (or scanning) ebuilds
+#        This would make manifest creation being more controlled
 
 class PackageDirBase ( object ):
    """The PackageDir base class that implements most functionality except
@@ -44,10 +46,9 @@ class PackageDirBase ( object ):
 
    #DISTROOT =
    #DISTMAP  =
-   EBUILD_SUFFIX       = '.ebuild'
    #FETCHENV =
-   #HASHES   =
-   SUPPRESS_EXCEPTIONS = True
+
+   EBUILD_SUFFIX  = '.ebuild'
 
    # MANIFEST_THREADSAFE (tri-state)
    # * None  -- unknown (e.g. write_manifest() not implemented)
@@ -56,6 +57,12 @@ class PackageDirBase ( object ):
    #
    MANIFEST_THREADSAFE = None
 
+   # DOEBUILD_FETCH
+   #  doebuild function that fetches $SRC_URI
+   #  can be overridden by subclasses if e.g. on-the-fly manifest creation
+   #  is required, too
+   DOEBUILD_FETCH = roverlay.tools.ebuild.doebuild_fetch
+
    @classmethod
    def init_base_cls ( cls ):
       # env for calling "ebuild <ebuild file> fetch"
@@ -66,9 +73,6 @@ class PackageDirBase ( object ):
 
       cls.DISTROOT = roverlay.overlay.pkgdir.distroot.static.get_configured()
       cls.DISTMAP  = roverlay.recipe.distmap.access()
-      # FIXME/TODO: config
-      #cls.HASHES = frozenset ({ 'sha256', 'sha512', 'whirlpool', })
-      cls.HASHES = frozenset ({ 'sha256', })
       cls.FETCHENV = fetch_env
    # --- end of init_cls (...) ---
 
@@ -581,6 +585,18 @@ class PackageDirBase ( object ):
       return success
    # --- end of write (...) ---
 
+   def get_distdir ( self ):
+      return self.DISTROOT.get_distdir ( self.name )
+   # --- end of get_distdir (...) ---
+
+   def fetch_src_for_ebuild ( self, efile ):
+      return self.DOEBUILD_FETCH (
+         ebuild_file = efile,
+         logger      = self.logger,
+         env         = self.FETCHENV.get_env ( self.get_distdir().get_root() )
+      )
+   # --- end of fetch_src_for_ebuild (...) ---
+
    def import_ebuilds ( self, eview, overwrite, nosync=False ):
       """Imports ebuilds from an additions dir into this package dir.
 
@@ -625,12 +641,7 @@ class PackageDirBase ( object ):
             self._need_manifest = True
 
             # fetch SRC_URI using ebuild(1)
-            if not nosync and not roverlay.tools.ebuild.doebuild_fetch (
-               efile_dest, self.logger,
-               self.FETCHENV.get_env (
-                  self.DISTROOT.get_distdir ( self.name ).get_root()
-               )
-            ):
+            if not nosync and not self.fetch_src_for_ebuild ( efile_dest ):
                raise Exception ( "doebuild_fetch() failed." )
 
             # imported ebuilds cannot be used for generating metadata.xml
@@ -810,7 +821,6 @@ class PackageDirBase ( object ):
 
             # generate hashes here (benefit from threading)
             # FIXME/TODO: ^ actually faster?
-            ##p_info.make_hashes ( self.HASHES )
             p_info.make_distmap_hash()
 
             self._need_manifest = True
@@ -884,6 +894,7 @@ class PackageDirBase ( object ):
             return True
          else:
             return False
+
       elif (
          hasattr ( self, '_write_import_manifest' )
          and self._write_import_manifest()

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/

[André Erdmann]

commit:     d0803a228f07e2c535dac48b21d07499d0a6e8db
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 15:07:16 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 15:07:16 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d0803a22

remove portagemanifest

---
 .../overlay/pkgdir/packagedir_portagemanifest.py   | 183 ---------------------
 1 file changed, 183 deletions(-)

diff --git a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py b/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
deleted file mode 100644
index f4527a1..0000000
--- a/roverlay/overlay/pkgdir/packagedir_portagemanifest.py
+++ /dev/null
@@ -1,183 +0,0 @@
-# R overlay -- overlay package, package directory (portage manifest)
-# -*- coding: utf-8 -*-
-# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
-# Distributed under the terms of the GNU General Public License;
-# either version 2 of the License, or (at your option) any later version.
-
-__all__ = [ 'PackageDir', ]
-
-import os
-import shlex
-
-import portage.manifest
-import portage.exception
-
-import logging
-logging.getLogger ( __name__ ).warning (
-   "experimental code, importing ebuilds doesn't work!"
-)
-del logging
-
-import roverlay.config
-import roverlay.strutil
-import roverlay.packageinfo
-import roverlay.overlay.pkgdir.packagedir_base
-import roverlay.overlay.pkgdir.distroot
-
-class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
-   # FIXME: portagemanifest is broken, it cannot create Manifest files for
-   # package dirs with imported ebuilds
-
-   MANIFEST_THREADSAFE = True
-
-   def import_ebuilds ( self, *args, **kwargs ):
-      raise NotImplementedError (
-         "ebuild imports not supported by portagemanifest!"
-      )
-   # --- end of import_ebuilds (...) ---
-
-   def _scan_add_package ( self, efile, pvr ):
-      """Called for each ebuild that is found during scan().
-      Creates a PackageInfo for the ebuild and adds it to self._packages.
-
-      arguments:
-      * efile -- full path to the ebuild file
-      * pvr   -- version ($PVR) of the ebuild
-      """
-
-      # read SRC_URI from the ebuild file which is the only way to get
-      # the correct package name
-      src_uri = None
-      with open ( efile, 'r' ) as FH:
-         reader                   = shlex.shlex ( FH )
-         reader.whitespace_split  = False
-         reader.wordchars        += ' ,./$()[]:+-@*~<>'
-
-         # assumption: only one SRC_URI= statement per ebuild
-         # (true for roverlay ebuilds)
-         mode  = 0
-         token = reader.get_token()
-         while token:
-            if mode == 0 and token == 'SRC_URI':
-               mode = 1
-            elif mode == 1 and token == '=':
-               mode = 2
-            elif mode == 2:
-               mode = 3
-               src_uri = tuple (
-                  roverlay.strutil.split_whitespace (
-                     roverlay.strutil.unquote ( token )
-                  )
-               )
-
-            # break loop if SRC_URI parsed
-            token = reader.get_token() if mode < 3 else None
-         # --- while;
-         del token, reader, mode
-      # --- with;
-
-      # another assumption:
-      #  name of the R package is src_uri[2] if src_uri[1] == '->', else [0]
-      #
-      if src_uri:
-         # > 2, > 1? ebuild would be broken if $SRC_URI ends with '->'
-         #if len ( src_uri ) > 2 and src_uri [1] == '->':
-         if len ( src_uri ) > 1 and src_uri [1] == '->':
-            package_filename = src_uri[2]
-         else:
-            package_filename = src_uri[0].rpartition ( '/' ) [2]
-
-         p = roverlay.packageinfo.PackageInfo (
-            physical_only=True, pvr=pvr, ebuild_file=efile,
-            package_filename=package_filename
-         )
-      else:
-         p = roverlay.packageinfo.PackageInfo (
-            physical_only=True, pvr=pvr, ebuild_file=efile
-         )
-      # --- if;
-
-      self._packages [ p ['ebuild_verstr'] ] = p
-   # --- end of _scan_add_package (...) ---
-
-   def _write_manifest ( self, pkgs_for_manifest ):
-      """Generates and writes the Manifest file for this package.
-
-      expects: called after writing metadata/ebuilds
-
-      returns: True
-      """
-
-      # TODO: this needs proper testing
-      # * the written Manifest file must not differ from the one generated by
-      #   ebuild(1) (the order of the entries is not important, though)
-      # * safe for incremental usage?
-      # * correct usage of the portage libs (also see other comments below)
-      # * what happens if an ebuild has been removed?
-      # * ...
-      #
-
-      distdir = self.DISTROOT.get_distdir ( self.name )
-
-      # allow_missing=True -- don't write empty Manifest files
-      manifest = portage.manifest.Manifest (
-         self.physical_location,
-         distdir.get_root(),
-         allow_missing=True,
-      )
-
-      # metadata.xml
-      #os.path.basename ( self._metadata.filepath )
-      try:
-         manifest.addFile ( 'MISC', 'metadata.xml' )
-      except portage.exception.FileNotFound as f404:
-         # package dir has no metadata.xml file
-         #
-         # This happens for a few package dirs that dont contain any package
-         # with enough information required for metadata creation
-         #
-         # packagedir_ebuildmanifest accepts this case without complaining,
-         # so ignore it here, too.
-         #
-         pass
-
-
-      for p in pkgs_for_manifest:
-
-         ebuild_filename  = p ['ebuild_filename']
-         package_filename = p ['package_src_destpath']
-
-         if not manifest.hasFile ( 'EBUILD', ebuild_filename ):
-            manifest.addFile ( 'EBUILD', ebuild_filename )
-
-         if not manifest.hasFile ( 'DIST', package_filename ):
-            distdir.add ( p ['package_file'], package_filename, p )
-            manifest.addFile (
-               'DIST',
-               package_filename,
-               # TODO:
-               # ignoreMissing for DIST files, else addFile raises
-               # FileNotFound -- is this the correct way to add DIST files?
-               ignoreMissing=True
-            )
-         else:
-            # ebuildmanifest adds all package files to the distdir without
-            # doing the check above (if hasFile())
-            # be "fully compatible" and do that here, too
-            try:
-               distdir.add ( p ['package_file'], package_filename, p )
-            except Exception as err:
-               self.logger.exception ( err )
-
-      # ??? FIXME: (read) portage api docs
-      #
-      # manifest.create -- recreate from scratch (required?)
-      # -> assumeDistHashesSometimes:
-      #     use existing checksums for non-existent DIST files (required?)
-      #
-      #manifest.create ( assumeDistHashesSometimes=True )
-      #manifest.create ( assumeDistHashesSometimes=False )
-
-      manifest.write()
-      return True
-   # --- end of write_manifest (...) ---

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/

[André Erdmann]

commit:     d02a12fa7114abeb9cb53afc390f81d9e6657d03
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 15:12:12 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 15:12:12 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d02a12fa

use newmanifest by default

---
 roverlay/overlay/pkgdir/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roverlay/overlay/pkgdir/__init__.py b/roverlay/overlay/pkgdir/__init__.py
index fd49431..90dfb6e 100644
--- a/roverlay/overlay/pkgdir/__init__.py
+++ b/roverlay/overlay/pkgdir/__init__.py
@@ -21,7 +21,7 @@ _package_dir_class  = None
 
 _PACKAGE_DIR_IMPLEMENTATIONS = {
    'none'    : 'packagedir_base',
-   'default' : 'packagedir_ebuildmanifest',
+   'default' : 'packagedir_newmanifest',
    'e'       : 'packagedir_ebuildmanifest',
    'ebuild'  : 'packagedir_ebuildmanifest',
    'next'    : 'packagedir_newmanifest',

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/

[André Erdmann]

commit:     c598a34408dbfcc45938458ddd3317644a79f8d4
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 16:08:25 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 16:08:25 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=c598a344

roverlay/overlay/pkgdir: create all hashes at once

---
 roverlay/overlay/pkgdir/packagedir_base.py           | 4 ----
 roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py | 1 +
 roverlay/overlay/pkgdir/packagedir_newmanifest.py    | 2 ++
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 53ebc42..ad00a05 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -819,10 +819,6 @@ class PackageDirBase ( object ):
             or patch_ebuild ( efile, pvr, patchview.get_patches ( pvr ) )
          ):
 
-            # generate hashes here (benefit from threading)
-            # FIXME/TODO: ^ actually faster?
-            p_info.make_distmap_hash()
-
             self._need_manifest = True
 
             # update metadata for each successfully written ebuild

diff --git a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
index 9766606..71cfd23 100644
--- a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
@@ -102,6 +102,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
          # TODO: optimize this further?
          # -> "not has physical_only?"
          #     (should be covered by "has package_file")
+         p_info.make_distmap_hash()
          distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
       # -- end for;
 

diff --git a/roverlay/overlay/pkgdir/packagedir_newmanifest.py b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
index 04d202d..ae6c059 100644
--- a/roverlay/overlay/pkgdir/packagedir_newmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
@@ -89,6 +89,8 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
       #
       distdir = self.DISTROOT.get_distdir ( self.name )
       for p in pkgs_for_manifest:
+         # order is important here
+         # add_package_entry() calls multihash with all required digests
          manifest.add_package_entry ( p )
          distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
       # -- end for;