[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     fc3eeb0f06119515f8db68311183172923423ded
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun  5 18:02:09 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun  5 18:02:09 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=fc3eeb0f

overlay: fix dynpool rules, remove moved ebuilds

---
 roverlay/overlay/category.py | 44 +++++++++++++++++++-----
 roverlay/overlay/creator.py  |  6 ++++
 roverlay/overlay/root.py     | 82 ++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 120 insertions(+), 12 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 2d9b93d..baca2d1 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -90,6 +90,12 @@ class Category ( object ):
       )
    # --- end of add (...) ---
 
+   def drop_package ( self, name ):
+      p = self._subdirs [name]
+      del self._subdirs [name]
+      p.fs_destroy()
+   # --- end of drop_package (...) ---
+
    def empty ( self ):
       """Returns True if this category contains 0 ebuilds."""
       return (
@@ -98,6 +104,11 @@ class Category ( object ):
       )
    # --- end of empty (...) ---
 
+   def get_nonempty ( self, name ):
+      subdir = self._subdirs.get ( name, None )
+      return subdir if ( subdir and not subdir.empty() ) else None
+   # --- end of has_nonempty (...) ---
+
    def has ( self, subdir ):
       return subdir in self._subdirs
    # --- end of has (...) ---
@@ -106,22 +117,37 @@ class Category ( object ):
       return os.path.isdir ( self.physical_location + os.sep + _dir )
    # --- end of has_category (...) ---
 
-   def list_packages ( self, for_deprules=False ):
+   def list_package_names ( self ):
+      for name, subdir in self._subdirs.items():
+         if not subdir.empty():
+            yield name
+   # --- end of list_package_names (...) ---
+
+   def list_packages ( self,
+      for_deprules=False, is_default_category=False
+   ):
       """Lists all packages in this category.
       Yields <category>/<package name> or a dict (see for_deprules below).
 
       arguments:
-      * for_deprules -- if set and True:
-                         yield keyword args for dependency rules
+      * for_deprules        -- if set and True:
+                                yield keyword args for dependency rules
+      * is_default_category -- bool indicating whether this category is the
+                               default one or not
       """
-
-      for name, subdir in self._subdirs.items():
-         if not subdir.empty():
-            if for_deprules:
+      if for_deprules:
+         for name, subdir in self._subdirs.items():
+            if not subdir.empty():
                yield dict (
-                  dep_str=name, resolving_package=self.name + os.sep + name
+                  dep_str           = name,
+                  resolving_package = ( self.name + "/" + name ),
+                  is_selfdep        = is_default_category,
+                  # prefer packages from the default category (really?)
+                  priority          = 80 if is_default_category else 90,
                )
-            else:
+      else:
+         for name, subdir in self._subdirs.items():
+            if not subdir.empty():
                yield self.name + os.sep + name
    # --- end of list_packages (...) ---
 

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 8833aef..5ad86d4 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -274,6 +274,12 @@ class OverlayCreator ( object ):
       del self.package_rules
    # --- end of release_package_rules (...) ---
 
+   def remove_moved_ebuilds ( self, reverse ):
+      """See overlay.root.Overlay.remove_moved_ebuilds()."""
+      if self.overlay.remove_duplicate_ebuilds ( reverse=reverse ):
+         self.depresolver.reload_pools()
+   # --- end of remove_moved_ebuilds (...) ---
+
    def add_package ( self, package_info ):
       """Adds a PackageInfo to the package queue.
 

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 3adbb6e..b9030d2 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -259,7 +259,9 @@ class Overlay ( object ):
          write_profiles_file ( 'repo_name', self.name + '\n' )
 
          # profiles/categories
-         cats = '\n'.join ( self._categories.keys() )
+         cats = '\n'.join (
+            k for k, v in self._categories.items() if not v.empty()
+         )
          if cats:
             write_profiles_file ( 'categories', cats + '\n' )
 
@@ -319,6 +321,78 @@ class Overlay ( object ):
       return os.path.isdir ( self.physical_location + os.sep + _dir )
    # --- end of has_category (...) ---
 
+   def find_duplicate_packages ( self, _default_category=None ):
+      default_category = (
+         _default_category if _default_category is None
+         else self._categories.get ( self.default_category, None )
+      )
+
+      if default_category:
+         duplicate_pkg  = set()
+
+         for category in self._categories.values():
+            if category is not default_category:
+               for name in category.list_package_names():
+                  if default_category.get_nonempty ( name ):
+                     duplicate_pkg.add ( name )
+         # -- end for;
+
+         return frozenset ( duplicate_pkg )
+      else:
+         return None
+   # --- end of find_duplicate_packages (...) ---
+
+   def remove_duplicate_ebuilds ( self, reverse ):
+      default_category = self._categories.get ( self.default_category, None )
+      if default_category:
+         if reverse:
+            d_pkg = self.find_duplicate_packages (
+               _default_category=default_category
+            )
+            for pkg_name in d_pkg:
+               default_category.drop_package ( pkg_name )
+
+         else:
+            d_pkg = set()
+            for category in self._categories.values():
+               if category is not default_category:
+                  for name in category.list_package_names():
+                     if default_category.get_nonempty ( name ):
+                        d_pkg.add ( name )
+                        category.drop_package ( pkg_name )
+            # -- end for category;
+
+         # -- end if;
+
+         if d_pkg:
+            self.remove_empty_categories()
+
+            self.logger.info (
+               '{} ebuilds have been removed from the default category, '
+               'the overlay might be broken now!'.format ( len ( d_pkg ) )
+            )
+            return True
+         else:
+            return False
+      else:
+         return False
+
+   # --- end of remove_duplicate_ebuilds (...) ---
+
+   def remove_empty_categories ( self ):
+      catlist = self._categories.items()
+      for cat in catlist:
+         cat[1].remove_empty()
+         if cat[1].empty():
+            del self._categories [cat[0]]
+
+            try:
+               os.rmdir ( cat [1].physical_location )
+            except OSError as ose:
+               self.logger.exception ( ose )
+
+   # --- end of remove_empty_categories (...) ---
+
    def list_packages ( self, for_deprules=True ):
       for cat in self._categories.values():
          for package in cat.list_packages ( for_deprules=for_deprules ):
@@ -326,9 +400,11 @@ class Overlay ( object ):
    # --- end of list_packages (...) ---
 
    def list_rule_kwargs ( self ):
-      # FIXME/TODO: depres has to recognize categories
       for cat in self._categories.values():
-         for kwargs in cat.list_packages ( for_deprules=True ):
+         for kwargs in cat.list_packages (
+            for_deprules        = True,
+            is_default_category = ( cat.name is self.default_category )
+         ):
             yield kwargs
    # --- end of list_rule_kwargs (...) ---
 

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/

[André Erdmann]

commit:     fc3eeb0f06119515f8db68311183172923423ded
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun  5 18:02:09 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun  5 18:02:09 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=fc3eeb0f

overlay: fix dynpool rules, remove moved ebuilds

---
 roverlay/overlay/category.py | 44 +++++++++++++++++++-----
 roverlay/overlay/creator.py  |  6 ++++
 roverlay/overlay/root.py     | 82 ++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 120 insertions(+), 12 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 2d9b93d..baca2d1 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -90,6 +90,12 @@ class Category ( object ):
       )
    # --- end of add (...) ---
 
+   def drop_package ( self, name ):
+      p = self._subdirs [name]
+      del self._subdirs [name]
+      p.fs_destroy()
+   # --- end of drop_package (...) ---
+
    def empty ( self ):
       """Returns True if this category contains 0 ebuilds."""
       return (
@@ -98,6 +104,11 @@ class Category ( object ):
       )
    # --- end of empty (...) ---
 
+   def get_nonempty ( self, name ):
+      subdir = self._subdirs.get ( name, None )
+      return subdir if ( subdir and not subdir.empty() ) else None
+   # --- end of has_nonempty (...) ---
+
    def has ( self, subdir ):
       return subdir in self._subdirs
    # --- end of has (...) ---
@@ -106,22 +117,37 @@ class Category ( object ):
       return os.path.isdir ( self.physical_location + os.sep + _dir )
    # --- end of has_category (...) ---
 
-   def list_packages ( self, for_deprules=False ):
+   def list_package_names ( self ):
+      for name, subdir in self._subdirs.items():
+         if not subdir.empty():
+            yield name
+   # --- end of list_package_names (...) ---
+
+   def list_packages ( self,
+      for_deprules=False, is_default_category=False
+   ):
       """Lists all packages in this category.
       Yields <category>/<package name> or a dict (see for_deprules below).
 
       arguments:
-      * for_deprules -- if set and True:
-                         yield keyword args for dependency rules
+      * for_deprules        -- if set and True:
+                                yield keyword args for dependency rules
+      * is_default_category -- bool indicating whether this category is the
+                               default one or not
       """
-
-      for name, subdir in self._subdirs.items():
-         if not subdir.empty():
-            if for_deprules:
+      if for_deprules:
+         for name, subdir in self._subdirs.items():
+            if not subdir.empty():
                yield dict (
-                  dep_str=name, resolving_package=self.name + os.sep + name
+                  dep_str           = name,
+                  resolving_package = ( self.name + "/" + name ),
+                  is_selfdep        = is_default_category,
+                  # prefer packages from the default category (really?)
+                  priority          = 80 if is_default_category else 90,
                )
-            else:
+      else:
+         for name, subdir in self._subdirs.items():
+            if not subdir.empty():
                yield self.name + os.sep + name
    # --- end of list_packages (...) ---
 

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 8833aef..5ad86d4 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -274,6 +274,12 @@ class OverlayCreator ( object ):
       del self.package_rules
    # --- end of release_package_rules (...) ---
 
+   def remove_moved_ebuilds ( self, reverse ):
+      """See overlay.root.Overlay.remove_moved_ebuilds()."""
+      if self.overlay.remove_duplicate_ebuilds ( reverse=reverse ):
+         self.depresolver.reload_pools()
+   # --- end of remove_moved_ebuilds (...) ---
+
    def add_package ( self, package_info ):
       """Adds a PackageInfo to the package queue.
 

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 3adbb6e..b9030d2 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -259,7 +259,9 @@ class Overlay ( object ):
          write_profiles_file ( 'repo_name', self.name + '\n' )
 
          # profiles/categories
-         cats = '\n'.join ( self._categories.keys() )
+         cats = '\n'.join (
+            k for k, v in self._categories.items() if not v.empty()
+         )
          if cats:
             write_profiles_file ( 'categories', cats + '\n' )
 
@@ -319,6 +321,78 @@ class Overlay ( object ):
       return os.path.isdir ( self.physical_location + os.sep + _dir )
    # --- end of has_category (...) ---
 
+   def find_duplicate_packages ( self, _default_category=None ):
+      default_category = (
+         _default_category if _default_category is None
+         else self._categories.get ( self.default_category, None )
+      )
+
+      if default_category:
+         duplicate_pkg  = set()
+
+         for category in self._categories.values():
+            if category is not default_category:
+               for name in category.list_package_names():
+                  if default_category.get_nonempty ( name ):
+                     duplicate_pkg.add ( name )
+         # -- end for;
+
+         return frozenset ( duplicate_pkg )
+      else:
+         return None
+   # --- end of find_duplicate_packages (...) ---
+
+   def remove_duplicate_ebuilds ( self, reverse ):
+      default_category = self._categories.get ( self.default_category, None )
+      if default_category:
+         if reverse:
+            d_pkg = self.find_duplicate_packages (
+               _default_category=default_category
+            )
+            for pkg_name in d_pkg:
+               default_category.drop_package ( pkg_name )
+
+         else:
+            d_pkg = set()
+            for category in self._categories.values():
+               if category is not default_category:
+                  for name in category.list_package_names():
+                     if default_category.get_nonempty ( name ):
+                        d_pkg.add ( name )
+                        category.drop_package ( pkg_name )
+            # -- end for category;
+
+         # -- end if;
+
+         if d_pkg:
+            self.remove_empty_categories()
+
+            self.logger.info (
+               '{} ebuilds have been removed from the default category, '
+               'the overlay might be broken now!'.format ( len ( d_pkg ) )
+            )
+            return True
+         else:
+            return False
+      else:
+         return False
+
+   # --- end of remove_duplicate_ebuilds (...) ---
+
+   def remove_empty_categories ( self ):
+      catlist = self._categories.items()
+      for cat in catlist:
+         cat[1].remove_empty()
+         if cat[1].empty():
+            del self._categories [cat[0]]
+
+            try:
+               os.rmdir ( cat [1].physical_location )
+            except OSError as ose:
+               self.logger.exception ( ose )
+
+   # --- end of remove_empty_categories (...) ---
+
    def list_packages ( self, for_deprules=True ):
       for cat in self._categories.values():
          for package in cat.list_packages ( for_deprules=for_deprules ):
@@ -326,9 +400,11 @@ class Overlay ( object ):
    # --- end of list_packages (...) ---
 
    def list_rule_kwargs ( self ):
-      # FIXME/TODO: depres has to recognize categories
       for cat in self._categories.values():
-         for kwargs in cat.list_packages ( for_deprules=True ):
+         for kwargs in cat.list_packages (
+            for_deprules        = True,
+            is_default_category = ( cat.name is self.default_category )
+         ):
             yield kwargs
    # --- end of list_rule_kwargs (...) ---
 

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     8c67544ba3552ae6a67a10145036d42d8692d10c
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 29 18:22:04 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 29 18:22:04 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=8c67544b

roverlay/overlay/rulegen: fix dot handling

for scanned ebuilds: replace underscore chars '_' with dot chars '.' and the
resulting str as package name alias to the dyn. dep rule.

---
 roverlay/overlay/rulegen.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
index b2bf1e5..dcc4723 100644
--- a/roverlay/overlay/rulegen.py
+++ b/roverlay/overlay/rulegen.py
@@ -40,6 +40,8 @@ class DepresRuleGenerator ( object ):
                   )
                   if package_name:
                      package_names.add ( package_name )
+                  else:
+                     package_names.add ( pkgdir_name.replace('_','.') )
 
                   repo = p_info.get ( 'origin', do_fallback=True )
                   if repo is not None:

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     8c67544ba3552ae6a67a10145036d42d8692d10c
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 29 18:22:04 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 29 18:22:04 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=8c67544b

roverlay/overlay/rulegen: fix dot handling

for scanned ebuilds: replace underscore chars '_' with dot chars '.' and the
resulting str as package name alias to the dyn. dep rule.

---
 roverlay/overlay/rulegen.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
index b2bf1e5..dcc4723 100644
--- a/roverlay/overlay/rulegen.py
+++ b/roverlay/overlay/rulegen.py
@@ -40,6 +40,8 @@ class DepresRuleGenerator ( object ):
                   )
                   if package_name:
                      package_names.add ( package_name )
+                  else:
+                     package_names.add ( pkgdir_name.replace('_','.') )
 
                   repo = p_info.get ( 'origin', do_fallback=True )
                   if repo is not None:

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     093b182951ad7ab6eb7e519061c7b95dc218aacb
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 17 19:14:21 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 17 19:14:21 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=093b1829

overlay/abccontrol: convert_str()

---
 roverlay/overlay/abccontrol.py | 42 +++++++++++++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/roverlay/overlay/abccontrol.py b/roverlay/overlay/abccontrol.py
index 364a083..d031eec 100644
--- a/roverlay/overlay/abccontrol.py
+++ b/roverlay/overlay/abccontrol.py
@@ -82,17 +82,37 @@ class AdditionControlResult ( object ):
    ) = _gen_bits(5)
 
 
-#   PKG_DESCRIPTION_MAP      = {
-#      PKG_FORCE_DENY           : 'force-deny',
-#      PKG_DENY_REPLACE         : 'deny-replace',
-#      PKG_FORCE_REPLACE        : 'force-replace',
-#      PKG_REPLACE_ONLY         : 'replace-only',
-#      PKG_REVBUMP_ON_COLLISION : 'revbump-on-collision',
-#      PKG_DEFAULT_BEHAVIOR     : 'default',
-#   }
-#
-#   PKG_DESCRIPTION_REVMAP   = { v: k for k,v in PKG_DESCRIPTION_MAP.items() }
-#
+   PKG_DESCRIPTION_MAP      = {
+      PKG_FORCE_DENY           : 'force-deny',
+      PKG_DENY_REPLACE         : 'deny-replace',
+      PKG_FORCE_REPLACE        : 'force-replace',
+      PKG_REPLACE_ONLY         : 'replace-only',
+      PKG_REVBUMP_ON_COLLISION : 'revbump-on-collision',
+      PKG_DEFAULT_BEHAVIOR     : 'default',
+   }
+
+   PKG_DESCRIPTION_REVMAP   = { v: k for k,v in PKG_DESCRIPTION_MAP.items() }
+
+   @classmethod
+   def convert_str ( cls, s ):
+      desc_rmap = cls.PKG_DESCRIPTION_REVMAP
+
+      if not s:
+         raise ValueError ( "str must not be empty." )
+
+      bitmask = 0
+
+      for arg in filter ( None, s.strip().lower().split(",") ):
+         # ^ strip(),lower(),split() ...
+         print(arg)
+         try:
+            bitmask |= desc_rmap [arg]
+         except KeyError:
+            raise ValueError ( arg )
+      # -- end for
+
+      return bitmask
+   # --- end of convert_str (...) ---
 
    @classmethod
    def get_reversed_sort_key ( cls, k ):

[gentoo-commits] proj/R_overlay:wip/addition_control commit in: roverlay/overlay/

[André Erdmann]

commit:     9c4859591aa5d5388c192f932f3e4d8a1344e57d
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 18 02:23:31 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 18 02:23:31 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9c485959

roverlay/overlay/abccontrol: comment

---
 roverlay/overlay/abccontrol.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/roverlay/overlay/abccontrol.py b/roverlay/overlay/abccontrol.py
index a319e35..991343e 100644
--- a/roverlay/overlay/abccontrol.py
+++ b/roverlay/overlay/abccontrol.py
@@ -81,6 +81,10 @@ class AdditionControlResult ( object ):
       PKG_ALL,
    ) = _gen_bits(5)
 
+##   # virtuals
+##   PKG_REVBUMP_ONLY       = PKG_REVBUMP_ON_COLLISION|PKG_REPLACE_ONLY
+##   PKG_FORCE_REPLACE_ONLY = PKG_FORCE_REPLACE|PKG_REPLACE_ONLY
+
 
    PKG_DESCRIPTION_MAP      = {
       PKG_FORCE_DENY           : 'force-deny',
@@ -89,6 +93,10 @@ class AdditionControlResult ( object ):
       PKG_REPLACE_ONLY         : 'replace-only',
       PKG_REVBUMP_ON_COLLISION : 'revbump-on-collision',
       PKG_DEFAULT_BEHAVIOR     : 'default',
+
+##      # virtuals
+##      PKG_REVBUMP_ONLY         : 'revbump-on-collision,replace-only',
+##      PKG_FORCE_REPLACE_ONLY   : 'force-replace,replace-only',
    }
 
    PKG_DESCRIPTION_REVMAP   = { v: k for k,v in PKG_DESCRIPTION_MAP.items() }

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     9c4859591aa5d5388c192f932f3e4d8a1344e57d
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 18 02:23:31 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 18 02:23:31 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9c485959

roverlay/overlay/abccontrol: comment

---
 roverlay/overlay/abccontrol.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/roverlay/overlay/abccontrol.py b/roverlay/overlay/abccontrol.py
index a319e35..991343e 100644
--- a/roverlay/overlay/abccontrol.py
+++ b/roverlay/overlay/abccontrol.py
@@ -81,6 +81,10 @@ class AdditionControlResult ( object ):
       PKG_ALL,
    ) = _gen_bits(5)
 
+##   # virtuals
+##   PKG_REVBUMP_ONLY       = PKG_REVBUMP_ON_COLLISION|PKG_REPLACE_ONLY
+##   PKG_FORCE_REPLACE_ONLY = PKG_FORCE_REPLACE|PKG_REPLACE_ONLY
+
 
    PKG_DESCRIPTION_MAP      = {
       PKG_FORCE_DENY           : 'force-deny',
@@ -89,6 +93,10 @@ class AdditionControlResult ( object ):
       PKG_REPLACE_ONLY         : 'replace-only',
       PKG_REVBUMP_ON_COLLISION : 'revbump-on-collision',
       PKG_DEFAULT_BEHAVIOR     : 'default',
+
+##      # virtuals
+##      PKG_REVBUMP_ONLY         : 'revbump-on-collision,replace-only',
+##      PKG_FORCE_REPLACE_ONLY   : 'force-replace,replace-only',
    }
 
    PKG_DESCRIPTION_REVMAP   = { v: k for k,v in PKG_DESCRIPTION_MAP.items() }

[gentoo-commits] proj/R_overlay:wip/addition_control commit in: roverlay/overlay/

[André Erdmann]

commit:     13ae9d788d175488eaa5b4ba2637ed04fb6423bb
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 18 01:00:42 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 18 01:00:42 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=13ae9d78

roverlay/overlay/abccontrol: remove dbg statement

---
 roverlay/overlay/abccontrol.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/roverlay/overlay/abccontrol.py b/roverlay/overlay/abccontrol.py
index d031eec..a319e35 100644
--- a/roverlay/overlay/abccontrol.py
+++ b/roverlay/overlay/abccontrol.py
@@ -104,7 +104,6 @@ class AdditionControlResult ( object ):
 
       for arg in filter ( None, s.strip().lower().split(",") ):
          # ^ strip(),lower(),split() ...
-         print(arg)
          try:
             bitmask |= desc_rmap [arg]
          except KeyError:

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     13ae9d788d175488eaa5b4ba2637ed04fb6423bb
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 18 01:00:42 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 18 01:00:42 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=13ae9d78

roverlay/overlay/abccontrol: remove dbg statement

---
 roverlay/overlay/abccontrol.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/roverlay/overlay/abccontrol.py b/roverlay/overlay/abccontrol.py
index d031eec..a319e35 100644
--- a/roverlay/overlay/abccontrol.py
+++ b/roverlay/overlay/abccontrol.py
@@ -104,7 +104,6 @@ class AdditionControlResult ( object ):
 
       for arg in filter ( None, s.strip().lower().split(",") ):
          # ^ strip(),lower(),split() ...
-         print(arg)
          try:
             bitmask |= desc_rmap [arg]
          except KeyError:

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     6596fb255d1f17017420401a4e39f8a711bcc239
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Mar 31 15:38:36 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Mar 31 15:38:36 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6596fb25

overlay/creator: fix comment

---
 roverlay/overlay/creator.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index ea623e6..75119d3 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -404,7 +404,6 @@ class OverlayCreator ( object ):
       ##        num_removed += selfdep.reduce()
       ##    end loop
       ##
-      ##    num_removed <- 0
       ## end loop
       ##
       self.overlay.link_selfdeps ( selfdeps )

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     66b7738ec3d88f0766e785cdf890d71597cd5ba0
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Sep 18 13:56:53 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Sep 18 13:56:53 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=66b7738e

fix 4502c63561b4e6390ecc7457d11c4fc1c15e2829

fix a typo causing _write_rsuggests_use_desc() to fail

---
 roverlay/overlay/root.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 0a3b15e..34a15ee 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -385,8 +385,8 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
 
             self._write_rsuggests_use_desc (
                desc_file       = os.sep.join ([
-                  PROFILES_DIR, 'desc', use_expand_name.lower(), '.desc'
-               ]),
+                  PROFILES_DIR, 'desc', use_expand_name.lower(),
+               ]) + '.desc',
                use_expand_name = use_expand_name.upper(),
                backup_file     = CONFIG_GET ( 'OVERLAY.backup_desc', True ),
                flagdesc_file   = CONFIG_GET (

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     4502c63561b4e6390ecc7457d11c4fc1c15e2829
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Sep 16 14:56:43 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Sep 16 14:56:43 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=4502c635

initialize overlay before importing ebuilds

from import_ebuilds()'s comment:
 do minimal overlay initialization before importing ebuilds.
 This is recommended because portage expects metadata/layout.conf to exist,
 for example.

---
 roverlay/overlay/root.py | 227 +++++++++++++++++++++++++----------------------
 1 file changed, 123 insertions(+), 104 deletions(-)

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index b3be5b3..0a3b15e 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -135,7 +135,6 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
       self._runtime_incremental = write_allowed and runtime_incremental
       self._writeable           = write_allowed
 
-      self._profiles_dir        = self.physical_location + os.sep + 'profiles'
       self._catlock             = threading.Lock()
       self._categories          = dict()
 
@@ -148,8 +147,8 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
          ebuild_header, eapi
       )
       self._use_desc = (
-         use_desc.rstrip() if use_desc is not None else self.DEFAULT_USE_DESC
-      )
+         use_desc if use_desc is not None else self.DEFAULT_USE_DESC
+      ).rstrip()
 
       if keep_n_ebuilds:
          self.keep_n_ebuilds = keep_n_ebuilds
@@ -271,133 +270,143 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
             raise
    # --- end of _import_eclass (...) ---
 
-   def _init_overlay ( self, reimport_eclass ):
+   def _generate_layout_conf ( self ):
+      # create layout.conf file
+      # * create lines
+
+      kv_join    = lambda k, v: "{k} = {v}".format ( k=k, v=v )
+      v_join     = lambda v: ' '.join ( v )
+
+      yield kv_join ( "repo_name", self.name )
+
+      if self._masters:
+         yield kv_join ( "masters", v_join ( self._masters ) )
+      else:
+         yield "masters ="
+
+
+      # strictly speaking,
+      #  declaring cache-formats here is not correct since egencache
+      #  is run as hook
+      yield kv_join ( "cache-formats", "md5-dict" )
+
+      ##yield kv_join ( "sign-commits", "false" )
+      ##yield kv_join ( "sign-manifests", "false" )
+      ##yield kv_join ( "thin-manifests", "false" )
+
+      hashes = roverlay.overlay.pkgdir.base.get_class().HASH_TYPES
+      if hashes:
+         yield kv_join ( "manifest-hashes", v_join ( hashes ) )
+   # --- end of _generate_layout_conf (...) ---
+
+   def _init_overlay ( self, reimport_eclass, minimal=False ):
       """Initializes the overlay at its physical/filesystem location.
 
       arguments:
-      * reimport_eclass   -- whether to copy existing eclass files
-                               again (True) or not
-      * make_profiles_dir -- if True: create the profiles/ dir now
+      * reimport_eclass -- whether to copy existing eclass files
+                            again (True) or not
+      * minimal         -- whether to do full overlay initialization (False)
+                           or not (True). Defaults to False.
 
       raises:
-      * IOError
+      * IOError, OSError -- passed from file writing / dir creation
       """
-      NEWLINE   = '\n'
-      EMPTY_STR = ""
+      NEWLINE            = '\n'
+      CONFIG_GET         = roverlay.config.get
+      CONFIG_GET_OR_FAIL = roverlay.config.get_or_fail
+      ROOT               = str ( self.physical_location )
+      METADATA_DIR       = ROOT + os.sep + 'metadata'
+      PROFILES_DIR       = ROOT + os.sep + 'profiles'
 
 
-      def write_profiles_dir():
-         """Creates and updates the profiles/ dir."""
-         def write_profiles_file ( filename, to_write ):
-            """Writes a file in profiles/.
+      def get_subdir_file_write ( root_dir ):
+         """Returns a function for writing files in the given directory.
+
+         arguments:
+         * root_dir --
+         """
+         def write_subdir_file ( relpath, content, append_newline=True ):
+            """Writes a file (<root_dir>/<relpath>).
 
             arguments:
-            * filename -- name of the file to write (including file extension)
-            * to_write -- string to write (don't forget newline at the end)
+            * relpath        -- file path relative to root_dir
+            * content        -- data to write
+            * append_newline -- whether to append a newline at EOF
             """
-            fh = None
-            try:
-               fh = open ( self._profiles_dir + os.sep + filename, 'w' )
-               if to_write:
-                  # else touch file
-                  fh.write ( to_write )
-            except IOError as e:
-               self.logger.exception ( e )
-               raise
-            finally:
-               if fh: fh.close()
-         # --- end of write_profiles_file (...) ---
-
-         # always use the default category (+write it into profiles/categories)
-         self._get_category ( self.default_category )
+            content_str = str ( content )
+            if content_str:
+               with open ( root_dir + os.sep + relpath, 'wt' ) as FH:
+                  FH.write ( content_str )
+                  if append_newline:
+                     FH.write ( NEWLINE )
+            else:
+               # touch file
+               with open ( root_dir + os.sep + relpath, 'a' ) as FH:
+                  pass
+         # --- end of write_subdir_file (...) ---
+         return write_subdir_file
+      # --- end of get_subdir_file_write (...) ---
 
-         # profiles/
-         roverlay.util.dodir ( self._profiles_dir )
 
-         # profiles/repo_name
-         write_profiles_file ( 'repo_name', self.name + '\n' )
-
-         # profiles/categories
-         cats = '\n'.join (
-            k for k, v in self._categories.items() if not v.empty()
-         )
-         if cats:
-            write_profiles_file ( 'categories', cats + '\n' )
-
-         # profiles/desc/<r_suggests>.desc
-         use_expand_name = roverlay.config.get_or_fail (
-            "EBUILD.USE_EXPAND.name"
-         ).rstrip ( "_" )
-
-         self._write_rsuggests_use_desc (
-            desc_file = (
-               self._profiles_dir + os.sep + 'desc' + os.sep
-               + use_expand_name.lower() + '.desc'
-            ),
-            use_expand_name = use_expand_name.upper(),
-            backup_file = roverlay.config.get ( 'OVERLAY.backup_desc', True ),
-            flagdesc_file = roverlay.config.get (
-               'EBUILD.USE_EXPAND.desc_file', None
-            ),
-         )
+      write_profiles_file = get_subdir_file_write ( PROFILES_DIR )
+      write_metadata_file = get_subdir_file_write ( METADATA_DIR )
 
+      try:
+         layout_conf_str = NEWLINE.join ( self._generate_layout_conf() )
 
-         # profiles/use.desc
-         if self._use_desc:
-            write_profiles_file ( 'use.desc', self._use_desc + '\n' )
-      # --- end of write_profiles_dir (...) ---
+         ## make overlay dirs, root, metadata/, profiles/
+         roverlay.util.dodir ( ROOT, mkdir_p=True )
+         roverlay.util.dodir ( PROFILES_DIR )
+         roverlay.util.dodir ( METADATA_DIR )
 
-      def write_metadata_dir():
 
-         METADATA_DIR = self.physical_location + os.sep + 'metadata'
-         roverlay.util.dodir ( METADATA_DIR )
+         ## import eclass files
+         self._import_eclass ( reimport_eclass )
 
-         # create layout.conf file
-         # * create lines
-         layout_lines = list()
-         layout_add = layout_lines.append
-         kv_join    = lambda k, v: "{k} = {v}".format ( k=k, v=v )
-         v_join     = lambda v: ' '.join ( v )
 
-         layout_add ( kv_join ( "repo_name", self.name ) )
+         ## populate profiles/
 
-         if self._masters:
-            layout_add ( kv_join ( "masters", v_join ( self._masters ) ) )
-         else:
-            layout_add ( "masters =" )
+         # profiles/repo_name
+         write_profiles_file ( 'repo_name', self.name )
 
+         # profiles/categories
+         cats = NEWLINE.join (
+            k for k, v in self._categories.items() if not v.empty()
+         )
+         if cats:
+            write_profiles_file ( 'categories', cats )
 
-         # strictly speaking,
-         #  declaring cache-formats here is not correct since egencache
-         #  is run as hook
-         layout_add ( kv_join ( "cache-formats", "md5-dict" ) )
 
-         ##layout_add ( kv_join ( "sign-commits", "false" ) )
-         ##layout_add ( kv_join ( "sign-manifests", "false" ) )
-         ##layout_add ( kv_join ( "thin-manifests", "false" ) )
+         if not minimal:
+            # profiles/desc/<r_suggests>.desc
+            use_expand_name = (
+               CONFIG_GET_OR_FAIL ( "EBUILD.USE_EXPAND.name" ).rstrip ( "_" )
+            )
 
-         hashes = roverlay.overlay.pkgdir.base.get_class().HASH_TYPES
-         if hashes:
-            layout_add ( kv_join ( "manifest-hashes", v_join ( hashes ) ) )
+            self._write_rsuggests_use_desc (
+               desc_file       = os.sep.join ([
+                  PROFILES_DIR, 'desc', use_expand_name.lower(), '.desc'
+               ]),
+               use_expand_name = use_expand_name.upper(),
+               backup_file     = CONFIG_GET ( 'OVERLAY.backup_desc', True ),
+               flagdesc_file   = CONFIG_GET (
+                  'EBUILD.USE_EXPAND.desc_file', None
+               ),
+            )
 
-         # * write it
-         with open ( METADATA_DIR + os.sep + 'layout.conf', 'wt' ) as FH:
-            for line in layout_lines:
-               FH.write ( line )
-               FH.write ( NEWLINE )
-      # --- end of write_metadata_dir (...) ---
 
-      try:
-         # mkdir overlay root
-         roverlay.util.dodir ( self.physical_location, mkdir_p=True )
+            # profiles/use.desc
+            if self._use_desc:
+               write_profiles_file ( 'use.desc', self._use_desc )
 
-         self._import_eclass ( reimport_eclass )
+         # -- end if not minimal ~ profiles/
 
-         write_profiles_dir()
-         write_metadata_dir()
+         ## metadata/
 
-      except IOError as e:
+         # metadata/layout.conf
+         write_metadata_file ( 'layout.conf', layout_conf_str )
 
+      except ( OSError, IOError ) as e:
          self.logger.exception ( e )
          self.logger.critical ( "failed to init overlay" )
          raise
@@ -710,13 +719,21 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
       return not self._writeable
    # --- end of readonly (...) ---
 
-   def import_ebuilds ( self, overwrite, nosync=False ):
+   def import_ebuilds ( self, overwrite, nosync=False, init_overlay=True ):
       """Imports ebuilds from the additions dir.
 
       arguments:
-      * overwrite -- whether to overwrite existing ebuilds
-      * nosync    -- if True: don't fetch src files (defaults to False)
+      * overwrite    -- whether to overwrite existing ebuilds
+      * nosync       -- if True: don't fetch src files (defaults to False)
+      * init_overlay -- if True: do minimal overlay initialization before
+                                 importing ebuilds. This is recommended
+                                 because portage expects metadata/layout.conf
+                                 to exist, for example.
+                                 Defaults to True.
       """
+      if init_overlay:
+         self._init_overlay ( False, minimal=True )
+
       for catview in (
          roverlay.overlay.additionsdir.CategoryRootView ( self.additions_dir )
       ):
@@ -887,7 +904,9 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
       if self._writeable and not self.skip_manifest:
          # profiles/categories is required for successful Manifest
          # creation
-         if os.path.isfile ( self._profiles_dir + os.sep + 'categories' ):
+         if os.path.isfile ( os.path.join (
+            str ( self.physical_location ), 'profiles', 'categories'
+         ) ):
             for cat in self._categories.values():
                cat.write_manifest ( **manifest_kw )
          else:

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     e6f457f95052e4de5ad349db4b5331956115c7dc
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep  3 15:44:01 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep  3 15:44:01 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=e6f457f9

remove TODO note

---
 roverlay/overlay/rulegen.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
index 42b5d09..b2bf1e5 100644
--- a/roverlay/overlay/rulegen.py
+++ b/roverlay/overlay/rulegen.py
@@ -87,7 +87,6 @@ class DepresRuleGenerator ( object ):
          else:
             rules_without_repo.append ( rule )
 
-      # TODO: use distmap to restore repo ids
       assert '_' not in rule_dict
       rule_dict ['_'] = rules_without_repo
 

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     6dafe81ba380a2c7c01ca048bc7245996692fefa
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep  3 13:14:06 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep  3 13:14:49 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6dafe81b

roverlay/overlay/root: remove get_depres_rule_generator()

no longer used

---
 roverlay/overlay/root.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 80bf836..c6b205e 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -32,7 +32,6 @@ import roverlay.overlay.category
 import roverlay.overlay.header
 import roverlay.overlay.pkgdir.base
 import roverlay.overlay.pkgdir.distroot.static
-import roverlay.overlay.rulegen
 
 
 class Overlay ( roverlay.overlay.base.OverlayObject ):
@@ -703,10 +702,6 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
             yield package
    # --- end of list_packages (...) ---
 
-   def get_depres_rule_generator ( self ):
-      return roverlay.overlay.rulegen.DepresRuleGenerator ( self )
-   # --- end of get_depres_rule_generator (...) ---
-
    def readonly ( self ):
       return not self._writeable
    # --- end of readonly (...) ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     f874355dd601abd680b0a4978fb1bf6b11259099
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep  3 08:32:57 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep  3 08:32:57 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f874355d

depres rule generator: sort rules

+ don't ignore rules without repo ids

---
 roverlay/overlay/rulegen.py | 46 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 41 insertions(+), 5 deletions(-)

diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
index 7610e9a..ea15b72 100644
--- a/roverlay/overlay/rulegen.py
+++ b/roverlay/overlay/rulegen.py
@@ -4,6 +4,8 @@
 # Distributed under the terms of the GNU General Public License;
 # either version 2 of the License, or (at your option) any later version.
 
+import collections
+
 class DepresRuleGenerator ( object ):
 
    def __init__ ( self, overlay ):
@@ -58,14 +60,48 @@ class DepresRuleGenerator ( object ):
 
    def make_rule_dict ( self ):
       rule_dict = dict()
+      rules_without_repo = list()
+
       for repo_ids, rule_kwargs in self.make_rule_args():
          rule = self.rule_class ( **rule_kwargs )
-         for repo_id in repo_ids:
-            if repo_id in rule_dict:
-               rule_dict [repo_id].append ( rule )
-            else:
-               rule_dict [repo_id] = [ rule ]
+         if repo_ids:
+            for repo_id in repo_ids:
+               if repo_id in rule_dict:
+                  rule_dict [repo_id].append ( rule )
+               else:
+                  rule_dict [repo_id] = [ rule ]
+         else:
+            rules_without_repo.append ( rule )
+
+      # TODO: use distmap to restore repo ids
+      assert '_' not in rule_dict
+      rule_dict ['_'] = rules_without_repo
+
       return rule_dict
    # --- end of make_rule_dict (...) ---
 
+   def make_rule_list ( self, do_sort=False ):
+      if do_sort:
+         rule_dict = self.make_rule_dict()
+         for rules in rule_dict.values():
+            rules.sort ( key=( lambda k: k.priority ) )
+
+         undef_key = -1
+         #undef_key = max ( k for k in rule_dict if k != '_' ) + 1
+
+         return sorted (
+            rule_dict.items(),
+            key=lambda kv: ( undef_key if kv[0] == '_' else kv[0] )
+         )
+      else:
+         return list ( self.make_rule_dict().items() )
+   # --- end of make_rule_list (...) ---
+
+   def make_ordered_rule_dict ( self ):
+      # _not_ efficient:
+      #  build a dict -> build a list -> build a dict
+      #
+      return collections.OrderedDict ( self.make_rule_list ( do_sort=True ) )
+   # --- end of make_ordered_rule_dict (...) ---
+
 # --- end of DepresRuleGenerator ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     591df99455bd4f4f907e7d96bdca4aedef5e2258
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Sep  2 16:18:29 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Sep  2 16:18:29 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=591df994

roverlay/overlay: use new depres rule generator

---
 roverlay/overlay/category.py | 31 +++++++------------------------
 roverlay/overlay/creator.py  |  4 +++-
 roverlay/overlay/root.py     | 13 +++++--------
 3 files changed, 15 insertions(+), 33 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index d2b2028..4301734 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -198,15 +198,7 @@ class Category ( roverlay.overlay.base.OverlayObject ):
             yield p_info
    # --- end of iter_package_info (...) ---
 
-   def list_package_names ( self ):
-      for name, subdir in self._subdirs.items():
-         if not subdir.empty():
-            yield name
-   # --- end of list_package_names (...) ---
-
-   def list_packages ( self,
-      for_deprules=False, is_default_category=False
-   ):
+   def list_packages ( self, name_only=False ):
       """Lists all packages in this category.
       Yields <category>/<package name> or a dict (see for_deprules below).
 
@@ -216,29 +208,20 @@ class Category ( roverlay.overlay.base.OverlayObject ):
       * is_default_category -- bool indicating whether this category is the
                                default one or not
       """
-      if for_deprules:
+      if name_only:
          for name, subdir in self._subdirs.items():
             if not subdir.empty():
-               yield dict (
-                  dep_str               = name,
-                  resolving_package     = ( self.name + "/" + name ),
-                  is_selfdep            = 2 if is_default_category else 1,
-                  # prefer packages from the default category (really?)
-                  priority              = 80 if is_default_category else 90,
-                  finalize              = True,
-                  selfdep_package_names = filter (
-                     None, (
-                        p.get ( 'package_name', do_fallback=True )
-                        for p in subdir.iter_package_info()
-                     )
-                  ),
-               )
+               yield name
       else:
          for name, subdir in self._subdirs.items():
             if not subdir.empty():
                yield self.name + os.sep + name
    # --- end of list_packages (...) ---
 
+   def list_package_names ( self ):
+      return self.list_packages ( name_only=True )
+   # --- end of list_package_names (...) ---
+
    def supports_threadsafe_manifest_writing ( self, unsafe=True ):
       """Returns True if manifest writing is thread safe for this
       category, else False. Also returns True for empty categories.

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index fb57d39..f3157f3 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -78,7 +78,9 @@ class OverlayCreator ( object ):
       )
 
       self.depresolver = roverlay.recipe.easyresolver.setup ( self._err_queue )
-      self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
+      self.depresolver.make_selfdep_pool (
+         self.overlay.get_depres_rule_generator()
+      )
 
       if greedy_depres:
          self._depres_channel_cls = roverlay.depres.channels.EbuildJobChannel

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 1a07091..80bf836 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -32,6 +32,8 @@ import roverlay.overlay.category
 import roverlay.overlay.header
 import roverlay.overlay.pkgdir.base
 import roverlay.overlay.pkgdir.distroot.static
+import roverlay.overlay.rulegen
+
 
 class Overlay ( roverlay.overlay.base.OverlayObject ):
    DEFAULT_USE_DESC = (
@@ -701,14 +703,9 @@ class Overlay ( roverlay.overlay.base.OverlayObject ):
             yield package
    # --- end of list_packages (...) ---
 
-   def list_rule_kwargs ( self ):
-      for cat in self._categories.values():
-         for kwargs in cat.list_packages (
-            for_deprules        = True,
-            is_default_category = ( cat.name is self.default_category )
-         ):
-            yield kwargs
-   # --- end of list_rule_kwargs (...) ---
+   def get_depres_rule_generator ( self ):
+      return roverlay.overlay.rulegen.DepresRuleGenerator ( self )
+   # --- end of get_depres_rule_generator (...) ---
 
    def readonly ( self ):
       return not self._writeable

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     0b3a8fd9aada581d8510e4f2bd4a433702d792b0
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Sep  2 16:18:09 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Sep  2 16:18:09 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=0b3a8fd9

roverlay/overlay: depres rule generator

---
 roverlay/overlay/rulegen.py | 71 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/roverlay/overlay/rulegen.py b/roverlay/overlay/rulegen.py
new file mode 100644
index 0000000..7610e9a
--- /dev/null
+++ b/roverlay/overlay/rulegen.py
@@ -0,0 +1,71 @@
+# R overlay -- overlay package, depres rule generator
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+class DepresRuleGenerator ( object ):
+
+   def __init__ ( self, overlay ):
+      super ( DepresRuleGenerator, self ).__init__()
+      self.overlay_ref = overlay.get_ref()
+      self.rule_class  = None
+   # --- end of __init__ (...) ---
+
+   def make_rule_args ( self ):
+      overlay               = self.overlay_ref.deref_safe()
+      default_category_name = overlay.default_category
+
+      # COULDFIX: direct access to "private" attributes
+      for cat_name, cat in overlay._categories.items():
+         is_default_category = cat_name == default_category_name
+
+         for pkgdir_name, pkgdir in cat._subdirs.items():
+            if not pkgdir.empty():
+               repo_ids      = set()
+               package_names = set()
+               for p_info in pkgdir.iter_package_info():
+                  package_name = p_info.get (
+                     'package_name', do_fallback=True
+                  )
+                  if package_name:
+                     package_names.add ( package_name )
+
+                  repo = p_info.get ( 'origin', do_fallback=True )
+                  if repo is not None:
+                     repo_ids.add ( repo.get_identifier() )
+               # -- end for <get repo ids / package names>
+
+               yield (
+                  repo_ids,
+                  dict (
+                     dep_str               = pkgdir_name,
+                     resolving_package     = ( cat_name + '/' + pkgdir_name ),
+                     is_selfdep            = 2 if is_default_category else 1,
+                     priority              = 90,
+                     finalize              = True,
+                     selfdep_package_names = package_names,
+                  )
+               )
+         # -- end for pkgdir
+      # -- end for category
+   # --- end of make_rule_args (...) ---
+
+   def make_rules ( self ):
+      for repo_ids, rule_kwargs in self.make_rule_args():
+         yield ( repo_ids, self.rule_class ( **rule_kwargs ) )
+   # --- end of make_rules (...) ---
+
+   def make_rule_dict ( self ):
+      rule_dict = dict()
+      for repo_ids, rule_kwargs in self.make_rule_args():
+         rule = self.rule_class ( **rule_kwargs )
+         for repo_id in repo_ids:
+            if repo_id in rule_dict:
+               rule_dict [repo_id].append ( rule )
+            else:
+               rule_dict [repo_id] = [ rule ]
+      return rule_dict
+   # --- end of make_rule_dict (...) ---
+
+# --- end of DepresRuleGenerator ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     5e738c15639dddb86446e5c749d2c986f4350580
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug 29 12:28:14 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug 29 12:28:14 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=5e738c15

common base object for Overlay,Category,...

---
 roverlay/overlay/base.py     | 34 ++++++++++++++++++++++++++++++++++
 roverlay/overlay/category.py | 10 +++++-----
 roverlay/overlay/root.py     | 12 ++++++------
 3 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/roverlay/overlay/base.py b/roverlay/overlay/base.py
new file mode 100644
index 0000000..b380e8f
--- /dev/null
+++ b/roverlay/overlay/base.py
@@ -0,0 +1,34 @@
+# R overlay -- overlay package, overlay base object
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import logging
+
+import roverlay.util.objects
+
+class OverlayObject ( roverlay.util.objects.ReferenceTree ):
+
+   # always keep a (weak) reference to self:
+   # (a) Overlay: multiple Category objects use this ref
+   # (b) Category: ^ PackageDir ...
+   # (c) PackageDir: ^ PackageInfo ...
+   #
+   CACHE_REF = True
+
+   def __init__ ( self, name, logger, directory, parent ):
+      super ( OverlayObject, self ).__init__ ( parent )
+      self.name   = name
+      self.logger = (
+         logger.getChild ( name ) if logger else logging.getLogger ( name )
+      )
+      self.physical_location = directory
+   # --- end of __init__ (...) ---
+
+   # inherited:
+   #def get_parent
+   #def get_upper
+   #def get_ref
+
+# --- end of OverlayObject ---

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 2fc3a4f..d2b2028 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -27,6 +27,7 @@ except ImportError:
 import roverlay.stats.collector
 
 import roverlay.overlay.pkgdir.base
+import roverlay.overlay.base
 ##import roverlay.overlay.pkgdir.packagedir_ebuildmanifest
 ##import roverlay.overlay.pkgdir.packagedir_newmanifest
 
@@ -77,14 +78,14 @@ class WriteQueueJob ( object ):
 
 # --- end of WriteQueueJob ---
 
-class Category ( object ):
+class Category ( roverlay.overlay.base.OverlayObject ):
 
    WRITE_JOBCOUNT = 3
 
    STATS = roverlay.stats.collector.static.overlay
 
    def __init__ ( self,
-      name, logger, directory, get_header, runtime_incremental
+      name, logger, directory, get_header, runtime_incremental, parent
    ):
       """Initializes a overlay/portage category (such as 'app-text', 'sci-R').
 
@@ -95,12 +96,11 @@ class Category ( object ):
       * get_header          -- function that returns an ebuild header
       * runtime_incremental -- enable/disable runtime incremental writing
                                for this category (and all created PackageDirs)
+      * parent              -- overlay object that created/creates this object
       """
-      self.logger              = logger.getChild ( name )
-      self.name                = name
+      super ( Category, self ).__init__ ( name, logger, directory, parent )
       self._lock               = threading.RLock()
       self._subdirs            = dict()
-      self.physical_location   = directory
       self.get_header          = get_header
       self.runtime_incremental = runtime_incremental
       self.packagedir_cls      = roverlay.overlay.pkgdir.base.get_class()

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 6e3b1d7..1a07091 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -27,15 +27,15 @@ import threading
 import roverlay.config
 import roverlay.util
 import roverlay.overlay.additionsdir
+import roverlay.overlay.base
 import roverlay.overlay.category
 import roverlay.overlay.header
 import roverlay.overlay.pkgdir.base
 import roverlay.overlay.pkgdir.distroot.static
 
-class Overlay ( object ):
+class Overlay ( roverlay.overlay.base.OverlayObject ):
    DEFAULT_USE_DESC = (
       'byte-compile - enable byte compiling\n'
-      'R_suggests - install recommended packages'
    )
 
    @classmethod
@@ -124,9 +124,8 @@ class Overlay ( object ):
       * keep_n_ebuilds      -- number of ebuilds to keep (per package),
                                any "false" Value (None, 0, ...) disables this
       """
-      self.name                 = name
-      self.logger               = logger.getChild ( 'overlay' )
-      self.physical_location    = directory
+      super ( Overlay, self ).__init__ ( name, logger, directory, None )
+
       self.default_category     = default_category
 
       self._eclass_files        = eclass_files
@@ -203,7 +202,8 @@ class Overlay ( object ):
                   self.logger,
                   self.physical_location + os.sep + category,
                   get_header=self._header.get,
-                  runtime_incremental=self._runtime_incremental
+                  runtime_incremental=self._runtime_incremental,
+                  parent=self,
                )
                self._categories [category] = newcat
          finally:

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     6174d0fd60ff404e54d51ddf0286d1a13c59aa5d
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Aug 20 16:33:13 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Aug 20 16:33:13 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6174d0fd

roverlay/overlay/worker: remove unused imports

---
 roverlay/overlay/worker.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/roverlay/overlay/worker.py b/roverlay/overlay/worker.py
index 6ec5da6..28b3ab7 100644
--- a/roverlay/overlay/worker.py
+++ b/roverlay/overlay/worker.py
@@ -15,9 +15,6 @@ __all__ = [ 'OverlayWorker', ]
 import sys
 import threading
 
-from roverlay.depres.channels    import EbuildJobChannel
-from roverlay.ebuild.creation    import EbuildCreation
-
 # this controls whether debug message from OverlayWorker.run() are printed
 # to stderr or suppressed
 DEBUG = True

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     abb8299807a7df2474ebcf52c7ad0187da94d810
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Aug 20 16:33:39 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Aug 20 16:33:39 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=abb82998

overlay creator: optionally use non-greedy channels

---
 roverlay/overlay/creator.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 96c5531..fb57d39 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -50,7 +50,7 @@ class OverlayCreator ( object ):
 
    def __init__ ( self,
       skip_manifest, incremental, immediate_ebuild_writes,
-      logger=None, allow_write=True
+      logger=None, allow_write=True, greedy_depres=True,
    ):
       if logger is None:
          self.logger = self.__class__.LOGGER
@@ -80,6 +80,13 @@ class OverlayCreator ( object ):
       self.depresolver = roverlay.recipe.easyresolver.setup ( self._err_queue )
       self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
 
+      if greedy_depres:
+         self._depres_channel_cls = roverlay.depres.channels.EbuildJobChannel
+      else:
+         self._depres_channel_cls = (
+            roverlay.depres.channels.NonGreedyDepresChannel
+         )
+
       self.package_rules = PackageRules.get_configured()
 
       self.NUMTHREADS  = config.get ( 'EBUILD.jobcount', 0 )
@@ -118,7 +125,7 @@ class OverlayCreator ( object ):
       * **channel_kw -- keywords for EbuildJobChannel.__init__
       """
       return self.depresolver.register_channel (
-         roverlay.depres.channels.EbuildJobChannel ( **channel_kw )
+         self._depres_channel_cls ( **channel_kw )
       )
    # --- end of _get_resolver_channel (...) ---
 

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     a1290081b70d34ac70315970cffe5f70c3b31bc5
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 29 09:25:20 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 29 09:25:20 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a1290081

remove old stats code in overlay/creator.py

---
 roverlay/overlay/creator.py | 94 ---------------------------------------------
 1 file changed, 94 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 4bbedbf..0666335 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -94,100 +94,6 @@ class OverlayCreator ( object ):
 
    # --- end of __init__ (...) ---
 
-   def get_stats ( self ):
-      raise NotImplementedError ( "about to be removed!" )
-      pkg_added   = self.package_added.get_nowait()
-      pkg_created = self.create_success.get_nowait()
-      pkg_failed  = self.create_fail.get_nowait()
-      ov_added    = self.overlay_added.get_nowait()
-      ov_failed   = pkg_created - ov_added
-      processed   = pkg_created + pkg_failed
-      failed      = pkg_failed + ov_failed
-
-      return (
-         pkg_added, pkg_created, pkg_failed,
-         ov_added, ov_failed,
-         processed, failed
-      )
-   # --- end of get_stats (...) ---
-
-   def stats_str ( self, enclose=True ):
-      """Returns a string with some overlay creation stats."""
-      raise NotImplementedError ( "about to be removed!" )
-      def stats_gen():
-         """Yields stats strings."""
-         stats = self.get_stats()
-
-         # the length of the highest number in stats (^=digit count)
-         # max_number_len := { 1,...,5 }
-         max_number_len = min ( 5, len ( str ( max ( stats ) ) ) )
-
-         for stats_tuple in zip (
-            stats,
-            (
-               'packages added to the ebuild creation queue',
-               'packages passed ebuild creation',
-               'packages failed ebuild creation',
-               'ebuilds could be added to the overlay',
-               'ebuilds couldn\'t be added to the overlay',
-               'packages processed in total',
-               'packages failed in total',
-            ),
-         ):
-            yield "{num:<{l}} {s}".format (
-               num = stats_tuple [0],
-               s   = stats_tuple [1],
-               l   = max_number_len,
-            )
-
-         yield ""
-
-         k_len = min (
-            39,
-            max ( len ( k ) for k in self._timestats.keys() )
-         )
-
-         for k, v in self._timestats.items():
-            if v < 0:
-               yield "time for {:<{l}} : <unknown>".format ( k, l=k_len, )
-
-            elif v < 1:
-               yield "time for {:<{l}} : {} ms".format (
-                  k,
-                  round ( v * 1000, 2 ),
-                  l = k_len,
-               )
-
-            elif v > 300:
-               yield "time for {:<{l}} : {} minutes".format (
-                  k,
-                  round ( v / 60., 2 ),
-                  l = k_len,
-               )
-
-            else:
-               yield "time for {}: {} seconds".format ( k, round ( v, 2 ) )
-      # --- end of stats_gen (...) ---
-
-      if enclose:
-         stats = list ( stats_gen() )
-
-         # maxlen := { 2,...,80 }
-         maxlen = 2 + min ( 78,
-            len ( max ( stats, key=lambda s : len( s ) ) )
-         )
-
-         return (
-            "{0:-^{1}}\n".format ( " Overlay creation stats ", maxlen )
-            + '\n'.join ( stats )
-            #+ '\n{0:-^{1}}'.format ( '', maxlen )
-            + '\n' + ( maxlen * '-' )
-         )
-
-      else:
-         return '\n'.join ( stats_gen() )
-   # --- end of stats_str (...) ---
-
    def release_package_rules ( self ):
       """Removes all package rules from this object.
 

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/

[André Erdmann]

commit:     35368b2daf90b7be2a76f2af4608961bcbc1eb46
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 14:02:22 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 14:02:22 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=35368b2d

roverlay/overlay/root: remove/replace debug code

---
 roverlay/overlay/root.py | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 0c291da..099d6d5 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -1,6 +1,6 @@
 # R overlay -- overlay package, overlay root
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
 # Distributed under the terms of the GNU General Public License;
 # either version 2 of the License, or (at your option) any later version.
 
@@ -703,10 +703,9 @@ class Overlay ( object ):
       ##     drop p
       ##
 
-      #FIXME: PKG_REMOVED*: debug code
-      print ( "REMOVE_BROKEN_PACKAGES: 'balance' ... " )
-      PKG_REMOVED     = list()
-      PKG_REMOVED_ADD = PKG_REMOVED.append
+
+      #PKG_REMOVED     = list()
+      #PKG_REMOVED_ADD = PKG_REMOVED.append
 
 
       num_pkg_removed = 0
@@ -714,21 +713,17 @@ class Overlay ( object ):
          for pkgdir in cat._subdirs.values():
             for pvr, p_info in pkgdir._packages.items():
                if not p_info.is_valid():
-                  # FIXME: debug print
-                  PKG_REMOVED_ADD ( "{}-{}".format ( pkgdir.name, pvr ) )
+                  #PKG_REMOVED_ADD ( "{}-{}".format ( pkgdir.name, pvr ) )
                   pkgdir.purge_package ( pvr )
                   num_pkg_removed += 1
       # -- end for cat;
 
-      print ( "REMOVE_BROKEN_PACKAGES: 'finalize' ... " )
-
-      #FIXME: PKG_REMOVED*
-      if PKG_REMOVED:
-         with open ( "/tmp/roverlay_selfdep_redux.dbg", 'wt' ) as DEBUG_FH:
-            for line in PKG_REMOVED:
-               DEBUG_FH.write ( line )
-               DEBUG_FH.write ( '\n' )
 
+#      if PKG_REMOVED:
+#         with open ( "/tmp/roverlay_selfdep_redux.dbg", 'wt' ) as DEBUG_FH:
+#            for line in PKG_REMOVED:
+#               DEBUG_FH.write ( line )
+#               DEBUG_FH.write ( '\n' )
 
       # FIXME: debug prints (use logging, ...)
 
@@ -736,13 +731,13 @@ class Overlay ( object ):
          # remove_empty_categories() could be done in the loop above
          self.remove_empty_categories()
 
-         print (
-            'REMOVE_BROKEN_PACKAGES: {:d} ebuilds have been removed'.format (
+         self.logger.info (
+            'remove_broken_packages: {:d} ebuilds have been dropped.'.format (
                num_pkg_removed
             )
          )
       else:
-         print ( "REMOVE_BROKEN_PACKAGES: nothing done ;)" )
+         self.logger.info ( 'remove_broken_packages: no ebuilds removed.' )
 
       return num_pkg_removed
    # --- end of remove_broken_packages (...) ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     35368b2daf90b7be2a76f2af4608961bcbc1eb46
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 14:02:22 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 14:02:22 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=35368b2d

roverlay/overlay/root: remove/replace debug code

---
 roverlay/overlay/root.py | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 0c291da..099d6d5 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -1,6 +1,6 @@
 # R overlay -- overlay package, overlay root
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
 # Distributed under the terms of the GNU General Public License;
 # either version 2 of the License, or (at your option) any later version.
 
@@ -703,10 +703,9 @@ class Overlay ( object ):
       ##     drop p
       ##
 
-      #FIXME: PKG_REMOVED*: debug code
-      print ( "REMOVE_BROKEN_PACKAGES: 'balance' ... " )
-      PKG_REMOVED     = list()
-      PKG_REMOVED_ADD = PKG_REMOVED.append
+
+      #PKG_REMOVED     = list()
+      #PKG_REMOVED_ADD = PKG_REMOVED.append
 
 
       num_pkg_removed = 0
@@ -714,21 +713,17 @@ class Overlay ( object ):
          for pkgdir in cat._subdirs.values():
             for pvr, p_info in pkgdir._packages.items():
                if not p_info.is_valid():
-                  # FIXME: debug print
-                  PKG_REMOVED_ADD ( "{}-{}".format ( pkgdir.name, pvr ) )
+                  #PKG_REMOVED_ADD ( "{}-{}".format ( pkgdir.name, pvr ) )
                   pkgdir.purge_package ( pvr )
                   num_pkg_removed += 1
       # -- end for cat;
 
-      print ( "REMOVE_BROKEN_PACKAGES: 'finalize' ... " )
-
-      #FIXME: PKG_REMOVED*
-      if PKG_REMOVED:
-         with open ( "/tmp/roverlay_selfdep_redux.dbg", 'wt' ) as DEBUG_FH:
-            for line in PKG_REMOVED:
-               DEBUG_FH.write ( line )
-               DEBUG_FH.write ( '\n' )
 
+#      if PKG_REMOVED:
+#         with open ( "/tmp/roverlay_selfdep_redux.dbg", 'wt' ) as DEBUG_FH:
+#            for line in PKG_REMOVED:
+#               DEBUG_FH.write ( line )
+#               DEBUG_FH.write ( '\n' )
 
       # FIXME: debug prints (use logging, ...)
 
@@ -736,13 +731,13 @@ class Overlay ( object ):
          # remove_empty_categories() could be done in the loop above
          self.remove_empty_categories()
 
-         print (
-            'REMOVE_BROKEN_PACKAGES: {:d} ebuilds have been removed'.format (
+         self.logger.info (
+            'remove_broken_packages: {:d} ebuilds have been dropped.'.format (
                num_pkg_removed
             )
          )
       else:
-         print ( "REMOVE_BROKEN_PACKAGES: nothing done ;)" )
+         self.logger.info ( 'remove_broken_packages: no ebuilds removed.' )
 
       return num_pkg_removed
    # --- end of remove_broken_packages (...) ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     b3c06fe525a85dbd2820c14a47cd7628739fa78b
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Jun 22 10:19:38 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Jun 22 10:19:38 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b3c06fe5

overlay, scan(): don't filter all exceptions

---
 roverlay/overlay/root.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index a3b02f1..35ab5ad 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -682,6 +682,8 @@ class Overlay ( object ):
          for cat in scan_categories():
             try:
                cat.scan ( **kw )
+            except ( RuntimeError, SystemError, KeyboardInterrupt, ):
+               raise
             except Exception as e:
                self.logger.exception ( e )
    # --- end of scan (...) ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     dea12476091298811456b8c8f6ca596ae195e7ee
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Jun 15 08:58:16 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Jun 15 08:58:16 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=dea12476

overlay: write/update USE_EXPAND description files

---
 roverlay/overlay/header.py |  13 ++++
 roverlay/overlay/root.py   | 168 ++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 171 insertions(+), 10 deletions(-)

diff --git a/roverlay/overlay/header.py b/roverlay/overlay/header.py
index 6cd1f1f..8c7c05a 100644
--- a/roverlay/overlay/header.py
+++ b/roverlay/overlay/header.py
@@ -29,6 +29,19 @@ class EbuildHeader ( object ):
       return self._cached_header
    # --- end of get (...) ---
 
+   def get_use_expand_header ( self, use_expand_name ):
+      if self.default_header:
+         return (
+            self.default_header + '\n'
+            + (
+               '# This file contains descriptions of the {} '
+               'USE_EXPAND flags.'.format ( use_expand_name )
+            ) + '\n'
+         )
+      else:
+         return None
+   # --- end of get_use_expand_header (...) ---
+
    def _make ( self ):
       if self.eclasses:
          inherit = 'inherit ' + ' '.join ( sorted ( self.eclasses ) )

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 98d0b26..71156df 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -17,6 +17,7 @@ due do double-linkage between PackageInfo and PackageDir.
 
 __all__ = [ 'Overlay', ]
 
+import errno
 import logging
 import os
 import shutil
@@ -313,13 +314,18 @@ class Overlay ( object ):
 
          # profiles/desc/<r_suggests>.desc
          # !!! (late) config access (FIXME)
+
+         use_expand_name = roverlay.config.get_or_fail (
+            "EBUILD.USE_EXPAND.name"
+         ).rstrip ( "_" )
+
          self._write_rsuggests_use_desc (
             (
                self._profiles_dir + os.sep + 'desc' + os.sep
-               + roverlay.config.get_or_fail (
-                  "EBUILD.USE_EXPAND.name"
-               ).rstrip ( "_" ).lower()+ '.desc'
-            )
+               + use_expand_name.lower() + '.desc'
+            ),
+            use_expand_name.upper(),
+            roverlay.config.get ( 'OVERLAY.backup_desc', True )
          )
 
          # profiles/use.desc
@@ -342,12 +348,154 @@ class Overlay ( object ):
          raise
    # --- end of _init_overlay (...) ---
 
-   def _write_rsuggests_use_desc ( self, desc_file ):
-      # TODO
-      #open ( desc_file, "r" ), errno == errno.ENOENT (2)
-      self.logger.error (
-         "Cannot write {!r}: code is missing ;)".format ( desc_file )
-      )
+   def _write_rsuggests_use_desc (
+      self, desc_file, use_expand_name, backup_file, rewrite=False
+   ):
+      """Creates a USE_EXPAND description file.
+
+      Reads the old file (if it exists) and imports its header / flag desc.
+
+      arguments:
+      * desc_file       -- path to the file that should be written/read
+      * use_expand_name -- name of the USE_EXPAND variable, e.g. R_SUGGESTS
+      * backup_file     -- move desc_file to backup_file before overwriting it
+                           This can also be an int i (=> desc_file + '.<i>')
+                           or a bool (if True => desc_file + '.bak').
+      * rewrite         -- force recreation of the desc file
+      """
+      FLAG_SEPA  = ' - '
+      DESC_UNDEF = 'unknown'
+
+      def do_backup ( dest, makedir=False ):
+         self.logger.debug ( "Moving old desc file to {!r}".format ( dest ) )
+         if makedir:
+            roverlay.util.dodir ( os.path.dirname ( dest ), mkdir_p=True )
+         shutil.move ( desc_file, dest )
+      # --- end of do_backup (...) ---
+
+      def read_desc_file():
+         """Reads the old desc file (if it exists).
+         Returns a 3-tuple ( list header, dict flags, bool file_existed ).
+
+         arguments:
+         * @implicit desc_file --
+
+         Passes all exceptions (IOError, ...) but "file does not exist".
+         """
+
+         FH     = None
+         header = list()
+         # flags := dict { <flag name> => <desc>|None }
+         flags  = dict()
+         fexist = False
+         try:
+            FH = open ( desc_file, 'rt' )
+
+            addto_header = True
+
+            for line in FH.readlines():
+               rsline = line.rstrip()
+               if rsline and rsline [0] != '#':
+                  flag, sepa, desc = rsline.partition ( FLAG_SEPA )
+                  # <desc> == DESC_UNDEF -- ignore
+                  flags [flag]     = desc if sepa else None
+                  addto_header     = False
+               elif addto_header is True:
+                  header.append ( rsline )
+               elif rsline:
+                  self.logger.warning (
+                     "dropping line from {f!r}: {l}".format (
+                        f=desc_file, l=rsline
+                     )
+                  )
+            # -- end for
+
+            FH.close()
+            fexist = True
+         except IOError as ioerr:
+            if ioerr.errno == errno.ENOENT:
+               pass
+            else:
+               raise
+         finally:
+            if FH: FH.close()
+
+         return ( header, flags, fexist )
+      # --- end of read_desc_file ---
+
+      def gen_desc ( header, flags ):
+         """Creates new text lines for the desc file.
+
+         arguments:
+         * @implicit use_expand_name --
+         * header -- header line(s) to use (a default header will be created
+                     if this is empty)
+         * flags  -- flag=>desc dict
+         """
+         NEWLINE = '\n'
+
+         if header:
+            for line in header:
+               yield line
+               yield NEWLINE
+            #yield NEWLINE
+         else:
+            defheader = self._header.get_use_expand_header ( use_expand_name )
+            if defheader:
+               yield defheader
+               yield NEWLINE
+
+         for flag, desc in sorted ( flags.items(), key=lambda e: e[0] ):
+            if desc:
+               yield flag + FLAG_SEPA + desc
+            else:
+               yield flag + FLAG_SEPA + DESC_UNDEF
+            yield NEWLINE
+      # --- end of gen_desc (...) ---
+
+      header, old_flags, can_backup = read_desc_file()
+
+      if self._incremental:
+         # incremental: add new flags
+         #  Create dict flag=>None that contains all new flags
+         #  and copy old_flags "over" it.
+         #
+         flags = dict.fromkeys ( self._rsuggests_flags )
+         flags.update ( old_flags )
+      else:
+         # not incremental: discard old flags
+         #  Create a dict that contains the new flags only and use desc
+         #  from old_flags if available.
+         #
+         flags = {
+            flag: old_flags.get ( flag, None )
+            for flag in self._rsuggests_flags
+         }
+      # -- end if
+
+      # don't rewrite the desc file if nothing has changed
+      if rewrite or (
+         frozenset ( flags.keys() ) != frozenset ( old_flags.keys() )
+      ):
+         if can_backup:
+            if backup_file is True:
+               do_backup ( desc_file + '.bak' )
+            elif isinstance ( backup_file, int ):
+               do_backuo ( desc_file + '.' + str ( backup_file ) )
+            elif backup_file:
+               do_backup ( str ( backup_file ), makedir=True )
+         # -- end if
+
+         self.logger.debug ( "writing desc file {!r}".format ( desc_file ) )
+
+         roverlay.util.dodir ( os.path.dirname ( desc_file ) )
+         with open ( desc_file, 'wt' ) as FH:
+            for line in gen_desc ( header, flags ):
+               FH.write ( line )
+      else:
+         self.logger.debug (
+            "not writing desc file {!r} (nothing changed)".format ( desc_file )
+         )
    # --- end of _write_rsuggests_use_desc (...) ---
 
    def add ( self, package_info ):

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     f53903b04fc9f3ae739ea88bfbda616fb5de32e3
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Apr 25 15:07:08 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Apr 25 15:07:08 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f53903b0

overlay/creator: add scan_overlay time stats

---
 roverlay/overlay/creator.py |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 6aa5af9..931a2ff 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -95,6 +95,7 @@ class OverlayCreator ( object ):
 		# this queue is used to propagate exceptions from threads
 		self._err_queue = errorqueue.ErrorQueue()
 
+		time_scan_overlay = time.time()
 		# init overlay using config values
 		self.overlay = Overlay.new_configured (
 			logger              = self.logger,
@@ -103,6 +104,7 @@ class OverlayCreator ( object ):
 			skip_manifest       = skip_manifest,
 			runtime_incremental = immediate_ebuild_writes,
 		)
+		time_scan_overlay = time.time() - time_scan_overlay
 
 		self.depresolver = easyresolver.setup ( self._err_queue )
 		self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
@@ -138,6 +140,9 @@ class OverlayCreator ( object ):
 
 		self._timestats     = collections.OrderedDict()
 
+		if incremental and time_scan_overlay >= 0.1:
+			self._timestats ['scan_overlay'] = time_scan_overlay
+
 		for k in (
 			'sync_packages',
 			'add_packages',

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     2e601a6f8b3171fde698c7f5284dbc94b35705bc
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Apr 25 15:09:44 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Apr 25 15:09:44 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2e601a6f

overlay/category: stop queue workers

Stop write queue workers when an exception is caught.
Also changed the RERAISE code (again).

---
 roverlay/overlay/category.py |   19 ++++++++++++++++---
 1 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 97f87e7..dd55c14 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -199,7 +199,7 @@ class Category ( object ):
 			* q        -- queue
 			* write_kw -- keywords for write(...)
 			"""
-			while not q.empty():
+			while not q.empty() and not hasattr ( self, 'RERAISE' ):
 				try:
 					pkg = q.get_nowait()
 					# remove manifest writing from threaded writing since it's
@@ -207,7 +207,8 @@ class Category ( object ):
 					pkg.write ( **write_kw )
 				except queue.Empty:
 					break
-				except:
+				except ( Exception, KeyboardInterrupt ) as err:
+					self.logger.exception ( err )
 					self.RERAISE = sys.exc_info()
 		# --- end of run_write_queue (...) ---
 
@@ -258,7 +259,19 @@ class Category ( object ):
 			for w in workers: w.join()
 
 			if hasattr ( self, 'RERAISE' ) and self.RERAISE:
-				raise ( self.RERAISE [0], self.RERAISE [1], self.RERAISE [2] )
+				# ref: PEP 3109
+				#  results in correct traceback when running python 3.x
+				#  and inaccurate traceback with python 2.x,
+				#  which can be tolerated since the exception has been logged
+				try:
+					reraise = self.RERAISE[0] ( self.RERAISE[1] )
+				except TypeError:
+					# "portage.exception.FileNotFound is not subscriptable"
+					reraise = self.RERAISE[1]
+
+				reraise.__traceback__ = self.RERAISE [2]
+				raise reraise
+			# --- end RERAISE;
 
 			self.remove_empty()
 

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     d6bbb89d92aca4dc6cd351d1b249ebbbc2876b69
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb  9 21:27:12 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb  9 21:27:12 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d6bbb89d

overlay/category: fix RERAISE

this fixes a python 2 vs 3 issue

---
 roverlay/overlay/category.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 4a03cce..97f87e7 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -258,7 +258,7 @@ class Category ( object ):
 			for w in workers: w.join()
 
 			if hasattr ( self, 'RERAISE' ) and self.RERAISE:
-				raise self.RERAISE [0], self.RERAISE [1], self.RERAISE [2]
+				raise ( self.RERAISE [0], self.RERAISE [1], self.RERAISE [2] )
 
 			self.remove_empty()
 

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     d21225c2eba72f010b9ca824db192f450895bc03
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb  9 19:34:29 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb  9 20:08:15 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d21225c2

overlay/category: RERAISE with correct traceback

self.RERAISE contains the exception info (as returned by sys.exc_info()) now

---
 roverlay/overlay/category.py |   11 +++++------
 1 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 3a58367..4a03cce 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -16,6 +16,7 @@ __all__ = [ 'Category', ]
 
 import threading
 import os
+import sys
 
 try:
 	import queue
@@ -206,9 +207,8 @@ class Category ( object ):
 					pkg.write ( **write_kw )
 				except queue.Empty:
 					break
-				except Exception as e:
-					#self.logger.exception ( e )
-					self.RERAISE = e
+				except:
+					self.RERAISE = sys.exc_info()
 		# --- end of run_write_queue (...) ---
 
 		if len ( self._subdirs ) == 0: return
@@ -257,15 +257,14 @@ class Category ( object ):
 			for w in workers: w.start()
 			for w in workers: w.join()
 
-			if hasattr ( self, 'RERAISE' ) and self.RERAISE is not None:
-				raise self.RERAISE
+			if hasattr ( self, 'RERAISE' ) and self.RERAISE:
+				raise self.RERAISE [0], self.RERAISE [1], self.RERAISE [2]
 
 			self.remove_empty()
 
 			# write manifest files
 			# fixme: debug print
 			if write_manifest and ( not manifest_threadsafe ):
-				#self.logger.info ( "Writing Manifest files for {}".format ( name ) )
 				print ( "Writing Manifest files ..." )
 				for package in self._subdirs.values():
 					package.write_manifest ( ignore_empty=True )

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     0e093591866ba15abd66b692ec190602ee9d9673
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb  9 19:16:45 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb  9 20:08:14 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=0e093591

overlay/creator: timestats

added (vague) time measurement for get_stats().

---
 roverlay/overlay/creator.py |  102 ++++++++++++++++++++++++++++++++----------
 1 files changed, 77 insertions(+), 25 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 6674e24..6aa5af9 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -12,13 +12,12 @@ main script).
 """
 __all__ = [ 'OverlayCreator', ]
 
+import collections
 import time
 import logging
 import threading
 import sys
 
-from collections import deque
-
 try:
 	import queue
 except ImportError:
@@ -137,8 +136,22 @@ class OverlayCreator ( object ):
 		# counts packages that passed adding to overlay
 		self.overlay_added  = PseudoAtomicCounter()
 
+		self._timestats     = collections.OrderedDict()
+
+		for k in (
+			'sync_packages',
+			'add_packages',
+			'ebuild_creation',
+			'overlay_write',
+		):
+			self._timestats [k] = -1
+
 	# --- end of __init__ (...) ---
 
+	def set_timestats ( self, name, seconds ):
+		self._timestats [name] = seconds
+	# --- end of set_timestats (...) ---
+
 	def get_stats ( self ):
 		pkg_added   = self.package_added.get_nowait()
 		pkg_created = self.create_success.get_nowait()
@@ -164,33 +177,68 @@ class OverlayCreator ( object ):
 			# the length of the highest number in stats (^=digit count)
 			# max_number_len := { 1,...,5 }
 			max_number_len = min ( 5, len ( str ( max ( stats ) ) ) )
-			num_fmt = '%(num)-' + str ( max_number_len ) + 's '
-
-			for i, s in enumerate ((
-				'packages added to the ebuild creation queue',
-				'packages passed ebuild creation',
-				'packages failed ebuild creation',
-				'ebuilds could be added to the overlay',
-				'ebuilds couldn\'t be added to the overlay',
-				'packages processed in total',
-				'packages failed in total',
-			)):
-				yield num_fmt % { 'num' : stats [i] } + s
+
+			for stats_tuple in zip (
+				stats,
+				(
+					'packages added to the ebuild creation queue',
+					'packages passed ebuild creation',
+					'packages failed ebuild creation',
+					'ebuilds could be added to the overlay',
+					'ebuilds couldn\'t be added to the overlay',
+					'packages processed in total',
+					'packages failed in total',
+				),
+			):
+				yield "{num:<{l}} {s}".format (
+					num = stats_tuple [0],
+					s   = stats_tuple [1],
+					l   = max_number_len,
+				)
+
+			yield ""
+
+			k_len = min (
+				39,
+				max ( len ( k ) for k in self._timestats.keys() )
+			)
+
+			for k, v in self._timestats.items():
+				if v < 0:
+					yield "time for {:<{l}} : <unknown>".format ( k, l=k_len, )
+
+				elif v < 1:
+					yield "time for {:<{l}} : {} ms".format (
+						k,
+						round ( v * 1000, 2 ),
+						l = k_len,
+					)
+
+				elif v > 300:
+					yield "time for {:<{l}} : {} minutes".format (
+						k,
+						round ( v / 60., 2 ),
+						l = k_len,
+					)
+
+				else:
+					yield "time for {}: {} seconds".format ( k, round ( v, 2 ) )
 		# --- end of stats_gen (...) ---
 
 		if enclose:
-			stats_str = deque ( stats_gen() )
+			stats = list ( stats_gen() )
+
 			# maxlen := { 2,...,80 }
 			maxlen = 2 + min ( 78,
-				len ( max ( stats_str, key=lambda s : len( s ) ) )
+				len ( max ( stats, key=lambda s : len( s ) ) )
 			)
 
-			stats_str.appendleft (
-				" Overlay creation stats ".center ( maxlen, '-' )
+			return (
+				"{0:-^{1}}\n".format ( " Overlay creation stats ", maxlen )
+				+ '\n'.join ( stats )
+				#+ '\n{0:-^{1}}'.format ( '', maxlen )
+				+ '\n' + ( maxlen * '-' )
 			)
-			stats_str.append ( '-' * maxlen )
-
-			return '\n'.join ( stats_str )
 
 		else:
 			return '\n'.join ( stats_gen() )
@@ -210,7 +258,7 @@ class OverlayCreator ( object ):
 		self.logger.debug (
 			"timestamp: {} (after {} seconds)".format ( description, delta )
 		)
-		return _stop
+		return delta
 	# --- end of _timestamp (...) ---
 
 	def add_package ( self, package_info ):
@@ -232,7 +280,9 @@ class OverlayCreator ( object ):
 		if self.overlay.writeable():
 			start = time.time()
 			self.overlay.write()
-			self._timestamp ( "overlay written", start )
+			self._timestats ['overlay_write'] = (
+				self._timestamp ( "overlay written", start )
+			)
 		else:
 			self.logger.warning ( "Not allowed to write overlay!" )
 	# --- end of write_overlay (...) ---
@@ -245,12 +295,14 @@ class OverlayCreator ( object ):
 	def run ( self, close_when_done=False ):
 		"""Starts ebuild creation and waits until done."""
 		self._runlock.acquire()
-		#self._time_start_run.append ( time.time() )
+		t_start = time.time()
 		try:
 			self.start()
 			self.join()
 		finally:
-			#self._time_stop_run.append ( time.time() )
+			self._timestats ['ebuild_creation'] = (
+				self._timestamp ( "run() done", t_start )
+			)
 			self._runlock.release()
 			if close_when_done:
 				self.close()

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     b1425ef200dd984ffd6387baa4eebc6bb0f6fb68
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jan 30 20:15:18 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jan 30 20:15:18 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b1425ef2

overlay/creator: use package rules

Currently, this is a no-op as package rule loading is not implemented (also
previous commit).

---
 roverlay/overlay/creator.py |   15 +++++++++++----
 1 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 1e1c3cb..6674e24 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -27,12 +27,15 @@ except ImportError:
 
 
 from roverlay                    import config, errorqueue
+
 from roverlay.overlay            import Overlay
 from roverlay.overlay.worker     import OverlayWorker
 from roverlay.packageinfo        import PackageInfo
+from roverlay.packagerules.rules import PackageRules
 
 from roverlay.recipe             import easyresolver
 
+
 class PseudoAtomicCounter ( object ):
 
 	def __init__ ( self, start=0, long_int=False ):
@@ -105,6 +108,8 @@ class OverlayCreator ( object ):
 		self.depresolver = easyresolver.setup ( self._err_queue )
 		self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
 
+		self.package_rules = PackageRules.get_configured()
+
 		self.NUMTHREADS  = config.get ( 'EBUILD.jobcount', 0 )
 
 		self._pkg_queue = queue.Queue()
@@ -214,10 +219,12 @@ class OverlayCreator ( object ):
 		arguments:
 		* package_info --
 		"""
-		if self.overlay.add ( package_info ):
-			self._pkg_queue.put ( package_info )
-			# FIXME package_added is now the # of packages queued for creation
-			self.package_added.inc()
+		if self.package_rules.apply_actions ( package_info ):
+			if self.overlay.add ( package_info ):
+				self._pkg_queue.put ( package_info )
+				# FIXME package_added is now the # of packages queued for creation
+				self.package_added.inc()
+		# else filtered out
 	# --- end of add_package (...) ---
 
 	def write_overlay ( self ):

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     56fcb45535c4457d9dff7844c54f8a0c49a8500a
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 17 17:26:15 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 17 17:26:15 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=56fcb455

code comment for overlay root

---
 roverlay/overlay/root.py |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 8c22ce2..99c0850 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -10,6 +10,9 @@ This module provides the Overlay class that acts as an interface between
 PackageInfo instances (in memory) and the R overlay (directory in filesystem).
 Most requests are redirected to its subdirectories (Category), which, in turn,
 pass them to their subdirectories (PackageDir).
+
+Caution: Never deep-copy an overlay object. This leads to infinite recursion
+due do double-linkage between PackageInfo and PackageDir.
 """
 
 __all__ = [ 'Overlay', ]

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     63afbb8871376b2e08168115d12f703e4b35afc8
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug  3 09:38:34 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug  3 09:38:43 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=63afbb88

package.py: filter takes no keyword arguments

---
 roverlay/overlay/package.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 722be8b..18b1141 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -240,8 +240,8 @@ class PackageDir ( object ):
 		# sort them by version in reverse order
 		packages = reversed ( sorted (
 			filter (
-				function=is_ebuild if not cautious else is_ebuild_cautious,
-				iterable=self._packages.items()
+				is_ebuild if not cautious else is_ebuild_cautious,
+				self._packages.items()
 			),
 			key=lambda p : p [1] ['version']
 		) )

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     91736d8f9c07b1d64c42b6ddb0ab6b7da80ceb99
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Aug  1 12:17:02 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Aug  1 12:17:02 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=91736d8f

actually skip manifest creation with --no-manifest

---
 roverlay/overlay/category.py |   13 +++++++------
 roverlay/overlay/root.py     |   12 ++++++------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 6623446..3551b81 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -137,7 +137,7 @@ class Category ( object ):
 		overwrite_ebuilds,
 		keep_n_ebuilds,
 		cautious,
-		skip_manifest=False
+		write_manifest
 	):
 		"""Writes this category to its filesystem location.
 
@@ -205,14 +205,15 @@ class Category ( object ):
 
 			# write manifest files
 			# fixme: debug print
-			#self.logger.info ( "Writing Manifest files for {}".format ( name ) )
-			print ( "Writing Manifest files ..." )
-			for package in self._subdirs.values():
-				package.write_manifest ( ignore_empty=True )
+			if write_manifest:
+				#self.logger.info ( "Writing Manifest files for {}".format ( name ) )
+				print ( "Writing Manifest files ..." )
+				for package in self._subdirs.values():
+					package.write_manifest ( ignore_empty=True )
 
 		else:
 			for package in self._subdirs.values():
-				package.write ( **write_kwargs )
+				package.write ( write_manifest=write_manifest, **write_kwargs )
 
 			self.remove_empty()
 	# --- end of write (...) ---

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 53babdf..3efdf66 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -330,10 +330,10 @@ class Overlay ( object ):
 
 			for cat in self._categories.values():
 				cat.write (
-					overwrite_ebuilds=False,
-					keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
-					cautious=True,
-					skip_manifest=self.skip_manifest
+					overwrite_ebuilds = False,
+					keep_n_ebuilds    = config.get ( 'OVERLAY.keep_nth_latest', None ),
+					cautious          = True,
+					write_manifest    = not self.skip_manifest
 				)
 		else:
 			# FIXME debug print
@@ -353,11 +353,11 @@ class Overlay ( object ):
 
 		returns: None (implicit)
 		"""
-		if self._writeable:
+		if self._writeable and not self.skip_manifest:
 			# FIXME: it would be good to ensure that profiles/categories exist
 			for cat in self._categories.values():
 				cat.write_manifest ( **manifest_kw )
-		else:
+		elif not self.skip_manifest:
 			# FIXME debug print
 			print (
 				"Dropped write_manifest request for readonly overlay {}!".format (

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     2d4af7c3ebe183400bcb0b732518e03d189cf35b
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 30 15:50:05 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 30 15:53:31 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2d4af7c3

overlay: skip_manifest

---
 roverlay/overlay/category.py |    7 ++++++-
 roverlay/overlay/creator.py  |    5 +++--
 roverlay/overlay/root.py     |   10 +++++++++-
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 2d66246..6623446 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -133,7 +133,12 @@ class Category ( object ):
 			package.show ( **show_kw )
 	# --- end of show (...) ---
 
-	def write ( self, overwrite_ebuilds, keep_n_ebuilds, cautious ):
+	def write ( self,
+		overwrite_ebuilds,
+		keep_n_ebuilds,
+		cautious,
+		skip_manifest=False
+	):
 		"""Writes this category to its filesystem location.
 
 		returns: None (implicit)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 5b0a973..5f346fe 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -72,7 +72,7 @@ class OverlayCreator ( object ):
 
 	LOGGER = logging.getLogger ( 'OverlayCreator' )
 
-	def __init__ ( self, logger=None, allow_write=True ):
+	def __init__ ( self, skip_manifest, logger=None, allow_write=True ):
 
 		if logger is None:
 			self.logger = self.__class__.LOGGER
@@ -91,7 +91,8 @@ class OverlayCreator ( object ):
 			eclass_files     = config.get ( 'OVERLAY.eclass_files', None ),
 			ebuild_header    = config.get ( 'EBUILD.default_header', None ),
 			incremental      = USE_INCREMENTAL_WRITE,
-			write_allowed    = allow_write
+			write_allowed    = allow_write,
+			skip_manifest    = skip_manifest
 		)
 
 		self.depresolver = easyresolver.setup ( self._err_queue )

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index ae3267f..53babdf 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -30,6 +30,7 @@ class Overlay ( object ):
 		ebuild_header,
 		write_allowed,
 		incremental,
+		skip_manifest,
 		runtime_incremental=True
 	):
 		"""Initializes an overlay.
@@ -47,8 +48,12 @@ class Overlay ( object ):
 		* incremental         -- enable/disable incremental writing:
 		                         use already existing ebuilds (don't recreate
 		                         them)
+		* skip_manifest       -- skip Manifest generation to save time
+		                         !!! The created overlay cannot be used with
+		                         emerge/portage
 		* runtime_incremental -- see package.py:PackageDir.__init__ (...),
 		                          Defaults to ?FIXME?
+
 		"""
 		self.name                 = name
 		self.logger               = logger.getChild ( 'overlay' )
@@ -66,6 +71,8 @@ class Overlay ( object ):
 		self._categories          = dict()
 		self._header              = EbuildHeader ( ebuild_header )
 
+		self.skip_manifest        = skip_manifest
+
 		# fixme or ignore: calculating eclass names twice,
 		# once here and another time when calling _init_overlay
 		self._header.set_eclasses ( frozenset (
@@ -325,7 +332,8 @@ class Overlay ( object ):
 				cat.write (
 					overwrite_ebuilds=False,
 					keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
-					cautious=True
+					cautious=True,
+					skip_manifest=self.skip_manifest
 				)
 		else:
 			# FIXME debug print

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     b568e1b218a2ed83dc0ae36bb4a3326b19078034
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:58:09 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:58:09 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b568e1b2

stop thread when queue is empty

---
 roverlay/overlay/category.py |   25 ++++++++++++-------------
 1 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index a0b1607..2d66246 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -145,19 +145,18 @@ class Category ( object ):
 			* q        -- queue
 			* write_kw -- keywords for write(...)
 			"""
-			try:
-				while not q.empty():
-					try:
-						pkg = q.get_nowait()
-						# remove manifest writing from threaded writing since it's
-						# single-threaded
-						pkg.write ( write_manifest=False, **write_kw )
-					#except ( Exception, KeyboardInterrupt ) as e:
-					except Exception as e:
-						# FIXME: reintroduce RERAISE
-						self.logger.exception ( e )
-			except queue.Empty:
-				pass
+			while not q.empty():
+				try:
+					pkg = q.get_nowait()
+					# remove manifest writing from threaded writing since it's
+					# single-threaded
+					pkg.write ( write_manifest=False, **write_kw )
+				except queue.Empty:
+					break
+				#except ( Exception, KeyboardInterrupt ) as e:
+				except Exception as e:
+					# FIXME: reintroduce RERAISE
+					self.logger.exception ( e )
 		# --- end of run_write_queue (...) ---
 
 		if len ( self._subdirs ) == 0: return

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     c456433d70dae2e58fb0297ab1b82822b2405b2d
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:58:40 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:58:40 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=c456433d

add "ebuild uncreatable" event

Immediately removes the PackageInfo object from the overlay
structure to save some memory.

	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/package.py

---
 roverlay/overlay/creator.py |    1 +
 roverlay/overlay/package.py |   22 ++++++++++++++++++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 60e9517..5b0a973 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -382,6 +382,7 @@ class OverlayCreator ( object ):
 			if package_info.overlay_package_ref.new_ebuild():
 				self.overlay_added.inc()
 		else:
+			package_info.overlay_package_ref.ebuild_uncreateable ( package_info )
 			self.create_fail.inc()
 
 	# --- end of _add_to_overlay (...) ---

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index bedbce3..cb5cf72 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -145,6 +145,25 @@ class PackageDir ( object ):
 			return False
 	# --- end of check_empty (...) ---
 
+	def ebuild_uncreateable ( self, package_info ):
+		"""Called when ebuild creation (finally) failed for a PackageInfo
+		object of this PackageDir.
+
+		arguments:
+		* package_info --
+		"""
+		try:
+			self._lock.acquire()
+			pvr = package_info ['ebuild_verstr']
+			# FIXME debug print
+			print ( "removing {PVR} from {PN}".format ( PVR=pvr, PN=self.name ) )
+			del self._packages [pvr]
+		except KeyError:
+			pass
+		finally:
+			self._lock.release()
+	# --- end of uncreateable_ebuild (...) ---
+
 	def empty ( self ):
 		"""Returns True if no ebuilds stored, else False.
 		Note that "not empty" doesn't mean "has ebuilds to write" or "has
@@ -255,12 +274,11 @@ class PackageDir ( object ):
 		return self._packages.keys()
 	# --- end of list_versions (...) ---
 
-	def new_ebuild ( self, write=False ):
+	def new_ebuild ( self ):
 		"""Called when a new ebuild has been created for this PackageDir."""
 		self._need_manifest = True
 		self._need_metadata = True
 		self.modified       = True
-		# FIXME: delete PackageInfo if ebuild creation did not succeed
 		if self.runtime_incremental:
 			with self._lock:
 				return self.write_ebuilds ( overwrite=False )

[gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/

[André Erdmann]

commit:     daed3bde53ce1fec42ad806813bb2798321b76d4
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:21:07 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:21:07 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=daed3bde

overlay: split incremental/runtime_incremental

Also added a _writeable bool to Overlay which controls
whether writing is allowed or not.

	geändert:   roverlay/overlay/category.py
	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/package.py
	geändert:   roverlay/overlay/root.py

---
 roverlay/overlay/category.py |   36 +++++++++-----
 roverlay/overlay/creator.py  |   20 +++----
 roverlay/overlay/package.py  |   23 ++++++--
 roverlay/overlay/root.py     |  113 +++++++++++++++++++++++++++++------------
 4 files changed, 129 insertions(+), 63 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index fab6373..a0b1607 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -16,25 +16,35 @@ class Category ( object ):
 
 	WRITE_JOBCOUNT = 3
 
-	def __init__ ( self, name, logger, directory, get_header, incremental ):
+	def __init__ ( self,
+		name, logger, directory, get_header, runtime_incremental
+	):
 		"""Initializes a overlay/portage category (such as 'app-text', 'sci-R').
 
 		arguments:
-		* name       -- name of the category
-		* logger     -- parent logger
-		* directory  -- filesystem location
-		* get_header -- function that returns an ebuild header
+		* name                -- name of the category
+		* logger              -- parent logger
+		* directory           -- filesystem location
+		* get_header          -- function that returns an ebuild header
+		* runtime_incremental -- enable/disable runtime incremental writing
+		                         for this category (and all created PackageDirs)
 		"""
-		self.logger            = logger.getChild ( name )
-		self.name              = name
-		self._lock             = threading.RLock()
-		self._subdirs          = dict()
-		self.physical_location = directory
-		self.get_header        = get_header
-		self.incremental       = incremental
+		self.logger              = logger.getChild ( name )
+		self.name                = name
+		self._lock               = threading.RLock()
+		self._subdirs            = dict()
+		self.physical_location   = directory
+		self.get_header          = get_header
+		self.runtime_incremental = runtime_incremental
 	# --- end of __init__ (...) ---
 
 	def _get_package_dir ( self, pkg_name ):
+		"""Returns a PackageDir object for pkg_name.
+		(so that <new object>.name == pkg_name and pkg_name in self._subdirs)
+
+		arguments:
+		* pkg_name --
+		"""
 		if not pkg_name in self._subdirs:
 			self._lock.acquire()
 			try:
@@ -44,7 +54,7 @@ class Category ( object ):
 						logger      = self.logger,
 						directory   = self.physical_location + os.sep + pkg_name,
 						get_header  = self.get_header,
-						incremental = self.incremental
+						runtime_incremental = self.runtime_incremental
 					)
 					self._subdirs [pkg_name] = newpkg
 			finally:

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index d12e0a5..60e9517 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -82,18 +82,16 @@ class OverlayCreator ( object ):
 		# this queue is used to propagate exceptions from threads
 		self._err_queue = errorqueue.ErrorQueue()
 
-		self.can_write_overlay = allow_write
-		self.write_incremental = allow_write and USE_INCREMENTAL_WRITE
-
 		# init overlay using config values
 		self.overlay = Overlay (
-			name=config.get_or_fail ( 'OVERLAY.name' ),
-			logger=self.logger,
-			directory=config.get_or_fail ( 'OVERLAY.dir' ),
-			default_category= config.get_or_fail ( 'OVERLAY.category' ),
-			eclass_files=config.get ( 'OVERLAY.eclass_files', None ),
-			ebuild_header=config.get ( 'EBUILD.default_header', None ),
-			incremental=self.write_incremental
+			name             = config.get_or_fail ( 'OVERLAY.name' ),
+			logger           = self.logger,
+			directory        = config.get_or_fail ( 'OVERLAY.dir' ),
+			default_category = config.get_or_fail ( 'OVERLAY.category' ),
+			eclass_files     = config.get ( 'OVERLAY.eclass_files', None ),
+			ebuild_header    = config.get ( 'EBUILD.default_header', None ),
+			incremental      = USE_INCREMENTAL_WRITE,
+			write_allowed    = allow_write
 		)
 
 		self.depresolver = easyresolver.setup ( self._err_queue )
@@ -219,7 +217,7 @@ class OverlayCreator ( object ):
 
 		arguments:
 		"""
-		if self.can_write_overlay:
+		if self.overlay.writeable():
 			start = time.time()
 			self.overlay.write()
 			self._timestamp ( "overlay written", start )

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index d2662bd..bedbce3 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -13,15 +13,25 @@ SUPPRESS_EXCEPTIONS = True
 class PackageDir ( object ):
 	EBUILD_SUFFIX = '.ebuild'
 
-	def __init__ ( self, name, logger, directory, get_header, incremental ):
+	def __init__ ( self,
+		name, logger, directory, get_header, runtime_incremental
+	):
 		"""Initializes a PackageDir which contains ebuilds, metadata and
 		a Manifest file.
 
 		arguments:
-		* name       -- name of the directory (${PN} in ebuilds)
-		* logger     -- parent logger
-		* directory  -- filesystem location of this PackageDir
-		* get_header -- function that returns an ebuild header
+		* name                -- name of the directory (${PN} in ebuilds)
+		* logger              -- parent logger
+		* directory           -- filesystem location of this PackageDir
+		* get_header          -- function that returns an ebuild header
+		* runtime_incremental -- enable/disable runtime incremental ebuild
+		                         writing. This trades speed (disabled) for
+		                         memory consumption (enabled) 'cause it will
+		                         write _all_ successfully created ebuilds
+		                         directly after they've been created.
+		                         Writing all ebuilds at once is generally faster
+		                         (+threading), but all PackageInfos must be
+		                         kept in memory for that.
 		"""
 		self.logger              = logger.getChild ( name )
 		self.name                = name
@@ -30,7 +40,7 @@ class PackageDir ( object ):
 		self._packages           = dict()
 		self.physical_location   = directory
 		self.get_header          = get_header
-		self.runtime_incremental = incremental
+		self.runtime_incremental = runtime_incremental
 
 		self._metadata = MetadataJob (
 			filepath = self.physical_location + os.sep + 'metadata.xml',
@@ -250,6 +260,7 @@ class PackageDir ( object ):
 		self._need_manifest = True
 		self._need_metadata = True
 		self.modified       = True
+		# FIXME: delete PackageInfo if ebuild creation did not succeed
 		if self.runtime_incremental:
 			with self._lock:
 				return self.write_ebuilds ( overwrite=False )

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 219fd70..ae3267f 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -22,23 +22,49 @@ class Overlay ( object ):
 
 	def __init__ (
 		self,
-		name, logger, directory,
-		default_category, eclass_files,
+		name,
+		logger,
+		directory,
+		default_category,
+		eclass_files,
 		ebuild_header,
-		incremental
+		write_allowed,
+		incremental,
+		runtime_incremental=True
 	):
-		self.name              = name
-		self.logger            = logger.getChild ( 'overlay' )
-		self.physical_location = directory
-		self.default_category  = default_category
-		self.eclass_files      = eclass_files
+		"""Initializes an overlay.
 
-		self.ignore_existing_ebuilds = False
-
-		self._profiles_dir     = self.physical_location + os.sep + 'profiles'
-		self._catlock          = threading.Lock()
-		self._categories       = dict()
-		self._header           = EbuildHeader ( ebuild_header )
+		arguments:
+		* name                -- name of this overlay
+		* logger              -- parent logger to use
+		* directory           -- filesystem location of this overlay
+		* default_category    -- category of packages being added without a
+		                          specific category
+		* eclass_files        -- eclass files to import and
+		                          inherit in all ebuilds
+		* ebuild_header       -- the header text included in all created ebuilds
+		* write_allowed       -- whether writing is allowed
+		* incremental         -- enable/disable incremental writing:
+		                         use already existing ebuilds (don't recreate
+		                         them)
+		* runtime_incremental -- see package.py:PackageDir.__init__ (...),
+		                          Defaults to ?FIXME?
+		"""
+		self.name                 = name
+		self.logger               = logger.getChild ( 'overlay' )
+		self.physical_location    = directory
+		self.default_category     = default_category
+
+		self._eclass_files        = eclass_files
+		#self._incremental         = incremental
+		# disable runtime_incremental if writing not allowed
+		self._runtime_incremental = write_allowed and runtime_incremental
+		self._writeable           = write_allowed
+
+		self._profiles_dir        = self.physical_location + os.sep + 'profiles'
+		self._catlock             = threading.Lock()
+		self._categories          = dict()
+		self._header              = EbuildHeader ( ebuild_header )
 
 		# fixme or ignore: calculating eclass names twice,
 		# once here and another time when calling _init_overlay
@@ -46,13 +72,12 @@ class Overlay ( object ):
 			self._get_eclass_import_info ( only_eclass_names=True )
 		) )
 
-		self.incremental = incremental
-		if self.incremental:
+		#if self._incremental:
+		if incremental:
 			# this is multiple-run incremental writing (in contrast to runtime
 			# incremental writing, which writes ebuilds as soon as they're
-			# ready) FIXME: split incremental <-> runtime_incremental
+			# ready)
 			self.scan()
-
 	# --- end of __init__ (...) ---
 
 	def _get_category ( self, category ):
@@ -70,7 +95,7 @@ class Overlay ( object ):
 						self.logger,
 						self.physical_location + os.sep + category,
 						get_header=self._header.get,
-						incremental=self.incremental
+						runtime_incremental=self._runtime_incremental
 					)
 					self._categories [category] = newcat
 			finally:
@@ -89,9 +114,9 @@ class Overlay ( object ):
 
 		raises: AssertionError if a file does not end with '.eclass'.
 		"""
-		if self.eclass_files:
+		if self._eclass_files:
 
-			for eclass in self.eclass_files:
+			for eclass in self._eclass_files:
 				dest = os.path.splitext ( os.path.basename ( eclass ) )
 
 				if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ):
@@ -116,7 +141,7 @@ class Overlay ( object ):
 		* Exception if copying fails
 		"""
 
-		if self.eclass_files:
+		if self._eclass_files:
 			# import eclass files
 			eclass_dir = self.physical_location + os.sep +  'eclass'
 			try:
@@ -240,6 +265,10 @@ class Overlay ( object ):
 				yield kwargs
 	# --- end of list_rule_kwargs (...) ---
 
+	def readonly ( self ):
+		return not self._writeable
+	# --- end of readonly (...) ---
+
 	def scan ( self, **kw ):
 		def scan_categories():
 			for x in os.listdir ( self.physical_location ):
@@ -271,6 +300,10 @@ class Overlay ( object ):
 			cat.show ( **show_kw )
 	# --- end of show (...) ---
 
+	def writeable ( self ):
+		return self._writeable
+	# --- end of writeable (...) ---
+
 	def write ( self ):
 		"""Writes the overlay to its physical location (filesystem), including
 		metadata and Manifest files as well as cleanup actions.
@@ -285,14 +318,21 @@ class Overlay ( object ):
 		! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
 		when ebuild creation is done.
 		"""
-		self._init_overlay ( reimport_eclass=True )
-
-		for cat in self._categories.values():
-			cat.write (
-				overwrite_ebuilds=False,
-				keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
-				cautious=True
-			)
+		if self._writeable:
+			self._init_overlay ( reimport_eclass=True )
+
+			for cat in self._categories.values():
+				cat.write (
+					overwrite_ebuilds=False,
+					keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
+					cautious=True
+				)
+		else:
+			# FIXME debug print
+			print (
+				"Dropped write request for readonly overlay {}!".format (
+					self.name
+			) )
 	# --- end of write (...) ---
 
 	def write_manifest ( self, **manifest_kw ):
@@ -305,7 +345,14 @@ class Overlay ( object ):
 
 		returns: None (implicit)
 		"""
-		# FIXME: it would be good to ensure that profiles/categories exist
-		for cat in self._categories.values():
-			cat.write_manifest ( **manifest_kw )
+		if self._writeable:
+			# FIXME: it would be good to ensure that profiles/categories exist
+			for cat in self._categories.values():
+				cat.write_manifest ( **manifest_kw )
+		else:
+			# FIXME debug print
+			print (
+				"Dropped write_manifest request for readonly overlay {}!".format (
+					self.name
+			) )
 	# --- end of write_manifest (...) ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     daed3bde53ce1fec42ad806813bb2798321b76d4
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:21:07 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:21:07 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=daed3bde

overlay: split incremental/runtime_incremental

Also added a _writeable bool to Overlay which controls
whether writing is allowed or not.

	geändert:   roverlay/overlay/category.py
	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/package.py
	geändert:   roverlay/overlay/root.py

---
 roverlay/overlay/category.py |   36 +++++++++-----
 roverlay/overlay/creator.py  |   20 +++----
 roverlay/overlay/package.py  |   23 ++++++--
 roverlay/overlay/root.py     |  113 +++++++++++++++++++++++++++++------------
 4 files changed, 129 insertions(+), 63 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index fab6373..a0b1607 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -16,25 +16,35 @@ class Category ( object ):
 
 	WRITE_JOBCOUNT = 3
 
-	def __init__ ( self, name, logger, directory, get_header, incremental ):
+	def __init__ ( self,
+		name, logger, directory, get_header, runtime_incremental
+	):
 		"""Initializes a overlay/portage category (such as 'app-text', 'sci-R').
 
 		arguments:
-		* name       -- name of the category
-		* logger     -- parent logger
-		* directory  -- filesystem location
-		* get_header -- function that returns an ebuild header
+		* name                -- name of the category
+		* logger              -- parent logger
+		* directory           -- filesystem location
+		* get_header          -- function that returns an ebuild header
+		* runtime_incremental -- enable/disable runtime incremental writing
+		                         for this category (and all created PackageDirs)
 		"""
-		self.logger            = logger.getChild ( name )
-		self.name              = name
-		self._lock             = threading.RLock()
-		self._subdirs          = dict()
-		self.physical_location = directory
-		self.get_header        = get_header
-		self.incremental       = incremental
+		self.logger              = logger.getChild ( name )
+		self.name                = name
+		self._lock               = threading.RLock()
+		self._subdirs            = dict()
+		self.physical_location   = directory
+		self.get_header          = get_header
+		self.runtime_incremental = runtime_incremental
 	# --- end of __init__ (...) ---
 
 	def _get_package_dir ( self, pkg_name ):
+		"""Returns a PackageDir object for pkg_name.
+		(so that <new object>.name == pkg_name and pkg_name in self._subdirs)
+
+		arguments:
+		* pkg_name --
+		"""
 		if not pkg_name in self._subdirs:
 			self._lock.acquire()
 			try:
@@ -44,7 +54,7 @@ class Category ( object ):
 						logger      = self.logger,
 						directory   = self.physical_location + os.sep + pkg_name,
 						get_header  = self.get_header,
-						incremental = self.incremental
+						runtime_incremental = self.runtime_incremental
 					)
 					self._subdirs [pkg_name] = newpkg
 			finally:

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index d12e0a5..60e9517 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -82,18 +82,16 @@ class OverlayCreator ( object ):
 		# this queue is used to propagate exceptions from threads
 		self._err_queue = errorqueue.ErrorQueue()
 
-		self.can_write_overlay = allow_write
-		self.write_incremental = allow_write and USE_INCREMENTAL_WRITE
-
 		# init overlay using config values
 		self.overlay = Overlay (
-			name=config.get_or_fail ( 'OVERLAY.name' ),
-			logger=self.logger,
-			directory=config.get_or_fail ( 'OVERLAY.dir' ),
-			default_category= config.get_or_fail ( 'OVERLAY.category' ),
-			eclass_files=config.get ( 'OVERLAY.eclass_files', None ),
-			ebuild_header=config.get ( 'EBUILD.default_header', None ),
-			incremental=self.write_incremental
+			name             = config.get_or_fail ( 'OVERLAY.name' ),
+			logger           = self.logger,
+			directory        = config.get_or_fail ( 'OVERLAY.dir' ),
+			default_category = config.get_or_fail ( 'OVERLAY.category' ),
+			eclass_files     = config.get ( 'OVERLAY.eclass_files', None ),
+			ebuild_header    = config.get ( 'EBUILD.default_header', None ),
+			incremental      = USE_INCREMENTAL_WRITE,
+			write_allowed    = allow_write
 		)
 
 		self.depresolver = easyresolver.setup ( self._err_queue )
@@ -219,7 +217,7 @@ class OverlayCreator ( object ):
 
 		arguments:
 		"""
-		if self.can_write_overlay:
+		if self.overlay.writeable():
 			start = time.time()
 			self.overlay.write()
 			self._timestamp ( "overlay written", start )

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index d2662bd..bedbce3 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -13,15 +13,25 @@ SUPPRESS_EXCEPTIONS = True
 class PackageDir ( object ):
 	EBUILD_SUFFIX = '.ebuild'
 
-	def __init__ ( self, name, logger, directory, get_header, incremental ):
+	def __init__ ( self,
+		name, logger, directory, get_header, runtime_incremental
+	):
 		"""Initializes a PackageDir which contains ebuilds, metadata and
 		a Manifest file.
 
 		arguments:
-		* name       -- name of the directory (${PN} in ebuilds)
-		* logger     -- parent logger
-		* directory  -- filesystem location of this PackageDir
-		* get_header -- function that returns an ebuild header
+		* name                -- name of the directory (${PN} in ebuilds)
+		* logger              -- parent logger
+		* directory           -- filesystem location of this PackageDir
+		* get_header          -- function that returns an ebuild header
+		* runtime_incremental -- enable/disable runtime incremental ebuild
+		                         writing. This trades speed (disabled) for
+		                         memory consumption (enabled) 'cause it will
+		                         write _all_ successfully created ebuilds
+		                         directly after they've been created.
+		                         Writing all ebuilds at once is generally faster
+		                         (+threading), but all PackageInfos must be
+		                         kept in memory for that.
 		"""
 		self.logger              = logger.getChild ( name )
 		self.name                = name
@@ -30,7 +40,7 @@ class PackageDir ( object ):
 		self._packages           = dict()
 		self.physical_location   = directory
 		self.get_header          = get_header
-		self.runtime_incremental = incremental
+		self.runtime_incremental = runtime_incremental
 
 		self._metadata = MetadataJob (
 			filepath = self.physical_location + os.sep + 'metadata.xml',
@@ -250,6 +260,7 @@ class PackageDir ( object ):
 		self._need_manifest = True
 		self._need_metadata = True
 		self.modified       = True
+		# FIXME: delete PackageInfo if ebuild creation did not succeed
 		if self.runtime_incremental:
 			with self._lock:
 				return self.write_ebuilds ( overwrite=False )

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 219fd70..ae3267f 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -22,23 +22,49 @@ class Overlay ( object ):
 
 	def __init__ (
 		self,
-		name, logger, directory,
-		default_category, eclass_files,
+		name,
+		logger,
+		directory,
+		default_category,
+		eclass_files,
 		ebuild_header,
-		incremental
+		write_allowed,
+		incremental,
+		runtime_incremental=True
 	):
-		self.name              = name
-		self.logger            = logger.getChild ( 'overlay' )
-		self.physical_location = directory
-		self.default_category  = default_category
-		self.eclass_files      = eclass_files
+		"""Initializes an overlay.
 
-		self.ignore_existing_ebuilds = False
-
-		self._profiles_dir     = self.physical_location + os.sep + 'profiles'
-		self._catlock          = threading.Lock()
-		self._categories       = dict()
-		self._header           = EbuildHeader ( ebuild_header )
+		arguments:
+		* name                -- name of this overlay
+		* logger              -- parent logger to use
+		* directory           -- filesystem location of this overlay
+		* default_category    -- category of packages being added without a
+		                          specific category
+		* eclass_files        -- eclass files to import and
+		                          inherit in all ebuilds
+		* ebuild_header       -- the header text included in all created ebuilds
+		* write_allowed       -- whether writing is allowed
+		* incremental         -- enable/disable incremental writing:
+		                         use already existing ebuilds (don't recreate
+		                         them)
+		* runtime_incremental -- see package.py:PackageDir.__init__ (...),
+		                          Defaults to ?FIXME?
+		"""
+		self.name                 = name
+		self.logger               = logger.getChild ( 'overlay' )
+		self.physical_location    = directory
+		self.default_category     = default_category
+
+		self._eclass_files        = eclass_files
+		#self._incremental         = incremental
+		# disable runtime_incremental if writing not allowed
+		self._runtime_incremental = write_allowed and runtime_incremental
+		self._writeable           = write_allowed
+
+		self._profiles_dir        = self.physical_location + os.sep + 'profiles'
+		self._catlock             = threading.Lock()
+		self._categories          = dict()
+		self._header              = EbuildHeader ( ebuild_header )
 
 		# fixme or ignore: calculating eclass names twice,
 		# once here and another time when calling _init_overlay
@@ -46,13 +72,12 @@ class Overlay ( object ):
 			self._get_eclass_import_info ( only_eclass_names=True )
 		) )
 
-		self.incremental = incremental
-		if self.incremental:
+		#if self._incremental:
+		if incremental:
 			# this is multiple-run incremental writing (in contrast to runtime
 			# incremental writing, which writes ebuilds as soon as they're
-			# ready) FIXME: split incremental <-> runtime_incremental
+			# ready)
 			self.scan()
-
 	# --- end of __init__ (...) ---
 
 	def _get_category ( self, category ):
@@ -70,7 +95,7 @@ class Overlay ( object ):
 						self.logger,
 						self.physical_location + os.sep + category,
 						get_header=self._header.get,
-						incremental=self.incremental
+						runtime_incremental=self._runtime_incremental
 					)
 					self._categories [category] = newcat
 			finally:
@@ -89,9 +114,9 @@ class Overlay ( object ):
 
 		raises: AssertionError if a file does not end with '.eclass'.
 		"""
-		if self.eclass_files:
+		if self._eclass_files:
 
-			for eclass in self.eclass_files:
+			for eclass in self._eclass_files:
 				dest = os.path.splitext ( os.path.basename ( eclass ) )
 
 				if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ):
@@ -116,7 +141,7 @@ class Overlay ( object ):
 		* Exception if copying fails
 		"""
 
-		if self.eclass_files:
+		if self._eclass_files:
 			# import eclass files
 			eclass_dir = self.physical_location + os.sep +  'eclass'
 			try:
@@ -240,6 +265,10 @@ class Overlay ( object ):
 				yield kwargs
 	# --- end of list_rule_kwargs (...) ---
 
+	def readonly ( self ):
+		return not self._writeable
+	# --- end of readonly (...) ---
+
 	def scan ( self, **kw ):
 		def scan_categories():
 			for x in os.listdir ( self.physical_location ):
@@ -271,6 +300,10 @@ class Overlay ( object ):
 			cat.show ( **show_kw )
 	# --- end of show (...) ---
 
+	def writeable ( self ):
+		return self._writeable
+	# --- end of writeable (...) ---
+
 	def write ( self ):
 		"""Writes the overlay to its physical location (filesystem), including
 		metadata and Manifest files as well as cleanup actions.
@@ -285,14 +318,21 @@ class Overlay ( object ):
 		! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
 		when ebuild creation is done.
 		"""
-		self._init_overlay ( reimport_eclass=True )
-
-		for cat in self._categories.values():
-			cat.write (
-				overwrite_ebuilds=False,
-				keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
-				cautious=True
-			)
+		if self._writeable:
+			self._init_overlay ( reimport_eclass=True )
+
+			for cat in self._categories.values():
+				cat.write (
+					overwrite_ebuilds=False,
+					keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
+					cautious=True
+				)
+		else:
+			# FIXME debug print
+			print (
+				"Dropped write request for readonly overlay {}!".format (
+					self.name
+			) )
 	# --- end of write (...) ---
 
 	def write_manifest ( self, **manifest_kw ):
@@ -305,7 +345,14 @@ class Overlay ( object ):
 
 		returns: None (implicit)
 		"""
-		# FIXME: it would be good to ensure that profiles/categories exist
-		for cat in self._categories.values():
-			cat.write_manifest ( **manifest_kw )
+		if self._writeable:
+			# FIXME: it would be good to ensure that profiles/categories exist
+			for cat in self._categories.values():
+				cat.write_manifest ( **manifest_kw )
+		else:
+			# FIXME debug print
+			print (
+				"Dropped write_manifest request for readonly overlay {}!".format (
+					self.name
+			) )
 	# --- end of write_manifest (...) ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     9d2a243a6e6eadc71e3962726b29c84d1251d732
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul  5 15:59:40 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul  5 15:59:40 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9d2a243a

overlay: calculate inherit statement for ebuilds

	modified:   roverlay/overlay/__init__.py

---
 roverlay/overlay/__init__.py |   83 +++++++++++++++++++++++++++++++++---------
 1 files changed, 66 insertions(+), 17 deletions(-)

diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py
index 13b6615..b6a65a1 100644
--- a/roverlay/overlay/__init__.py
+++ b/roverlay/overlay/__init__.py
@@ -60,6 +60,7 @@ class Overlay ( object ):
 		else:
 			self.eclass_files = eclass_files
 
+		self.eclass_names = None
 
 		#
 		self._profiles_dir = os.path.join ( self.physical_location, 'profiles' )
@@ -115,7 +116,7 @@ class Overlay ( object ):
 		returns: None (implicit)
 		"""
 		for cat in self._categories.values():
-			cat.show ( default_header=self._default_header )
+			cat.show ( default_header=self._get_header() )
 	# --- end of show (...) ---
 
 	def write ( self, **write_kw ):
@@ -138,7 +139,7 @@ class Overlay ( object ):
 		for cat in self._categories.values():
 			if cat.physical_location and not cat.empty():
 				util.dodir ( cat.physical_location )
-				cat.write ( default_header=self._default_header )
+				cat.write ( default_header=self._get_header() )
 
 		self._write_categories ( only_active=True )
 	# --- end of write (...) ---
@@ -253,29 +254,58 @@ class Overlay ( object ):
 			self._write_profiles_file ( 'use.desc', use_desc + '\n' )
 	# --- end of _write_usedesc (...) ---
 
+	def _get_eclass_import_info ( self, only_eclass_names=False ):
+		"""Yields eclass import information (eclass names and files).
+
+		arguments:
+		* only_eclass_names -- if True: yield eclass dest names only,
+		                       else   : yield (eclass name, eclass src file)
+		                        Defaults to False.
+
+		raises: AssertionError if a file does not end with '.eclass'.
+		"""
+		if self.eclass_files:
+
+			for eclass in self.eclass_files:
+				dest = os.path.splitext ( os.path.basename ( eclass ) )
+
+				if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ):
+					if only_eclass_names:
+						yield dest[0]
+					else:
+						yield ( dest[0], eclass )
+				else:
+					raise AssertionError (
+						"{!r} does not end with '.eclass'!".format ( eclass )
+					)
+	# --- end of _get_eclass_import_info (...) ---
+
 	def _import_eclass ( self, reimport_eclass ):
+		"""Imports eclass files to the overlay. Also sets ebuild_names.
+
+		arguments:
+		* reimport_eclass -- whether to import existing eclass files (again)
+
+		raises:
+		* AssertionError, passed from _get_eclass_import_info()
+		* Exception if copying fails
+		"""
 
 		if self.eclass_files:
 			# import eclass files
 			eclass_dir = os.path.join ( self.physical_location, 'eclass' )
 			try:
+				eclass_names = list()
 				util.dodir ( eclass_dir )
 
-				for eclass in self.eclass_files:
-					src  = eclass
-					dest = None
-					if isinstance ( eclass, str ):
-						dest = os.path.basename ( eclass )
-					else:
-						# list-like specification ( src, destname )
-						src  = eclass [0]
-						dest = eclass [1]
-
-					dest = os.path.join ( eclass_dir, dest )
-
+				for destname, eclass in self._get_eclass_import_info ( False ):
+					dest = os.path.join ( eclass_dir, destname + '.eclass' )
 					if reimport_eclass or not os.path.isfile ( dest ):
-						shutil.copyfile ( src, dest )
+						shutil.copyfile ( eclass, dest )
 
+					eclass_names.append ( destname )
+
+				self.eclass_names = frozenset ( eclass_names )
 
 			except Exception as e:
 				self.logger.critical ( "Cannot import eclass files!" )
@@ -311,10 +341,29 @@ class Overlay ( object ):
 			self.logger.exception ( e )
 			self.logger.critical ( "^failed to init overlay" )
 			raise
+	# --- end of _init_overlay (...) ---
+
+	def _get_header ( self ):
+		"""Returns the ebuild header (including inherit <eclasses>)."""
+		if self.eclass_names is None:
+				# writing is possibly disabled since eclass files have not been
+				# imported (or show() used before write())
+			inherit = ' '.join ( self._get_eclass_import_info ( True ) )
+		else:
+			inherit = ' '.join ( self.eclass_names )
 
+		inherit = "inherit " + inherit if inherit else None
 
+		# header and inherit is expected and therefore the first condition here
+		if inherit and self._default_header:
+			return '\n'.join (( self._default_header, '', inherit ))
 
+		elif inherit:
+			return inherit
 
+		elif self._default_header:
+			return self._default_header
 
-
-
+		else:
+			return None
+	# --- end of _get_header (...) ---

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     6d1ec878f71025e6681255f8b63cd404f9ce74f3
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul  4 18:10:14 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul  4 18:10:14 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6d1ec878

overlay: collect some stats

	modified:   roverlay/overlay/__init__.py
	modified:   roverlay/overlay/category.py
	modified:   roverlay/overlay/creator.py
	modified:   roverlay/overlay/package.py

---
 roverlay/overlay/__init__.py |    4 +-
 roverlay/overlay/category.py |    4 +-
 roverlay/overlay/creator.py  |  167 +++++++++++++++++++++++++++++++++++++++---
 roverlay/overlay/package.py  |   22 +++---
 4 files changed, 171 insertions(+), 26 deletions(-)

diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py
index a8a939d..13b6615 100644
--- a/roverlay/overlay/__init__.py
+++ b/roverlay/overlay/__init__.py
@@ -99,9 +99,9 @@ class Overlay ( object ):
 		* category     -- category where the pkg should be put in, defaults to
 		                   self.default_category
 
-		returns: None (implicit)
+		returns: True if successfully added else False
 		"""
-		self._get_category (
+		return self._get_category (
 			self.default_category if category is None else category
 		) . add ( package_info )
 	# --- end of add (...) ---

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index e938dbf..271b524 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -46,7 +46,7 @@ class Category ( object ):
 		arguments:
 		* package_info --
 
-		returns: None (implicit)
+		returns: success
 		"""
 		pkg_name = package_info ['name']
 
@@ -63,7 +63,7 @@ class Category ( object ):
 			finally:
 				self._lock.release()
 
-		self._subdirs [pkg_name].add ( package_info )
+		return self._subdirs [pkg_name].add ( package_info )
 	# --- end of add (...) ---
 
 	def generate_metadata ( self, **metadata_kw ):

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 79f2da3..bb2391b 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -5,10 +5,12 @@
 import time
 import logging
 import threading
-import signal
-import traceback
+#import signal
+#import traceback
 import sys
 
+from collections import deque
+
 try:
 	import queue
 except ImportError:
@@ -23,18 +25,61 @@ from roverlay.packageinfo        import PackageInfo
 
 from roverlay.recipe             import easyresolver
 
-LOGGER = logging.getLogger ( 'OverlayCreator' )
-
+# this is used as the default value for can_write_overlay in OverlayCreator
+# instances
 OVERLAY_WRITE_ALLOWED = False
 
+class PseudoAtomicCounter ( object ):
+
+	def __init__ ( self, start=0, long_int=False ):
+		if long_int and sys.version_info < ( 3, 0 ):
+			self._value = long ( start )
+		else:
+			self._value = int ( start )
+		self._lock  = threading.Lock()
+
+	def _get_and_inc ( self, step ):
+		ret = None
+		self._lock.acquire()
+		try:
+			old_val = self._value
+			if step > 0:
+				self._value += step
+				ret = ( self._value, old_val )
+			else:
+				ret = old_val
+		finally:
+			self._lock.release()
+
+		return ret
+	# --- end of _get_and_inc (...) ---
+
+	def inc ( self, step=1 ):
+		self._get_and_inc ( step )
+	# --- end of inc (...) ---
+
+	def get ( self ):
+		return self._get_and_inc ( 0 )
+	# --- end of get (...) ---
+
+	def get_nowait ( self ):
+		return self._value
+	# --- end of get_nowait (...) ---
+
+	def __str__ ( self ):
+		return str ( self._value )
+	# --- end of __str__ (...) ---
+
 
 class OverlayCreator ( object ):
 	"""This is a 'R packages -> Overlay' interface."""
 
+	LOGGER = logging.getLogger ( 'OverlayCreator' )
+
 	def __init__ ( self, logger=None ):
 
 		if logger is None:
-			self.logger = LOGGER
+			self.logger = self.__class__.LOGGER
 		else:
 			self.logger = logger.getChild ( 'OverlayCreator' )
 
@@ -52,19 +97,97 @@ class OverlayCreator ( object ):
 		#  it's
 		self._err_queue = queue.Queue()
 
+		#self._time_start_run = list()
+		#self._time_stop_run  = list()
+
 
 		self._workers   = None
 		self._runlock   = threading.RLock()
 
 		self.can_write_overlay = OVERLAY_WRITE_ALLOWED
 
-		# this is a method that adds PackageInfo objects to the pkg queue
-		self.add_package = self._pkg_queue.put
 
 		self.depresolver.set_exception_queue ( self._err_queue )
 
+		self.closed = False
+
+
+		# queued packages counter,
+		#  package_added != (create_success + create_fail) if a thread hangs
+		#  or did not call _pkg_done
+		self.package_added  = PseudoAtomicCounter()
+
+		# counts packages that passed ebuild creation
+		self.create_success = PseudoAtomicCounter()
+
+		# counts packages that failed ebuild creation
+		self.create_fail    = PseudoAtomicCounter()
+
+		# counts packages that passed adding to overlay
+		self.overlay_added  = PseudoAtomicCounter()
+
 	# --- end of __init__ (...) ---
 
+	def get_stats ( self ):
+		pkg_added   = self.package_added.get_nowait()
+		pkg_created = self.create_success.get_nowait()
+		pkg_failed  = self.create_fail.get_nowait()
+		ov_added    = self.overlay_added.get_nowait()
+		ov_failed   = pkg_created - ov_added
+		processed   = pkg_created + pkg_failed
+		failed      = pkg_failed + ov_failed
+
+
+		# namedtuple? TODO
+
+		return (
+			pkg_added, pkg_created, pkg_failed,
+			ov_added, ov_failed,
+			processed, failed
+		)
+	# --- end of get_stats (...) ---
+
+	def stats_str ( self, enclose=True ):
+		"""Returns a string with some overlay creation stats."""
+		def stats_gen():
+			"""Yields stats strings."""
+			stats = self.get_stats()
+
+			# the length of the highest number in stats (^=digit count)
+			# max_number_len := { 1,...,5 }
+			max_number_len = min ( 5, len ( str ( max ( stats ) ) ) )
+			num_fmt = '%(num)-' + str ( max_number_len ) + 's '
+
+			for i, s in enumerate ((
+				'packages added to the ebuild creation queue',
+				'packages passed ebuild creation',
+				'packages failed ebuild creation',
+				'ebuilds could be added to the overlay',
+				'ebuilds couldn\'t be added to the overlay',
+				'packages processed in total',
+				'packages failed in total',
+			)):
+				yield num_fmt % { 'num' : stats [i] } + s
+		# --- end of stats_gen (...) ---
+
+		if enclose:
+			stats_str = deque ( stats_gen() )
+			# maxlen := { 2,...,80 }
+			maxlen = 2 + min ( 78,
+				len ( max ( stats_str, key=lambda s : len( s ) ) )
+			)
+
+			stats_str.appendleft (
+				" Overlay creation stats ".center ( maxlen, '-' )
+			)
+			stats_str.append ( '-' * maxlen )
+
+			return '\n'.join ( stats_str )
+
+		else:
+			return '\n'.join ( stats_gen() )
+	# --- end of stats_str (...) ---
+
 	def _timestamp ( self, description, start, stop=None ):
 		"""Logs a timestamp, used for testing.
 
@@ -82,14 +205,25 @@ class OverlayCreator ( object ):
 		return _stop
 	# --- end of _timestamp (...) ---
 
+	def add_package ( self, package_info ):
+		"""Adds a PackageInfo to the package queue.
+
+		arguments:
+		* package_info --
+		"""
+		self._pkg_queue.put ( package_info )
+		self.package_added.inc()
+
 	def add_package_file ( self, package_file ):
 		"""Adds a single R package."""
 		self._pkg_queue.put ( PackageInfo ( filepath=package_file ) )
+		self.package_added.inc()
 	# --- end of add_package (...) ---
 
 	def add_package_files ( self, *package_files ):
 		"""Adds multiple R packages."""
 		for p in package_files: self.add_package_file ( p )
+		self.package_added.inc()
 	# --- end of add_packages (...) ---
 
 
@@ -117,14 +251,18 @@ class OverlayCreator ( object ):
 		self.overlay.show()
 	# --- end of show_overlay (...) ---
 
-	def run ( self ):
+	def run ( self, close_when_done=False ):
 		"""Starts ebuild creation and waits until done."""
 		self._runlock.acquire()
+		#self._time_start_run.append ( time.time() )
 		try:
 			self.start()
 			self.join()
 		finally:
+			#self._time_stop_run.append ( time.time() )
 			self._runlock.release()
+			if close_when_done:
+				self.close()
 	# --- end of run (...) ---
 
 	def start ( self ):
@@ -145,11 +283,11 @@ class OverlayCreator ( object ):
 		self._join_workers()
 	# --- end of wait (...) ---
 
-	def close ( self, write=False ):
+	def close ( self ):
 		"""Closes this OverlayCreator."""
 		self._close_workers()
 		self._close_resolver()
-		if write: self.write_overlay()
+		self.closed = True
 	# --- end of close (...) ---
 
 	def _close_resolver ( self ):
@@ -266,8 +404,13 @@ class OverlayCreator ( object ):
 		#  * request an incremental write to save memory etc.
 
 		# if <>:
-		if package_info ['ebuild'] is not None:
-			self.overlay.add ( package_info )
+		if package_info ['ebuild'] is None:
+			self.create_fail.inc()
+		else:
+			self.create_success.inc()
+			if self.overlay.add ( package_info ):
+				self.overlay_added.inc()
+
 	# --- end of _add_to_overlay (...) ---
 
 	def _get_worker ( self, start_now=False, use_threads=True ):

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 77fcbee..429200f 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -203,11 +203,9 @@ class PackageDir ( object ):
 		"""
 		shortver = package_info ['ebuild_verstr']
 
-		def already_exists ( release=False ):
+		def already_exists ():
 			if shortver in self._packages:
 
-				if release: self._lock.release()
-
 				msg = "'%s-%s.ebuild' already exists, cannot add it!" % (
 					self.name, shortver
 				)
@@ -221,14 +219,18 @@ class PackageDir ( object ):
 				return False
 		# --- end of already_exists (...) ---
 
-		if already_exists ( release=False ): return False
-		self._lock.acquire()
-		if already_exists ( release=True  ): return False
-
-		self._packages [shortver] = package_info
+		_success = False
 
-		self._lock.release()
-		return True
+		if not already_exists():
+			try:
+				self._lock.acquire()
+				if not already_exists():
+					self._packages [shortver] = package_info
+					_success = True
+			finally:
+				self._lock.release()
+
+		return _success
 	# --- end of add (...) ---
 
 	def _regen_metadata ( self ):

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     396f148a3912c82357e68533eb6858a252ca7307
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul  3 17:41:06 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul  3 17:41:06 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=396f148a

reduce default numthreads to 0

---
 roverlay/overlay/creator.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index aa1a3a8..79f2da3 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -43,7 +43,7 @@ class OverlayCreator ( object ):
 
 		self.depresolver = easyresolver.setup()
 
-		self.NUMTHREADS  = config.get ( 'EBUILD.jobcount', 10 )
+		self.NUMTHREADS  = config.get ( 'EBUILD.jobcount', 0 )
 
 		# --
 		self._pkg_queue = queue.Queue()

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     d145bb5b3a9f0fc8f93d52cfe8c3002cc1bb416d
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun 27 14:40:43 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun 27 14:40:43 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d145bb5b

typo

---
 roverlay/overlay/creator.py |    5 +++--
 1 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 6d97d48..608ba77 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -47,10 +47,11 @@ class OverlayCreator ( object ):
 		self._workers    = None
 		self._runlock    = threading.RLock()
 
-		self.package_add = self._pkg_queue.put
-
 		self.can_write_overlay = OVERLAY_WRITE_ALLOWED
 
+		# this is a method that adds PackageInfo objects to the pkg queue
+		self.add_package = self._pkg_queue.put
+
 	# --- end of __init__ (...) ---
 
 	def _timestamp ( self, description, start, stop=None ):

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     544b6f2f6668a3bb254209a7afe498f9925fe2cb
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jun 26 15:39:51 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jun 26 15:39:51 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=544b6f2f

overlay creation: unused method stubs removed

	modified:   roverlay/overlay/creator.py

---
 roverlay/overlay/creator.py |   16 ++--------------
 1 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index bbaaa8e..6d97d48 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -47,6 +47,8 @@ class OverlayCreator ( object ):
 		self._workers    = None
 		self._runlock    = threading.RLock()
 
+		self.package_add = self._pkg_queue.put
+
 		self.can_write_overlay = OVERLAY_WRITE_ALLOWED
 
 	# --- end of __init__ (...) ---
@@ -68,11 +70,6 @@ class OverlayCreator ( object ):
 		return _stop
 	# --- end of _timestamp (...) ---
 
-	def add ( self, *to_add ):
-		"""Add a directory/package files/distdir to the package queue."""
-		raise Exception ( "method stub" )
-	# --- end of add (...) ---
-
 	def add_package_file ( self, package_file ):
 		"""Adds a single R package."""
 		self._pkg_queue.put ( PackageInfo ( filepath=package_file ) )
@@ -83,15 +80,6 @@ class OverlayCreator ( object ):
 		for p in package_files: self.add_package_file ( p )
 	# --- end of add_packages (...) ---
 
-	def add_directory ( self, directory ):
-		"""Adds all packages from a directory to the package queue."""
-		raise Exception ( "method stub" )
-	# --- end of add_directory (...) ---
-
-	def add_distdir ( self, repo_name ):
-		"""Adds a distfiles directory to the package queue."""
-		self.add_directory ( config.get_or_fail ( [ 'DISTFILES', repo_name ] ) )
-	# --- end of add_distdir (...) ---
 
 	def write_overlay ( self, incremental=False ):
 		"""Writes the overlay.

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     9646d809ea212ce68a04904a2d3f2d10d3fed718
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jun 22 18:04:31 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jun 22 18:04:31 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9646d809

fix wait() in overlay creation

* workers now run until the queue is empty after receiving stop_when_empty()
  instead of stopping after the next package (which is what close() does)

* added some comments in creator,worker

	modified:   overlay/creator.py
	modified:   overlay/worker.py

---
 roverlay/overlay/creator.py |   71 ++++++++++++++++++++++++++++++++++++++-----
 roverlay/overlay/worker.py  |   71 ++++++++++++++++++++++++++++++++++--------
 2 files changed, 120 insertions(+), 22 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 0936ca9..bbaaa8e 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -26,6 +26,7 @@ OVERLAY_WRITE_ALLOWED = False
 
 
 class OverlayCreator ( object ):
+	"""This is a 'R packages -> Overlay' interface."""
 
 	def __init__ ( self, logger=None ):
 
@@ -51,7 +52,13 @@ class OverlayCreator ( object ):
 	# --- end of __init__ (...) ---
 
 	def _timestamp ( self, description, start, stop=None ):
+		"""Logs a timestamp, used for testing.
 
+		arguments:
+		* description -- timestamp text
+		* start       -- when measuring for this timestamp has been started
+		* stop        -- stop time; defaults to now (time.time()) if unset
+		"""
 		_stop = time.time() if stop is None else stop
 		delta = _stop - start
 
@@ -87,6 +94,11 @@ class OverlayCreator ( object ):
 	# --- end of add_distdir (...) ---
 
 	def write_overlay ( self, incremental=False ):
+		"""Writes the overlay.
+
+		arguments:
+		* incremental -- (TODO)
+		"""
 		if self.can_write_overlay:
 			start = time.time()
 			if incremental:
@@ -101,10 +113,12 @@ class OverlayCreator ( object ):
 	# --- end of write_overlay (...) ---
 
 	def show_overlay ( self ):
+		"""Prints the overlay to the console. Does not create Manifest files."""
 		self.overlay.show()
 	# --- end of show_overlay (...) ---
 
 	def run ( self ):
+		"""Starts ebuild creation and waits until done."""
 		self._runlock.acquire()
 		self.start()
 		self.join()
@@ -112,25 +126,30 @@ class OverlayCreator ( object ):
 	# --- end of run (...) ---
 
 	def start ( self ):
+		"""Starts ebuild creation."""
 		self._runlock.acquire()
 		self.join()
 		self._make_workers()
 		self._runlock.release()
 	# --- end of start (...) ---
 
-	def wait ( self ):
-		self._close_workers()
+	def join ( self ):
+		"""Waits until ebuild creation is done."""
+		self._join_workers()
 	# --- end of wait (...) ---
 
-	join = wait
-
 	def close ( self, write=False ):
+		"""Closes this OverlayCreator."""
 		self._close_workers()
 		self._close_resolver()
 		if write: self.write_overlay()
 	# --- end of close (...) ---
 
 	def _close_resolver ( self ):
+		"""Tells the dependency resolver to close.
+		This is useful 'cause certain depres listener modules will write files
+		when told to exit.
+		"""
 		if self.depresolver is None: return
 
 		self._runlock.acquire()
@@ -143,7 +162,12 @@ class OverlayCreator ( object ):
 		self._runlock.release()
 	# --- end of _close_resolver (...) ---
 
-	def _close_workers ( self ):
+	def _waitfor_workers ( self, do_close ):
+		"""Waits until the workers are done.
+
+		arguments:
+		* do_close -- close (exit) workers if True, else wait until done.
+		"""
 		if self._workers is None: return
 
 		self._runlock.acquire()
@@ -151,7 +175,10 @@ class OverlayCreator ( object ):
 
 		if self.NUMTHREADS > 0: start = time.time()
 
-		for w in self._workers: w.enabled = False
+		if do_close:
+			for w in self._workers: w.enabled = False
+		else:
+			for w in self._workers: w.stop_when_empty()
 
 		while True in ( w.active() for w in self._workers ):
 			self._pkg_queue.put ( None )
@@ -159,13 +186,34 @@ class OverlayCreator ( object ):
 		del self._workers
 		self._workers = None
 
-		if self.NUMTHREADS > 0:
+		self._runlock.release()
+		if self.NUMTHREADS > 0: return start
+	# --- end of _waitfor_workers (...) ---
+
+	def _join_workers ( self ):
+		"""Waits until all workers are done."""
+		start = self._waitfor_workers ( False )
+		if start is not None:
 			self._timestamp ( "worker threads are done", start )
+	# --- end of _join_workers (...) ---
 
-		self._runlock.release()
+	def _close_workers ( self ):
+		"""Tells the workers to exit.
+		This is done by disabling them and inserting empty requests (None as
+		PackageInfo) to unblock them.
+		"""
+		start = self._waitfor_workers ( True )
+		if start is not None:
+			self._timestamp ( "worker threads are closed", start )
 	# --- end of _close_workers (...) ---
 
 	def _pkg_done ( self, package_info ):
+		"""This is an event method used by worker threads when they have
+		processed a package info.
+
+		arguments:
+		* package_info --
+		"""
 		# ... TODO
 		#  * increase the number of successful/failed packages,
 		#  * request an incremental write to save memory etc.
@@ -176,6 +224,12 @@ class OverlayCreator ( object ):
 	# --- end of _add_to_overlay (...) ---
 
 	def _get_worker ( self, start_now=False, use_threads=True ):
+		"""Creates and returns a worker.
+
+		arguments:
+		* start_now   -- if set and True: start the worker immediately
+		* use_threads -- if set and False: disable threads
+		"""
 		w = OverlayWorker (
 			self._pkg_queue, self.depresolver, self.logger, self._pkg_done,
 			use_threads=use_threads
@@ -185,6 +239,7 @@ class OverlayCreator ( object ):
 	# --- end of _get_worker (...) ---
 
 	def _make_workers ( self ):
+		"""Creates and starts workers."""
 		self._close_workers()
 
 		if self.NUMTHREADS > 0:

diff --git a/roverlay/overlay/worker.py b/roverlay/overlay/worker.py
index 4f03fe2..471cb94 100644
--- a/roverlay/overlay/worker.py
+++ b/roverlay/overlay/worker.py
@@ -1,34 +1,44 @@
+# R Overlay -- R package -> overlay interface, PackageInfo worker
+# Copyright 2006-2012 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
 
-import time
+#import time
 import threading
 
-try:
-	import queue
-except ImportError:
-	# python2
-	import Queue as queue
-
-
 from roverlay.depres.channels    import EbuildJobChannel
 from roverlay.ebuild.creation    import EbuildCreation
 
 class OverlayWorker ( object ):
-
-	def __init__ ( self, pkg_queue, depresolver, logger, pkg_done, use_threads ):
+	"""Overlay package queue worker."""
+
+	def __init__ ( self,
+		pkg_queue, depresolver, logger, pkg_done, use_threads
+	):
+		"""Initializes a worker.
+
+		arguments:
+		* pkg_queue   -- queue with PackageInfos
+		* depresolver -- dependency resolver to use
+		* logger      -- logger to use
+		* pkg_done    -- method to call when a PackageInfo has been
+		                 processed
+		* use_threads -- whether to run this worker as a thread or not
+		"""
 		self.logger      = logger
 		self.pkg_queue   = pkg_queue
 		self.pkg_done    = pkg_done
 		self.depresolver = depresolver
 
-
-		self.enabled     = True
-		self.running     = False
 		self._use_thread = use_threads
 		self._thread     = None
 
+		self.enabled     = True
+		self.running     = False
+		self.halting     = False
 	# --- end of __init__ (...) ---
 
 	def start ( self ):
+		"""Starts the worker."""
 		if self._thread is not None:
 			self._thread.join()
 			del self._thread
@@ -41,17 +51,34 @@ class OverlayWorker ( object ):
 			self._run_nothread()
 	# --- end of start (...) ---
 
+	def stop_when_empty ( self ):
+		"""Tells the worker thread to exit when the queue is empty."""
+		self.enabled = False
+		self.halting = True
+	# --- end of stop_when_empty (...) ---
+
 	def active ( self ):
+		"""Returns True if this worker is active (running or enabled)."""
 		return self.running or self.enabled
 	# --- end of active (...) ---
 
 	def _get_resolver_channel ( self, **channel_kw ):
+		"""Returns a resolver channel.
+
+		arguments:
+		* **channel_kw -- keywords for EbuildJobChannel.__init__
+		"""
 		return self.depresolver.register_channel (
 			EbuildJobChannel ( **channel_kw )
 		)
 	# --- end of _get_resolver_channel (...) ---
 
 	def _process ( self, package_info ):
+		"""Processes a PackageInfo taken from the queue.
+
+		arguments:
+		* package_info --
+		"""
 		job = EbuildCreation (
 			package_info,
 			depres_channel_spawner=self._get_resolver_channel
@@ -61,21 +88,37 @@ class OverlayWorker ( object ):
 	# --- end of _process (...) ---
 
 	def _run ( self ):
+		"""Runs the worker (thread mode)."""
 		self.running = True
-		while self.enabled:
+		self.halting = False
+		while self.enabled or (
+			self.halting and not self.pkg_queue.empty()
+		):
+			if not self.running:
+				# exit now
+				break
 			p = self.pkg_queue.get()
 
 			# drop empty requests that are used to unblock get()
 			if p is not None:
 				self._process ( p )
+			elif self.halting:
+				# receiving an empty request while halting means 'stop now',
+				self.enabled = False
+				self.halting = False
 
 			self.pkg_queue.task_done()
 		self.running = False
 	# --- end of run (...) ---
 
 	def _run_nothread ( self ):
+		"""Runs the worker (no-thread mode)."""
 		self.running = True
 		while self.enabled and not self.pkg_queue.empty():
+			if not self.running:
+				# exit now
+				break
+
 			p = self.pkg_queue.get()
 
 			# drop empty requests that are used to unblock get()

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     35da2e8aaccd1b1163016c861aa06fd53f4875a8
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jun 21 16:45:02 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jun 21 16:45:02 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=35da2e8a

fixes in overlay/

	modified:   roverlay/overlay/__init__.py
	modified:   roverlay/overlay/package.py

---
 roverlay/overlay/__init__.py |   10 +++++++++-
 roverlay/overlay/package.py  |    2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py
index 7e4bbea..dadf998 100644
--- a/roverlay/overlay/__init__.py
+++ b/roverlay/overlay/__init__.py
@@ -51,7 +51,15 @@ class Overlay ( object ):
 		else:
 			self.default_category = default_category
 
-		self.eclass_files = eclass_files
+
+		if eclass_files is None:
+			eclass_files = config.get ( 'OVERLAY.eclass_files', None )
+
+		if isinstance ( eclass_files, str ):
+			self.eclass_files = frozenset ( eclass_files )
+		else:
+			self.eclass_files = eclass_files
+
 
 		#
 		self._profiles_dir = os.path.join ( self.physical_location, 'profiles' )

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 3138779..b571ad9 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -218,7 +218,7 @@ class PackageDir ( object ):
 					self.name, shortver
 				)
 				if SUPPRESS_EXCEPTIONS:
-					logger.warning ( msg )
+					self.logger.warning ( msg )
 				else:
 					raise Exception ( msg )