* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/, roverlay/overlay/
@ 2013-01-28 23:54 André Erdmann
0 siblings, 0 replies; 4+ messages in thread
From: André Erdmann @ 2013-01-28 23:54 UTC (permalink / raw
To: gentoo-commits
commit: 2c3c9f6f789361c4e1438d8b8a05c9f8287caed9
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jan 28 21:15:14 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jan 28 23:37:04 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2c3c9f6f
roverlay/overlay: pass parent to PackageDir
* roverlay/overlay/category.py:
Pass "self" as parent parameter to the PackageDir constructor
* roverlay/overlay/pkgdir/packagedir_base.py:
Accept the parent parameter.
Also added some comments.
---
roverlay/overlay/category.py | 3 ++-
roverlay/overlay/pkgdir/packagedir_base.py | 17 ++++++++++++++---
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 01c1776..3a58367 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -66,7 +66,8 @@ class Category ( object ):
logger = self.logger,
directory = self.physical_location + os.sep + pkg_name,
get_header = self.get_header,
- runtime_incremental = self.runtime_incremental
+ runtime_incremental = self.runtime_incremental,
+ parent = self
)
self._subdirs [pkg_name] = newpkg
finally:
diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 34b7ef5..35350ee 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -6,11 +6,11 @@
"""overlay <-> filesystem interface (package directory)
-This module provides the PackageDir class that offers direct
+This module provides the PackageDir base class that offers direct
PackageInfo (in memory) <-> package directory (as directory in filesystem)
access, e.g. ebuild/metadata.xml/Manifest writing as well as scanning
the existing package directory.
-Each PackageDir represents one package name (e.g. "seewave").
+Each PackageDir instance represents one package name (e.g. "seewave").
"""
__all__ = [ 'PackageDirBase', ]
@@ -25,6 +25,8 @@ from roverlay.packageinfo import PackageInfo
from roverlay.overlay.pkgdir.metadata import MetadataJob
class PackageDirBase ( object ):
+ """The PackageDir base class that implements most functionality except
+ for Manifest file creation."""
EBUILD_SUFFIX = '.ebuild'
SUPPRESS_EXCEPTIONS = True
@@ -37,7 +39,7 @@ class PackageDirBase ( object ):
MANIFEST_THREADSAFE = None
def __init__ ( self,
- name, logger, directory, get_header, runtime_incremental
+ name, logger, directory, get_header, runtime_incremental, parent
):
"""Initializes a PackageDir which contains ebuilds, metadata and
a Manifest file.
@@ -55,6 +57,8 @@ class PackageDirBase ( object ):
Writing all ebuilds at once is generally faster
(+threading), but all PackageInfos must be
kept in memory for that.
+ * parent (pointer to) the object that is creating this
+ instance
"""
self.logger = logger.getChild ( name )
self.name = name
@@ -98,6 +102,13 @@ class PackageDirBase ( object ):
# --- end of remove_ebuild_file (...) ---
def _scan_add_package ( self, efile, pvr ):
+ """Called for each ebuild that is found during scan().
+ Creates a PackageInfo for the ebuild and adds it to self._packages.
+
+ arguments:
+ * efile -- full path to the ebuild file
+ * pvr -- version ($PVR) of the ebuild
+ """
p = PackageInfo (
physical_only=True, pvr=pvr, ebuild_file=efile
)
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/, roverlay/overlay/
@ 2013-07-25 15:20 André Erdmann
0 siblings, 0 replies; 4+ messages in thread
From: André Erdmann @ 2013-07-25 15:20 UTC (permalink / raw
To: gentoo-commits
commit: d62c753db719a96042db2320661a75e213a018a3
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 25 14:40:16 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 25 14:40:16 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=d62c753d
stats collection: overlay stats
* ebuild count before/after overlay creation
* # of written ebuilds
---
roverlay/overlay/category.py | 99 +++++++++++++++++++++---------
roverlay/overlay/pkgdir/packagedir_base.py | 37 ++++++++---
2 files changed, 100 insertions(+), 36 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 330058c..3126642 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -23,12 +23,63 @@ try:
except ImportError:
import Queue as queue
+
+import roverlay.stats.collector
+
from roverlay.overlay import pkgdir
+class WriteQueueJob ( object ):
+
+ def __init__ ( self, write_queue, write_kw, catref, additions_dir ):
+ super ( WriteQueueJob, self ).__init__()
+ self.write_queue = write_queue
+ self.write_kw = write_kw
+ self.catref = catref
+ self.additions_dir = additions_dir
+ self.stats = catref.STATS.get_new()
+ # --- end of __init__ (...) ---
+
+ def run ( self ):
+ """Calls <package>.write for every <package> received from the queue.
+
+ arguments:
+ * q -- queue
+ * write_kw -- keywords for write(...)
+ """
+ q = self.write_queue
+ write_kw = self.write_kw
+ stats = self.stats
+ catref = self.catref
+ additions_dir = self.additions_dir
+
+ while not q.empty() and not hasattr ( catref, 'RERAISE' ):
+ try:
+ pkg = q.get_nowait()
+ # remove manifest writing from threaded writing since it's
+ # single-threaded
+ pkg.write (
+ additions_dir = additions_dir.get_obj_subdir ( pkg ),
+ stats = stats,
+ **write_kw
+ )
+ stats.ebuild_count.inc (
+ len ( list ( pkg.iter_packages_with_efile() ) )
+ )
+ except queue.Empty:
+ break
+ except ( Exception, KeyboardInterrupt ) as err:
+ catref.logger.exception ( err )
+ catref.RERAISE = sys.exc_info()
+ # --- end of run (...) ---
+
+# --- end of WriteQueueJob ---
+
class Category ( object ):
WRITE_JOBCOUNT = 3
+ STATS = roverlay.stats.collector.static.overlay
+
def __init__ ( self,
name, logger, directory, get_header, runtime_incremental
):
@@ -221,11 +272,12 @@ class Category ( object ):
def scan ( self, **kw ):
"""Scans this category for existing ebuilds."""
+ stats = self.STATS
for subdir in os.listdir ( self.physical_location ):
if self.has_dir ( subdir ):
pkgdir = self._get_package_dir ( subdir )
try:
- pkgdir.scan ( **kw )
+ pkgdir.scan ( stats=stats, **kw )
finally:
if pkgdir.empty():
del self._subdirs [subdir]
@@ -251,29 +303,6 @@ class Category ( object ):
returns: None (implicit)
"""
- def run_write_queue ( q, write_kw ):
- """Calls <package>.write for every <package> received from the queue.
-
- arguments:
- * q -- queue
- * write_kw -- keywords for write(...)
- """
- while not q.empty() and not hasattr ( self, 'RERAISE' ):
- try:
- pkg = q.get_nowait()
- # remove manifest writing from threaded writing since it's
- # single-threaded
- pkg.write (
- additions_dir = additions_dir.get_obj_subdir ( pkg ),
- **write_kw
- )
- except queue.Empty:
- break
- except ( Exception, KeyboardInterrupt ) as err:
- self.logger.exception ( err )
- self.RERAISE = sys.exc_info()
- # --- end of run_write_queue (...) ---
-
if len ( self._subdirs ) == 0: return
# determine write keyword args
@@ -310,11 +339,13 @@ class Category ( object ):
write_manifest and manifest_threadsafe
)
+ jobs = frozenset (
+ WriteQueueJob ( write_queue, write_kwargs, self, additions_dir )
+ for n in range ( max_jobs )
+ )
+
workers = frozenset (
- threading.Thread (
- target=run_write_queue,
- args=( write_queue, write_kwargs )
- ) for n in range ( max_jobs )
+ threading.Thread ( target=job.run ) for job in jobs
)
for w in workers: w.start()
@@ -344,15 +375,25 @@ class Category ( object ):
for package in self._subdirs.values():
package.write_manifest ( ignore_empty=True )
+
+ # merge stats from threads with self.(__class__.)STATS
+ for job in jobs:
+ self.STATS.merge_with ( job.stats )
else:
+ stats = self.STATS
for package in self._subdirs.values():
package.write (
additions_dir = additions_dir.get_obj_subdir ( package ),
+ stats = stats,
**write_kwargs
)
+ stats.ebuild_count.inc (
+ len ( list ( package.iter_packages_with_efile() ) )
+ )
+
+ self.remove_empty()
# -- end if;
- self.remove_empty()
# --- end of write (...) ---
def write_manifest ( self, **manifest_kw ):
diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 8c0bd0b..7f1a0ba 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -48,7 +48,7 @@ class PackageDirBase ( object ):
#DISTMAP =
#FETCHENV =
- EBUILD_SUFFIX = '.ebuild'
+ EBUILD_SUFFIX = '.ebuild'
# MANIFEST_THREADSAFE (tri-state)
# * None -- unknown (e.g. write_manifest() not implemented)
@@ -124,10 +124,19 @@ class PackageDirBase ( object ):
self._need_metadata = False
# --- end of __init__ (...) ---
- def iter_package_info ( self ):
- return self._packages.values()
+ def iter_package_info ( self, pkg_filter=None ):
+ if pkg_filter is None:
+ return self._packages.values()
+ else:
+ return ( p for p in self._packages.values() if pkg_filter ( p ) )
# --- end of iter_package_info (...) --
+ def iter_packages_with_efile ( self ):
+ return (
+ p for p in self._packages.values() if p.has ( 'ebuild_file' )
+ )
+ # --- end of iter_packages_with_efile (...) ---
+
def _remove_ebuild_file ( self, pkg_info ):
"""Removes the ebuild file of a pkg_info object.
Returns True on success, else False.
@@ -420,7 +429,7 @@ class PackageDirBase ( object ):
self.fs_cleanup()
# --- end of fs_destroy (...) ---
- def scan ( self, **kw ):
+ def scan ( self, stats, **kw ):
"""Scans the filesystem location of this package for existing
ebuilds and adds them.
"""
@@ -457,6 +466,8 @@ class PackageDirBase ( object ):
)
# --- end of scan_ebuilds (...) ---
+ ebuild_found = stats.ebuilds_scanned.inc
+
# ignore directories without a Manifest file
if os.path.isfile ( self.physical_location + os.sep + 'Manifest' ):
for pvr, efile in scan_ebuilds():
@@ -470,6 +481,8 @@ class PackageDirBase ( object ):
)
)
raise
+ else:
+ ebuild_found()
# --- end of scan (...) ---
def show ( self, stream=sys.stderr ):
@@ -506,7 +519,7 @@ class PackageDirBase ( object ):
additions_dir,
overwrite_ebuilds=False,
write_ebuilds=True, write_manifest=True, write_metadata=True,
- cleanup=True, keep_n_ebuilds=None, cautious=True
+ cleanup=True, keep_n_ebuilds=None, cautious=True, stats=None,
):
"""Writes this directory to its (existent!) filesystem location.
@@ -526,6 +539,7 @@ class PackageDirBase ( object ):
* cautious -- be cautious when keeping the nth latest ebuilds,
this has some overhead
Defaults to True
+ * stats --
returns: success (True/False)
@@ -554,7 +568,8 @@ class PackageDirBase ( object ):
# possible in this if-branch
success = self.write_ebuilds (
additions_dir = additions_dir,
- overwrite = bool ( overwrite_ebuilds )
+ overwrite = bool ( overwrite_ebuilds ),
+ stats = stats,
)
# overwrite = overwrite_ebuilds \
# if overwrite_ebuilds is not None \
@@ -689,7 +704,9 @@ class PackageDirBase ( object ):
import_ebuild_efile ( pvr, efile, fname )
# --- end of import_ebuilds (...) ---
- def write_ebuilds ( self, overwrite, additions_dir, shared_fh=None ):
+ def write_ebuilds ( self,
+ overwrite, additions_dir, shared_fh=None, stats=None
+ ):
"""Writes all ebuilds.
arguments:
@@ -697,6 +714,7 @@ class PackageDirBase ( object ):
* additions_dir --
* shared_fh -- if set and not None: don't use own file handles
(i.e. write files), write everything into shared_fh
+ * stats --
"""
ebuild_header = self.get_header()
@@ -798,6 +816,10 @@ class PackageDirBase ( object ):
all_ebuilds_written = True
+ ebuild_written = (
+ stats.ebuilds_written.inc if stats is not None else ( lambda: None )
+ )
+
# don't call dodir if shared_fh is set
hasdir = bool ( shared_fh is not None )
@@ -831,6 +853,7 @@ class PackageDirBase ( object ):
ebuild_file=efile,
remove_auto='ebuild_written'
)
+ ebuild_written()
self.logger.info ( "Wrote ebuild {}.".format ( efile ) )
else:
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/, roverlay/overlay/
@ 2013-08-02 13:39 André Erdmann
0 siblings, 0 replies; 4+ messages in thread
From: André Erdmann @ 2013-08-02 13:39 UTC (permalink / raw
To: gentoo-commits
commit: 14fa2c10125834937c7c3db5f6495749b1d06d94
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 2 13:30:12 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 2 13:30:12 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=14fa2c10
manifest implementations: simplify code
* moved [overlay]/pkgdir/__init__ to pkgdir/base
* pkgdir/base no longer uses importlib, package dir modules/classes are used
directly instead (_PACKAGE_DIR_IMPLEMENTATIONS lists PackageDir classes now)
* overlay/category sets its pkgdir class in __init__ now
---
roverlay/overlay/__init__.py | 4 --
roverlay/overlay/category.py | 15 ++--
roverlay/overlay/creator.py | 2 +-
roverlay/overlay/pkgdir/__init__.py | 86 -----------------------
roverlay/overlay/pkgdir/{__init__.py => base.py} | 43 +++---------
roverlay/overlay/pkgdir/packagedir_newmanifest.py | 4 +-
6 files changed, 21 insertions(+), 133 deletions(-)
diff --git a/roverlay/overlay/__init__.py b/roverlay/overlay/__init__.py
index 84cdf31..708f8ae 100644
--- a/roverlay/overlay/__init__.py
+++ b/roverlay/overlay/__init__.py
@@ -5,7 +5,3 @@
# either version 2 of the License, or (at your option) any later version.
"""overlay package"""
-
-__all__ = [ 'Overlay', ]
-
-from roverlay.overlay.root import Overlay
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 751a26d..2fc3a4f 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -26,7 +26,10 @@ except ImportError:
import roverlay.stats.collector
-from roverlay.overlay import pkgdir
+import roverlay.overlay.pkgdir.base
+##import roverlay.overlay.pkgdir.packagedir_ebuildmanifest
+##import roverlay.overlay.pkgdir.packagedir_newmanifest
+
class WriteQueueJob ( object ):
@@ -100,6 +103,7 @@ class Category ( object ):
self.physical_location = directory
self.get_header = get_header
self.runtime_incremental = runtime_incremental
+ self.packagedir_cls = roverlay.overlay.pkgdir.base.get_class()
# --- end of __init__ (...) ---
def _get_package_dir ( self, pkg_name ):
@@ -113,7 +117,7 @@ class Category ( object ):
self._lock.acquire()
try:
if not pkg_name in self._subdirs:
- newpkg = pkgdir.get_class() (
+ newpkg = self.packagedir_cls (
name = pkg_name,
logger = self.logger,
directory = self.physical_location + os.sep + pkg_name,
@@ -249,12 +253,7 @@ class Category ( object ):
"""
if unsafe:
- try:
- return bool (
- next ( iter ( self._subdirs.values() ) ).MANIFEST_THREADSAFE
- )
- except StopIteration:
- return True
+ return self.packagedir_cls.MANIFEST_THREADSAFE
else:
for pkgdir in self._subdirs.values():
#if not pkgdir.__class__.MANIFEST_THREADSAFE:
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 32dacbc..96c5531 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -26,7 +26,7 @@ except ImportError:
from roverlay import config, errorqueue
-from roverlay.overlay import Overlay
+from roverlay.overlay.root import Overlay
from roverlay.overlay.worker import OverlayWorker
from roverlay.packageinfo import PackageInfo
from roverlay.packagerules.rules import PackageRules
diff --git a/roverlay/overlay/pkgdir/__init__.py b/roverlay/overlay/pkgdir/__init__.py
index 90dfb6e..e69de29 100644
--- a/roverlay/overlay/pkgdir/__init__.py
+++ b/roverlay/overlay/pkgdir/__init__.py
@@ -1,86 +0,0 @@
-# R overlay -- overlay package, package directory
-# -*- coding: utf-8 -*-
-# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
-# Distributed under the terms of the GNU General Public License;
-# either version 2 of the License, or (at your option) any later version.
-
-__all__ = [ 'create', 'get_class', ]
-
-# the actual PackageDir implementation will be loaded after initialization
-# of the config module (using importlib.import_module)
-import importlib
-
-import logging
-
-import roverlay.config
-
-
-# _package_dir_module is an imported module or None
-_package_dir_module = None
-_package_dir_class = None
-
-_PACKAGE_DIR_IMPLEMENTATIONS = {
- 'none' : 'packagedir_base',
- 'default' : 'packagedir_newmanifest',
- 'e' : 'packagedir_ebuildmanifest',
- 'ebuild' : 'packagedir_ebuildmanifest',
- 'next' : 'packagedir_newmanifest',
-}
-
-def _configure():
- """Determines which Manifest implementation to use and sets the
- _package_dir_module, _package_dir_class variables accordingly.
- """
-
- mf_impl = roverlay.config.get_or_fail ( 'OVERLAY.manifest_implementation' )
-
- # also accept the internal (module) name of the manifest implementation
- pkgdir_impl = (
- _PACKAGE_DIR_IMPLEMENTATIONS.get ( mf_impl, None )
- or (
- mf_impl
- if mf_impl in _PACKAGE_DIR_IMPLEMENTATIONS.values()
- else None
- )
- )
-
- if pkgdir_impl is not None:
- global _package_dir_module
- _package_dir_module = importlib.import_module (
- 'roverlay.overlay.pkgdir.' + pkgdir_impl
- )
-
- global _package_dir_class
- if hasattr ( _package_dir_module, 'PackageDir' ):
- _package_dir_class = _package_dir_module.PackageDir
- else:
- _package_dir_class = _package_dir_module.PackageDirBase
-
- if hasattr ( _package_dir_class, 'init_cls' ):
- _package_dir_class.init_cls()
- else:
- _package_dir_class.init_base_cls()
-
- logging.getLogger ('pkgdir').debug (
- 'Using {!r} as manifest implementation.'.format ( mf_impl )
- )
- else:
- # NameError, NotImplementedError, ...?
- raise Exception (
- "PackageDir/Manifest implementation {} is unknown".format ( mf_impl )
- )
-# --- end of configure (...) ---
-
-def get_class():
- """Returns the configured PackageDir class."""
- if _package_dir_class is None:
- _configure()
- return _package_dir_class
-# --- end of get_class (...) ---
-
-def create ( *args, **kwargs ):
- """Returns a new PackageDir object by calling the constructor
- of the configured PackageDir class (as returned by get_class()).
- """
- return get_class() ( *args, **kwargs )
-# --- end of create (...) ---
diff --git a/roverlay/overlay/pkgdir/__init__.py b/roverlay/overlay/pkgdir/base.py
similarity index 59%
copy from roverlay/overlay/pkgdir/__init__.py
copy to roverlay/overlay/pkgdir/base.py
index 90dfb6e..4d48a3f 100644
--- a/roverlay/overlay/pkgdir/__init__.py
+++ b/roverlay/overlay/pkgdir/base.py
@@ -6,55 +6,34 @@
__all__ = [ 'create', 'get_class', ]
-# the actual PackageDir implementation will be loaded after initialization
-# of the config module (using importlib.import_module)
-import importlib
-
import logging
import roverlay.config
+import roverlay.overlay.pkgdir.packagedir_base
+import roverlay.overlay.pkgdir.packagedir_ebuildmanifest
+import roverlay.overlay.pkgdir.packagedir_newmanifest
+
-# _package_dir_module is an imported module or None
-_package_dir_module = None
_package_dir_class = None
_PACKAGE_DIR_IMPLEMENTATIONS = {
- 'none' : 'packagedir_base',
- 'default' : 'packagedir_newmanifest',
- 'e' : 'packagedir_ebuildmanifest',
- 'ebuild' : 'packagedir_ebuildmanifest',
- 'next' : 'packagedir_newmanifest',
+ 'none' : roverlay.overlay.pkgdir.packagedir_base.PackageDirBase,
+ 'default' : roverlay.overlay.pkgdir.packagedir_newmanifest.PackageDir,
+ 'e' : roverlay.overlay.pkgdir.packagedir_ebuildmanifest.PackageDir,
+ 'ebuild' : roverlay.overlay.pkgdir.packagedir_ebuildmanifest.PackageDir,
+ 'next' : roverlay.overlay.pkgdir.packagedir_newmanifest.PackageDir,
}
def _configure():
"""Determines which Manifest implementation to use and sets the
_package_dir_module, _package_dir_class variables accordingly.
"""
-
mf_impl = roverlay.config.get_or_fail ( 'OVERLAY.manifest_implementation' )
- # also accept the internal (module) name of the manifest implementation
- pkgdir_impl = (
- _PACKAGE_DIR_IMPLEMENTATIONS.get ( mf_impl, None )
- or (
- mf_impl
- if mf_impl in _PACKAGE_DIR_IMPLEMENTATIONS.values()
- else None
- )
- )
-
- if pkgdir_impl is not None:
- global _package_dir_module
- _package_dir_module = importlib.import_module (
- 'roverlay.overlay.pkgdir.' + pkgdir_impl
- )
-
+ if mf_impl in _PACKAGE_DIR_IMPLEMENTATIONS:
global _package_dir_class
- if hasattr ( _package_dir_module, 'PackageDir' ):
- _package_dir_class = _package_dir_module.PackageDir
- else:
- _package_dir_class = _package_dir_module.PackageDirBase
+ _package_dir_class = _PACKAGE_DIR_IMPLEMENTATIONS [mf_impl]
if hasattr ( _package_dir_class, 'init_cls' ):
_package_dir_class.init_cls()
diff --git a/roverlay/overlay/pkgdir/packagedir_newmanifest.py b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
index 7bbfb86..fb9bd2d 100644
--- a/roverlay/overlay/pkgdir/packagedir_newmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
@@ -13,10 +13,10 @@ import threading
import roverlay.config
import roverlay.overlay.pkgdir.manifest.file
-import roverlay.overlay.pkgdir.packagedir_base
-
from roverlay.overlay.pkgdir.manifest.file import ManifestFile
+import roverlay.overlay.pkgdir.packagedir_base
+
class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
"""
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/, roverlay/overlay/
@ 2013-08-06 10:58 André Erdmann
0 siblings, 0 replies; 4+ messages in thread
From: André Erdmann @ 2013-08-06 10:58 UTC (permalink / raw
To: gentoo-commits
commit: 2b7c69657d7f3931a4aa316ba855988ea515b021
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Aug 6 10:23:38 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Aug 6 10:23:38 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2b7c6965
also import metadata.xml when importing ebuilds
---
roverlay/overlay/additionsdir.py | 14 ++
roverlay/overlay/pkgdir/packagedir_base.py | 158 ++++++++++++++++-----
.../overlay/pkgdir/packagedir_ebuildmanifest.py | 49 +------
roverlay/overlay/pkgdir/packagedir_newmanifest.py | 2 +-
4 files changed, 138 insertions(+), 85 deletions(-)
diff --git a/roverlay/overlay/additionsdir.py b/roverlay/overlay/additionsdir.py
index 3031a69..5e24622 100644
--- a/roverlay/overlay/additionsdir.py
+++ b/roverlay/overlay/additionsdir.py
@@ -155,6 +155,14 @@ class EbuildView ( _EbuildAdditionsView ):
return self._ebuilds
# --- end of get_ebuilds (...) ---
+ def has_metadata_xml ( self ):
+ return bool ( getattr ( self, '_metadata_xml', None ) )
+ # --- end of has_metadata_xml (...) ---
+
+ def get_metadata_xml ( self ):
+ return self._metadata_xml
+ # --- end of get_metadata_xml (...) ---
+
def __iter__ ( self ):
return iter ( self.get_ebuilds() )
# --- end of __iter__ (...) ---
@@ -173,6 +181,12 @@ class EbuildView ( _EbuildAdditionsView ):
)
self._ebuilds = ebuilds
+
+ metadata_xml = self._additions_dir.root + os.sep + 'metadata.xml'
+ if os.path.isfile ( metadata_xml ):
+ self._metadata_xml = metadata_xml
+ else:
+ self._metadata_xml = None
# --- end of prepare (...) --
# --- end of EbuildView ---
diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 36097c6..5c0d5fd 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -46,7 +46,8 @@ class PackageDirBase ( object ):
#DISTROOT =
#DISTMAP =
- #FETCHENV =
+ #FETCH_ENV =
+ #MANIFEST_ENV =
EBUILD_SUFFIX = '.ebuild'
@@ -62,23 +63,27 @@ class PackageDirBase ( object ):
# other subsystems might calculate them in advance if advertised here
HASH_TYPES = None
- # DOEBUILD_FETCH_WITH_MANIFEST
- # bool that indicates whether the "ebuild <...> fetch" should create
- # a Manifest file (when importing ebuilds)
+ # DOEBUILD_IMPORTMANIFEST
+ # bool that controls whether a Manifest file should be created when
+ # importing ebuilds.
#
- DOEBUILD_FETCH_WITH_MANIFEST = False
+ DOEBUILD_IMPORTMANIFEST = False
@classmethod
def init_base_cls ( cls ):
# env for calling "ebuild <ebuild file> fetch"
+ overlay_dir = roverlay.config.get_or_fail ( 'OVERLAY.dir' )
+
fetch_env = roverlay.tools.ebuildenv.FetchEnv()
- fetch_env.add_overlay_dir (
- roverlay.config.get_or_fail ( 'OVERLAY.dir' )
- )
+ fetch_env.add_overlay_dir ( overlay_dir )
+
+ mf_env = roverlay.tools.ebuildenv.ManifestEnv()
+ mf_env.add_overlay_dir ( overlay_dir )
cls.DISTROOT = roverlay.overlay.pkgdir.distroot.static.get_configured()
- cls.DISTMAP = roverlay.recipe.distmap.access()
- cls.FETCHENV = fetch_env
+ cls.DISTMAP = roverlay.recipe.distmap.access()
+ cls.FETCH_ENV = fetch_env
+ cls.MANIFEST_ENV = mf_env
# --- end of init_cls (...) ---
def __init__ ( self,
@@ -617,10 +622,28 @@ class PackageDirBase ( object ):
return self.DISTROOT.get_distdir ( self.name )
# --- end of get_distdir (...) ---
- def fetch_src_for_ebuild ( self, efile ):
- fetch_env = self.FETCHENV.get_env ( self.get_distdir().get_root() )
+ def get_fetch_env ( self, distdir=None ):
+ return self.FETCH_ENV.get_env (
+ (
+ self.DISTROOT.get_distdir ( self.name )
+ if distdir is None else distdir
+ ).get_root()
+ )
+ # --- end of get_fetch_env (...) ---
+
+ def get_manifest_env ( self, distdir=None ):
+ return self.MANIFEST_ENV.get_env (
+ (
+ self.DISTROOT.get_distdir ( self.name )
+ if distdir is None else distdir
+ ).get_root()
+ )
+ # --- end of get_manifest_env (...) ---
+
+ def fetch_src_for_ebuild ( self, efile, create_manifest=False ):
+ fetch_env = self.get_fetch_env()
- if self.DOEBUILD_FETCH_WITH_MANIFEST:
+ if create_manifest:
return roverlay.tools.ebuild.doebuild_fetch_and_manifest (
ebuild_file = efile,
logger = self.logger,
@@ -634,6 +657,37 @@ class PackageDirBase ( object ):
)
# --- end of fetch_src_for_ebuild (...) ---
+ def do_ebuildmanifest ( self, ebuild_file, distdir=None ):
+ """Calls doebuild_manifest().
+ Returns True on success, else False. Also handles result logging.
+
+ arguments:
+ * ebuild_file -- ebuild file that should be used for the doebuild call
+ * distdir -- distdir object (optional)
+ """
+ try:
+ call = roverlay.tools.ebuild.doebuild_manifest (
+ ebuild_file, self.logger, self.get_manifest_env ( distdir ),
+ return_success=False
+ )
+ except Exception as err:
+ self.logger.exception ( err )
+ raise
+ # -- end try
+
+ if call.returncode == os.EX_OK:
+ self.logger.debug ( "Manifest written." )
+ return True
+ else:
+ self.logger.error (
+ 'Couldn\'t create Manifest for {ebuild}! '
+ 'Return code was {ret}.'.format (
+ ebuild=ebuild_file, ret=call.returncode
+ )
+ )
+ return False
+ # --- end of do_ebuildmanifest (...) ---
+
def import_ebuilds ( self, eview, overwrite, nosync=False, stats=None ):
"""Imports ebuilds from an additions dir into this package dir.
@@ -643,17 +697,32 @@ class PackageDirBase ( object ):
* nosync -- if True: don't fetch src files (defaults to False)
* stats --
"""
+ if not self.physical_location:
+ raise Exception (
+ "import_ebuilds() needs a non-virtual package dir!"
+ )
+ elif not eview.has_ebuilds():
+ return None
+
+ # setup
stats_ebuild_imported = (
stats.ebuilds_imported.inc if stats is not None
else ( lambda: None )
)
+ imported_ebuild_files = list()
+ efile_imported = imported_ebuild_files.append
+
+ fetch_env = self.get_fetch_env()
+
+ roverlay.util.dodir ( self.physical_location, mkdir_p=True )
+ # -- end setup
def import_ebuild_efile ( pvr, efile_src, fname ):
"""Imports an ebuild file into this package dir and registers it
in self._packages.
- Returns the PackageInfo instance of the imported ebuild.
+ Returns the PackageInfo instance of the imported ebuild,
arguments:
* pvr --
@@ -687,12 +756,6 @@ class PackageDirBase ( object ):
if not nosync and not self.fetch_src_for_ebuild ( efile_dest ):
raise Exception ( "doebuild_fetch() failed." )
- # imported ebuilds cannot be used for generating metadata.xml
- ##self._need_metadata = True
-
- stats_ebuild_imported()
-
- return p
except:
# this package dir is "broken" now,
# so a new manifest would be good...
@@ -709,29 +772,50 @@ class PackageDirBase ( object ):
os.unlink ( efile_dest )
raise
+ else:
+ # imported ebuilds cannot be used for generating metadata.xml
+ ##self._need_metadata = True
+
+ stats_ebuild_imported()
+ efile_imported ( efile_dest )
+ return p
# --- end of import_ebuild_efile (...) ---
- if not self.physical_location:
- raise Exception (
- "import_ebuilds() needs a non-virtual package dir!"
- )
- elif eview.has_ebuilds():
- roverlay.util.dodir ( self.physical_location, mkdir_p=True )
+ if not self._packages:
+ for pvr, efile, fname in eview.get_ebuilds():
+ import_ebuild_efile ( pvr, efile, fname )
- if not self._packages:
- for pvr, efile, fname in eview.get_ebuilds():
- import_ebuild_efile ( pvr, efile, fname )
+ elif overwrite:
+ for pvr, efile, fname in eview.get_ebuilds():
+ if pvr in self._packages:
+ self.purge_package ( pvr )
+
+ import_ebuild_efile ( pvr, efile, fname )
+ # -- end for
- elif overwrite:
- for pvr, efile, fname in eview.get_ebuilds():
- if pvr in self._packages:
- self.purge_package ( pvr )
+ else:
+ for pvr, efile, fname in eview.get_ebuilds():
+ if pvr not in self._packages:
import_ebuild_efile ( pvr, efile, fname )
- else:
- for pvr, efile, fname in eview.get_ebuilds():
- if pvr not in self._packages:
- import_ebuild_efile ( pvr, efile, fname )
+
+ # import metadata.xml from eview
+ # in conjunction with overwrite=False, metadata.xml won't get
+ # updated even if all ebuilds are imports (since only new imports
+ # get the "imported" flag)
+ #
+ if eview.has_metadata_xml():
+ metadata_file = self._metadata.filepath
+ if not os.path.isfile ( metadata_file ) or all (
+ p.get ( 'imported', False ) for p in self._packages.values()
+ ):
+ shutil.copyfile ( eview.get_metadata_xml(), metadata_file )
+ #self._need_metadata = False
+ # -- end metadata.xml import
+
+ if self.DOEBUILD_IMPORTMANIFEST:
+ #self.do_ebuildmanifest ( next ( iter ( imported_ebuild_files ) ))
+ self.do_ebuildmanifest ( imported_ebuild_files [0] )
# --- end of import_ebuilds (...) ---
def write_ebuilds ( self,
diff --git a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
index 67aa937..4a4755c 100644
--- a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py
@@ -21,53 +21,8 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
"""
PackageDir class that uses the ebuild executable for Manifest writing.
"""
- #MANIFEST_ENV = None
MANIFEST_THREADSAFE = False
- @classmethod
- def init_cls ( cls ):
- cls.init_base_cls()
-
- env = roverlay.tools.ebuildenv.ManifestEnv()
- env.add_overlay_dir ( roverlay.config.get_or_fail ( 'OVERLAY.dir' ) )
- cls.MANIFEST_ENV = env
- # --- end of init_cls (...) ---
-
- def _do_ebuildmanifest ( self, ebuild_file, distdir=None ):
- """Calls doebuild_manifest().
- Returns True on success, else False. Also handles result logging.
-
- arguments:
- * ebuild_file -- ebuild file that should be used for the doebuild call
- * distdir -- distdir object (optional)
- """
- try:
- call = roverlay.tools.ebuild.doebuild_manifest (
- ebuild_file, self.logger,
- self.MANIFEST_ENV.get_env (
- distdir.get_root() if distdir is not None else (
- self.DISTROOT.get_distdir ( self.name ).get_root()
- )
- ),
- return_success=False
- )
- except Exception as err:
- self.logger.exception ( err )
- raise
- # -- end try
-
- if call.returncode == os.EX_OK:
- self.logger.debug ( "Manifest written." )
- return True
- else:
- self.logger.error (
- 'Couldn\'t create Manifest for {ebuild}! '
- 'Return code was {ret}.'.format (
- ebuild=ebuild_file, ret=call.returncode
- )
- )
- return False
- # --- end of _do_ebuildmanifest (...) ---
def _write_import_manifest ( self ):
"""Writes a Manifest file if this package has any imported ebuilds.
@@ -85,7 +40,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
# -- end try
self.logger.debug ( "Writing (import-)Manifest" )
- return self._do_ebuildmanifest ( pkg ['ebuild_file'] )
+ return self.do_ebuildmanifest ( pkg ['ebuild_file'] )
# --- end of _write_import_manifest (...) ---
def _write_manifest ( self, pkgs_for_manifest ):
@@ -108,7 +63,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
# -- end for;
- return self._do_ebuildmanifest ( ebuild_file, distdir )
+ return self.do_ebuildmanifest ( ebuild_file, distdir )
# --- end of write_manifest (...) ---
# --- end of PackageDir #ebuildmanifest ---
diff --git a/roverlay/overlay/pkgdir/packagedir_newmanifest.py b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
index fb9bd2d..5d04eb7 100644
--- a/roverlay/overlay/pkgdir/packagedir_newmanifest.py
+++ b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
@@ -29,7 +29,7 @@ class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
HASH_TYPES = frozenset ( ManifestFile.HASH_TYPES )
# Manifest entries for imported ebuilds have to be created during import
- DOEBUILD_FETCH_WITH_MANIFEST = True
+ DOEBUILD_IMPORTMANIFEST = True
def _get_manifest ( self ):
"""Returns a ManifestFile object."""
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2013-08-06 10:58 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-07-25 15:20 [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/, roverlay/overlay/ André Erdmann
-- strict thread matches above, loose matches on Subject: below --
2013-08-06 10:58 André Erdmann
2013-08-02 13:39 André Erdmann
2013-01-28 23:54 André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox