* [gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/manifest/, roverlay/overlay/
2012-07-30 8:52 [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/manifest/, roverlay/overlay/ André Erdmann
@ 2012-07-24 16:59 ` André Erdmann
0 siblings, 0 replies; 3+ messages in thread
From: André Erdmann @ 2012-07-24 16:59 UTC (permalink / raw
To: gentoo-commits
commit: ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 15:39:07 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 15:39:07 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ee870b2a
incremental overlay writing: merge write() funcs
* one write function for overlay/{root,category,package}.py
that replaces write_incremental, write, finalize_write_incremental
* also fixed some issues (proper PackageDir cleanup, make threaded writing
controllable)
geändert: roverlay/overlay/category.py
geändert: roverlay/overlay/creator.py
geändert: roverlay/overlay/manifest/__init__.py
geändert: roverlay/overlay/package.py
geändert: roverlay/overlay/root.py
---
roverlay/overlay/category.py | 141 +++++++----------
roverlay/overlay/creator.py | 29 +---
roverlay/overlay/manifest/__init__.py | 4 +-
roverlay/overlay/package.py | 278 +++++++++++++++++++++------------
roverlay/overlay/root.py | 96 +++---------
5 files changed, 266 insertions(+), 282 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 33149c4..fab6373 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -12,8 +12,6 @@ except ImportError:
from roverlay.overlay.package import PackageDir
-import roverlay.util
-
class Category ( object ):
WRITE_JOBCOUNT = 3
@@ -49,8 +47,6 @@ class Category ( object ):
incremental = self.incremental
)
self._subdirs [pkg_name] = newpkg
- if self.incremental:
- roverlay.util.dodir ( newpkg.physical_location )
finally:
self._lock.release()
@@ -76,26 +72,6 @@ class Category ( object ):
not False in ( d.empty() for d in self._subdirs.values() )
# --- end of empty (...) ---
- def finalize_write_incremental ( self ):
- for subdir in self._subdirs.values():
- if subdir.modified:
- subdir.write_incremental()
- subdir.finalize_write_incremental()
- # --- end of finalize_write_incremental (...) ---
-
- def generate_metadata ( self, **metadata_kw ):
- """Generates metadata for all packages in this category.
- Metadata are automatically generated when calling write().
-
- arguments:
- * **metadata_kw -- see PackageDir.generate_metadata(...)
-
- returns: None (implicit)
- """
- for package in self._subdirs.values():
- package.generate_metadata ( **metadata_kw )
- # --- end of generate_metadata (...) ---
-
def has ( self, subdir ):
return subdir in self._subdirs
# --- end of has (...) ---
@@ -104,13 +80,6 @@ class Category ( object ):
return os.path.isdir ( self.physical_location + os.sep + _dir )
# --- end of has_category (...) ---
- def keep_nth_latest ( self, *args, **kwargs ):
- """See package.py:PackageDir:keep_nth_latest."""
- for subdir in self._subdirs.values():
- subdir.keep_nth_latest ( *args, **kwargs )
- subdir.fs_cleanup()
- # --- end of keep_nth_latest (...) ---
-
def list_packages ( self, for_deprules=False ):
"""Lists all packages in this category.
Yields <category>/<package name> or a dict (see for_deprules below).
@@ -130,6 +99,14 @@ class Category ( object ):
yield self.name + os.sep + name
# --- end of list_packages (...) ---
+ def remove_empty ( self ):
+ """This removes all empty PackageDirs."""
+ with self._lock:
+ for key in tuple ( self._subdirs.keys() ):
+ if self._subdirs [key].check_empty():
+ del self._subdirs [key]
+ # --- end of remove_empty (...) ---
+
def scan ( self, **kw ):
"""Scans this category for existing ebuilds."""
for subdir in os.listdir ( self.physical_location ):
@@ -146,7 +123,7 @@ class Category ( object ):
package.show ( **show_kw )
# --- end of show (...) ---
- def write ( self, **write_kw ):
+ def write ( self, overwrite_ebuilds, keep_n_ebuilds, cautious ):
"""Writes this category to its filesystem location.
returns: None (implicit)
@@ -156,75 +133,75 @@ class Category ( object ):
arguments:
* q -- queue
- * write_kw --
+ * write_kw -- keywords for write(...)
"""
try:
while not q.empty():
- pkg = q.get_nowait()
- pkg.write ( write_manifest=False, **write_kw )
-
+ try:
+ pkg = q.get_nowait()
+ # remove manifest writing from threaded writing since it's
+ # single-threaded
+ pkg.write ( write_manifest=False, **write_kw )
+ #except ( Exception, KeyboardInterrupt ) as e:
+ except Exception as e:
+ # FIXME: reintroduce RERAISE
+ self.logger.exception ( e )
except queue.Empty:
pass
- except ( Exception, KeyboardInterrupt ) as e:
- self.RERAISE_EXCEPTION = e
# --- end of run_write_queue (...) ---
+ if len ( self._subdirs ) == 0: return
+
+ # determine write keyword args
+ write_kwargs = dict (
+ overwrite_ebuilds = overwrite_ebuilds,
+ keep_n_ebuilds = keep_n_ebuilds,
+ cautious = cautious,
+ )
+
+ # start writing:
+
max_jobs = self.__class__.WRITE_JOBCOUNT
- # todo len.. > 3: what's an reasonable number of min package dirs to
- # start threaded writing?
- if max_jobs > 1 and len ( self._subdirs ) > 3:
+ # FIXME/TODO: what's an reasonable number of min package dirs to
+ # start threaded writing?
+ # Ignoring it for now (and expecting enough pkg dirs)
+ if max_jobs > 1:
- # writing 1..self.__class__.WRITE_JOBCOUNT package dirs at once
+ # writing <=max_jobs package dirs at once
- modified_packages = tuple (
- p for p in self._subdirs.values() if p.modified
- )
- if len ( modified_packages ) > 0:
- write_queue = queue.Queue()
- for package in modified_packages:
- roverlay.util.dodir ( package.physical_location )
- write_queue.put_nowait ( package )
+ # don't create more workers than write jobs available
+ max_jobs = min ( max_jobs, len ( self._subdirs ) )
- workers = (
- threading.Thread (
- target=run_write_queue,
- args=( write_queue, write_kw )
- ) for n in range ( max_jobs )
- )
+ write_queue = queue.Queue()
+ for package in self._subdirs.values():
+ write_queue.put_nowait ( package )
- for w in workers: w.start()
- for w in workers: w.join()
+ workers = frozenset (
+ threading.Thread (
+ target=run_write_queue,
+ args=( write_queue, write_kwargs )
+ ) for n in range ( max_jobs )
+ )
+
+ for w in workers: w.start()
+ for w in workers: w.join()
- if hasattr ( self, 'RERAISE_EXCEPTION' ):
- raise self.RERAISE_EXCEPTION
+ self.remove_empty()
- # write manifest files
- for package in modified_packages:
- package.write_manifest()
+ # write manifest files
+ # fixme: debug print
+ #self.logger.info ( "Writing Manifest files for {}".format ( name ) )
+ print ( "Writing Manifest files ..." )
+ for package in self._subdirs.values():
+ package.write_manifest ( ignore_empty=True )
else:
for package in self._subdirs.values():
- if package.modified:
- roverlay.util.dodir ( package.physical_location )
- package.write ( write_manifest=True, **write_kw )
- # --- end of write (...) ---
+ package.write ( **write_kwargs )
- def write_incremental ( self, **write_kw ):
- """Writes this category incrementally."""
- try:
- with self._lock:
- # new package dirs could be added during overlay writing,
- # so collect the list of package dirs before iterating over it
- subdirs = tuple ( self._subdirs.values() )
-
- for subdir in subdirs:
- if subdir.modified:
- roverlay.util.dodir ( subdir.physical_location )
- subdir.write_incremental ( **write_kw )
- except Exception as e:
- self.logger.exception ( e )
- # --- end of write_incremental (...) ---
+ self.remove_empty()
+ # --- end of write (...) ---
def write_manifest ( self, **manifest_kw ):
"""Generates Manifest files for all packages in this category.
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 9de063c..d12e0a5 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -214,32 +214,15 @@ class OverlayCreator ( object ):
self.package_added.inc()
# --- end of add_package (...) ---
- def add_package_file ( self, package_file ):
- """Adds a single R package."""
- raise Exception ( "to be removed" )
- self._pkg_queue.put ( PackageInfo ( filepath=package_file ) )
- self.package_added.inc()
- # --- end of add_package_file (...) ---
-
- def add_package_files ( self, *package_files ):
- """Adds multiple R packages."""
- raise Exception ( "to be removed" )
- for p in package_files: self.add_package_file ( p )
- self.package_added.inc()
- # --- end of add_package_files (...) ---
-
def write_overlay ( self ):
"""Writes the overlay.
arguments:
"""
if self.can_write_overlay:
- if self.write_incremental:
- self.overlay.finalize_write_incremental()
- else:
- start = time.time()
- self.overlay.write()
- self._timestamp ( "overlay written", start )
+ start = time.time()
+ self.overlay.write()
+ self._timestamp ( "overlay written", start )
else:
self.logger.warning ( "Not allowed to write overlay!" )
# --- end of write_overlay (...) ---
@@ -304,7 +287,6 @@ class OverlayCreator ( object ):
self._close_workers()
close_resolver()
- self.overlay.keep_nth_latest ( n=1 )
self.closed = True
# --- end of close (...) ---
@@ -429,8 +411,9 @@ class OverlayCreator ( object ):
if self.NUMTHREADS > 0:
start = time.time()
self.logger.warning (
- "Running in concurrent mode with %i threads." % self.NUMTHREADS
- )
+ "Running in concurrent mode with {num} threads.".format (
+ num=self.NUMTHREADS
+ ) )
self._workers = frozenset (
self._get_worker ( start_now=True ) \
for n in range ( self.NUMTHREADS )
diff --git a/roverlay/overlay/manifest/__init__.py b/roverlay/overlay/manifest/__init__.py
index 9a49c7f..d9f9c0a 100644
--- a/roverlay/overlay/manifest/__init__.py
+++ b/roverlay/overlay/manifest/__init__.py
@@ -11,8 +11,6 @@ _manifest_creation = helpers.ExternalManifestCreation()
# for one directory/overlay
_manifest_lock = threading.Lock()
-
-
def create_manifest ( package_info_list, nofail=False ):
"""Creates a Manifest for package_info, using the <<best>> implementation
available.
@@ -35,4 +33,6 @@ def create_manifest ( package_info_list, nofail=False ):
raise
finally:
_manifest_lock.release()
+
+ return ret
# --- end of create_manifest (...) ---
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index a2e4d9b..d2662bd 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -120,6 +120,21 @@ class PackageDir ( object ):
return False
# --- end of add (...) ---
+ def check_empty ( self ):
+ """Similar to empty(),
+ but also removes the directory of this PackageDir.
+ """
+ if len ( self._packages ) == 0:
+ if os.path.isdir ( self.physical_location ):
+ try:
+ os.rmdir ( self.physical_location )
+ except Exception as e:
+ self.logger.exception ( e )
+ return True
+ else:
+ return False
+ # --- end of check_empty (...) ---
+
def empty ( self ):
"""Returns True if no ebuilds stored, else False.
Note that "not empty" doesn't mean "has ebuilds to write" or "has
@@ -129,25 +144,6 @@ class PackageDir ( object ):
return len ( self._packages ) == 0
# --- end of empty (...) ---
- def finalize_write_incremental ( self ):
- """Method that finalizes incremental writing, i.e. write outstanding
- ebuilds and write metadata.xml, Manifest.
- """
- with self._lock:
- if self.has_ebuilds():
- if self.modified:
- self.write_ebuilds ( overwrite=False )
- if self._need_metadata:
- self.write_metadata()
- if self._need_manifest:
- self.write_manifest()
- else:
- self.logger.critical (
- "<<todo>>: please clean up this dir: {}.".format (
- self.physical_location
- ) )
- # --- end of finalize_write_incremental (...) ---
-
def fs_cleanup ( self ):
"""Cleans up the filesystem location of this package dir.
To be called after keep_nth_latest, calls finalize_write_incremental().
@@ -158,11 +154,9 @@ class PackageDir ( object ):
# --- end of rmtree_error (...) ---
with self._lock:
- if self.has_ebuilds():
- # !!! FIXME this doesn't work if no ebuilds written, but
- # old ones removed -> DISTDIR unknown during Manifest creation
- self.finalize_write_incremental()
- elif os.path.isdir ( self.physical_location ):
+ if os.path.isdir ( self.physical_location ) \
+ and not self.has_ebuilds() \
+ :
# destroy self.physical_location
shutil.rmtree ( self.physical_location, onerror=rmtree_error )
# --- end of fs_cleanup (...) ---
@@ -193,53 +187,58 @@ class PackageDir ( object ):
arguments:
* n -- # of packages/ebuilds to keep
* cautious -- if True: be extra careful, verify that ebuilds exist
+ as file; note that this will ignore all
+ ebuilds that haven't been written to the file-
+ system yet (which implies an extra overhead,
+ you'll have to write all ebuilds first)
"""
-
- # create the list of packages to iterate over,
- # * package has to have an ebuild_file
- # * sort them by version in reverse order (latest package gets index 0)
+ def is_ebuild_cautious ( p_tuple ):
+ # package has to have an ebuild_file that exists
+ efile = p_tuple [1] ['ebuild_file' ]
+ if efile is not None:
+ return os.path.isfile ( efile )
+ else:
+ return False
+ # --- end of is_ebuild_cautious (...) ---
+
+ def is_ebuild ( p_tuple ):
+ # package has to have an ebuild_file or an ebuild entry
+ return (
+ p_tuple [1] ['ebuild_file'] or p_tuple [1] ['ebuild']
+ ) is not None
+ # --- end of is_ebuild (...) ---
+
+ # create the list of packages to iterate over (cautious/non-cautious),
+ # sort them by version in reverse order
packages = reversed ( sorted (
filter (
- lambda p : p [1] ['ebuild_file'] is not None,
- self._packages.items()
+ function=is_ebuild if not cautious else is_ebuild_cautious,
+ iterable=self._packages.items()
),
key=lambda p : p [1] ['version']
) )
+ if n < 1:
+ raise Exception ( "Must keep more than zero ebuilds." )
+
kept = 0
ecount = 0
- if not cautious:
- # could use a slice here, too
- for pvr, pkg in packages:
- ecount += 1
- if kept < n:
- printself.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
- kept += 1
- else:
- self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
- self.purge_package ( pvr )
- else:
- for pvr, pkg in packages:
- ecount += 1
- if os.path.isfile ( pkg ['ebuild_file'] ):
- if kept < n:
- self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
- kept += 1
- else:
- self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
- self.purge_package ( pvr )
- else:
- self.logger.error (
- "{efile} is assumed to exist as file but doesn't!".format (
- efile=pkg ['ebuild_file']
- ) )
- # FIXME/IGNORE: this doesn't count inexistent files as kept
+ for pvr, pkg in packages:
+ ecount += 1
+ if kept < n:
+ self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
+ kept += 1
+ else:
+ self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
+ self.purge_package ( pvr )
+
self.logger.debug (
"Kept {kept}/{total} ebuilds.".format ( kept=kept, total=ecount )
)
# FIXME: Manifest is now invalid and dir could be "empty" (no ebuilds)
+ # FIXME: force metadata regeneration
# --- end of keep_nth_latest (...) ---
def list_versions ( self ):
@@ -252,7 +251,8 @@ class PackageDir ( object ):
self._need_metadata = True
self.modified = True
if self.runtime_incremental:
- return self.write_incremental()
+ with self._lock:
+ return self.write_ebuilds ( overwrite=False )
else:
return True
# --- end of new_ebuild (...) ---
@@ -325,48 +325,104 @@ class PackageDir ( object ):
arguments:
* stream -- stream to use, defaults to sys.stderr
- returns: None (implicit)
+ returns: True
raises:
- * IOError
+ * passes all exceptions (IOError, ..)
"""
- return self.write ( shared_fh=stream )
+ self.write_ebuilds ( overwrite=True, shared_fh=stream )
+ self.write_metadata ( shared_fh=stream )
+ return True
# --- end of show (...) ---
+ def virtual_cleanup ( self ):
+ """Removes all PackageInfos from this structure that don't have an
+ 'ebuild_file' entry.
+ """
+ with self._lock:
+ # keyset may change during this method
+ for pvr in tuple ( self._packages.keys() ):
+ if self._packages [pvr] ['ebuild_file'] is None:
+ del self._packages [pvr]
+ # -- lock
+ # --- end of virtual_cleanup (...) ---
+
def write ( self,
- shared_fh=None, overwrite_ebuilds=True,
- write_ebuilds=True, write_manifest=True, write_metadata=True
+ overwrite_ebuilds=False,
+ write_ebuilds=True, write_manifest=True, write_metadata=True,
+ cleanup=True, keep_n_ebuilds=None, cautious=True
):
"""Writes this directory to its (existent!) filesystem location.
arguments:
- * shared_fh -- if set and not None: write everyting into <fh>
* write_ebuilds -- if set and False: don't write ebuilds
* write_manifest -- if set and False: don't write the Manifest file
* write_metadata -- if set and False: don't write the metadata file
- * overwrite_ebuilds -- if set and False: don't overwrite ebuilds
-
- returns: None (implicit)
+ * overwrite_ebuilds -- whether to overwrite ebuilds,
+ None means autodetect, enable overwriting
+ if not modified since last write
+ Defaults to False
+ * cleanup -- clean up after writing
+ Defaults to True
+ * keep_n_ebuilds -- # of ebuilds to keep (remove all others),
+ Defaults to None (disable) and implies cleanup
+ * cautious -- be cautious when keeping the nth latest ebuilds,
+ this has some overhead
+ Defaults to True
+
+ returns: success (True/False)
raises:
- * IOError (?)
+ * passes IOError
"""
- with self._lock:
- # mkdir not required here, overlay.Category does this
+ # NOTE, replaces:
+ # * old write: overwrite_ebuilds=True
+ # * finalize_write_incremental : no extra args
+ # * write_incremental : write_manifest=False, write_metadata=False,
+ # cleanup=False (or use write_ebuilds)
+ # BREAKS: show(), which has its own method/function now
- # write ebuilds
- if write_ebuilds:
- self.write_ebuilds (
- overwrite=overwrite_ebuilds, shared_fh=shared_fh
- )
+ cleanup = cleanup or ( keep_n_ebuilds is not None )
- # write metadata
- if write_metadata:
- self.write_metadata ( shared_fh=shared_fh )
+ success = True
+ with self._lock:
+ if self.has_ebuilds():
+ # not cautious: remove ebuilds before writing them
+ if not cautious and keep_n_ebuilds is not None:
+ self.keep_nth_latest ( n=keep_n_ebuilds, cautious=False )
+
+ # write ebuilds
+ if self.modified and write_ebuilds:
+ success = self.write_ebuilds (
+ # None ~ not modified
+ overwrite = overwrite_ebuilds \
+ if overwrite_ebuilds is not None \
+ else not self.modified
+ )
- # write manifest (only if shared_fh is None)
- if write_manifest and shared_fh is None:
- self.write_manifest()
+ # cautious: remove ebuilds after writing them
+ if cautious and keep_n_ebuilds is not None:
+ self.keep_nth_latest ( n=keep_n_ebuilds, cautious=True )
+
+ # write metadata
+ if self._need_metadata and write_metadata:
+ # don't mess around with short-circuit bool evaluation
+ if not self.write_metadata():
+ success = False
+
+ # write manifest (only if shared_fh is None)
+ if self._need_manifest and write_manifest:
+ if not self.write_manifest():
+ success = False
+ # -- has_ebuilds?
+
+ if cleanup:
+ self.virtual_cleanup()
+ self.fs_cleanup()
+
+ # FIXME / TODO call fs_cleanup
+ # -- lock
+ return success
# --- end of write (...) ---
def write_ebuilds ( self, overwrite, shared_fh=None ):
@@ -390,7 +446,6 @@ class PackageDir ( object ):
"""
_success = False
try:
- util.dodir ( self.physical_location )
fh = open ( efile, 'w' ) if shared_fh is None else shared_fh
if ebuild_header is not None:
fh.write ( str ( ebuild_header ) )
@@ -426,7 +481,14 @@ class PackageDir ( object ):
all_ebuilds_written = True
+ # don't call dodir if shared_fh is set
+ hasdir = bool ( shared_fh is not None )
+
for efile, p_info in ebuilds_to_write():
+ if not hasdir:
+ util.dodir ( self.physical_location, mkdir_p=True )
+ hasdir = True
+
if write_ebuild ( efile, p_info ['ebuild'] ):
self._need_manifest = True
@@ -452,17 +514,12 @@ class PackageDir ( object ):
return all_ebuilds_written
# --- end of write_ebuilds (...) ---
- def write_incremental ( self ):
- with self._lock:
- return self.write_ebuilds ( overwrite=False )
- # --- end of write_incremental (...) ---
-
- def write_manifest ( self ):
+ def write_manifest ( self, ignore_empty=False ):
"""Generates and writes the Manifest file for this package.
expects: called after writing metadata/ebuilds
- returns: None (implicit)
+ returns: success (True/False)
raises:
* Exception if no ebuild exists
@@ -471,47 +528,64 @@ class PackageDir ( object ):
# it should be sufficient to call create_manifest for one ebuild,
# choosing the latest one that exists in self.physical_location and
# has enough data (DISTDIR, EBUILD_FILE) for this task.
+ # Additionally, all DISTDIRs (multiple repos, sub directories) have
+ # to be collected and passed to Manifest creation.
+ # => collect suitable PackageInfo objects from self._packages
#
- # metadata.xml's full path cannot be used for manifest creation here
- # 'cause DISTDIR would be unknown
- #
-
- # collect suitable PackageInfo instances
pkgs_for_manifest = tuple (
p for p in self._packages.values() \
if p.has ( 'distdir', 'ebuild_file' )
)
if pkgs_for_manifest:
- manifest.create_manifest ( pkgs_for_manifest, nofail=False )
- self._need_manifest = False
+ if manifest.create_manifest ( pkgs_for_manifest, nofail=False ):
+ self._need_manifest = False
+ return True
+ elif ignore_empty:
+ return True
else:
+ # FIXME: debug statements
+ # FIXME: remove excpetion, maybe delete Manifest in this case,..
+ for pvr, p in self._packages.items():
+ print ( "{} {} ebuild={} efile={} has={}".format (
+ pvr, p, p.has ('ebuild'), p ['ebuild_file'], self.has_ebuilds()
+ ) )
+
raise Exception (
- "No ebuild written so far! I really don't know what do to!"
- )
+ 'In {mydir}: No ebuild written so far! '
+ 'I really don\'t know what do to!'.format (
+ mydir=self.physical_location
+ ) )
+
+ return False
# --- end of write_manifest (...) ---
def write_metadata ( self, shared_fh=None ):
- """Writes metadata for this package."""
+ """Writes metadata for this package.
+
+ returns: success (True/False)
+ """
+ success = False
try:
self.generate_metadata ( skip_if_existent=True )
if shared_fh is None:
+ util.dodir ( self.physical_location, mkdir_p=True )
if self._metadata.write():
self._need_metadata = False
self._need_manifest = True
- return True
+ success = True
else:
self.logger.error (
"Failed to write metadata file {}.".format (
self._metadata.filepath
)
)
- return False
else:
self._metadata.show ( shared_fh )
- return True
+ success = True
except Exception as e:
self.logger.exception ( e )
- return False
+
+ return success
# --- end of write_metadata (...) ---
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 363bdb5..219fd70 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -48,10 +48,10 @@ class Overlay ( object ):
self.incremental = incremental
if self.incremental:
-
- self._incremental_write_lock = threading.Lock()
+ # this is multiple-run incremental writing (in contrast to runtime
+ # incremental writing, which writes ebuilds as soon as they're
+ # ready) FIXME: split incremental <-> runtime_incremental
self.scan()
- self._init_overlay ( reimport_eclass=True )
# --- end of __init__ (...) ---
@@ -73,8 +73,6 @@ class Overlay ( object ):
incremental=self.incremental
)
self._categories [category] = newcat
- if self.incremental:
- util.dodir ( newcat.physical_location )
finally:
self._catlock.release()
@@ -226,50 +224,10 @@ class Overlay ( object ):
return cat.add ( package_info )
# --- end of add (...) ---
- def finalize_write_incremental ( self ):
- """Writes metadata + Manifest for all packages."""
- for cat in self._categories.values():
- cat.finalize_write_incremental()
- # --- end of finalize_incremental (...) ---
-
- def generate_manifest ( self, **manifest_kw ):
- """Generates Manifest files for all ebuilds in this overlay that exist
- physically/in filesystem.
- Manifest files are automatically created when calling write().
-
- arguments:
- * **manifest_kw -- see PackageDir.generate_manifest(...)
-
- returns: None (implicit)
- """
- for cat in self._categories.values():
- cat.generate_manifest ( **manifest_kw )
- # --- end of generate_manifest (...) ---
-
- def generate_metadata ( self, **metadata_kw ):
- """Tells the overlay's categories to create metadata.
- You don't have to call this before write()/show() unless you want to use
- special metadata options.
-
- arguments:
- * **metadata_kw -- keywords for package.PackageDir.generate_metadata(...)
-
- returns: None (implicit)
- """
- for cat in self._categories.values():
- cat.generate_metadata ( **metadata_kw )
- # --- end of generate_metadata (...) ---
-
def has_dir ( self, _dir ):
return os.path.isdir ( self.physical_location + os.sep + _dir )
# --- end of has_category (...) ---
- def keep_nth_latest ( self, *args, **kwargs ):
- """See package.py:PackageDir:keep_nth_latest."""
- for cat in self._categories.values():
- cat.keep_nth_latest ( *args, **kwargs )
- # --- end of keep_nth_latest (...) ---
-
def list_packages ( self, for_deprules=True ):
for cat in self._categories.values():
for package in cat.list_packages ( for_deprules=True ):
@@ -313,9 +271,9 @@ class Overlay ( object ):
cat.show ( **show_kw )
# --- end of show (...) ---
- def write ( self, **write_kw ):
+ def write ( self ):
"""Writes the overlay to its physical location (filesystem), including
- metadata and Manifest files.
+ metadata and Manifest files as well as cleanup actions.
arguments:
* **write_kw -- keywords for package.PackageDir.write(...)
@@ -327,35 +285,27 @@ class Overlay ( object ):
! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
when ebuild creation is done.
"""
- raise Exception ( "to be removed/replaced" )
- # writing profiles/ here, rewriting categories/ later
self._init_overlay ( reimport_eclass=True )
for cat in self._categories.values():
- if not cat.empty():
- util.dodir ( cat.physical_location )
- cat.write ( **write_kw )
+ cat.write (
+ overwrite_ebuilds=False,
+ keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
+ cautious=True
+ )
# --- end of write (...) ---
- def write_incremental ( self, **write_kw ):
- """Writes all ebuilds that have been modified since the last write call.
- Note that there are currently two modes of incremental writing:
- (a) per-PackageDir incremental writing triggered by the new_ebuild()
- event method and (b) "batched" incremental writing (this method) which
- writes all modified PackageDirs.
- """
- # FIXME merge with write(), making incremental writing the only option
- # FIXME finalize_write_incremental?
- if not self._incremental_write_lock.acquire():
- # another incremental write is running, drop this request
- return
+ def write_manifest ( self, **manifest_kw ):
+ """Generates Manifest files for all ebuilds in this overlay that exist
+ physically/in filesystem.
+ Manifest files are automatically created when calling write().
- try:
- util.dodir ( self.physical_location )
- cats = tuple ( self._categories.values() )
- for cat in cats:
- util.dodir ( cat.physical_location )
- cat.write_incremental ( **write_kw )
- finally:
- self._incremental_write_lock.release()
- # --- end of write_incremental (...) ---
+ arguments:
+ * **manifest_kw -- see PackageDir.generate_manifest(...)
+
+ returns: None (implicit)
+ """
+ # FIXME: it would be good to ensure that profiles/categories exist
+ for cat in self._categories.values():
+ cat.write_manifest ( **manifest_kw )
+ # --- end of write_manifest (...) ---
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/manifest/, roverlay/overlay/
2012-07-20 16:36 André Erdmann
@ 2012-07-30 8:52 ` André Erdmann
0 siblings, 0 replies; 3+ messages in thread
From: André Erdmann @ 2012-07-30 8:52 UTC (permalink / raw
To: gentoo-commits
commit: 9c53db664b4ff66472394582f6549a0509eb4df2
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 20 16:36:33 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 20 16:36:33 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9c53db66
fix manifest creation
---
roverlay/overlay/manifest/helpers.py | 5 -----
roverlay/overlay/package.py | 4 ++--
roverlay/overlay/root.py | 5 +----
3 files changed, 3 insertions(+), 11 deletions(-)
diff --git a/roverlay/overlay/manifest/helpers.py b/roverlay/overlay/manifest/helpers.py
index 168a119..d49ecf7 100644
--- a/roverlay/overlay/manifest/helpers.py
+++ b/roverlay/overlay/manifest/helpers.py
@@ -46,11 +46,6 @@ class ExternalManifestCreation ( object ):
raises: *passes Exceptions from failed config lookups
"""
- self.logger.critical (
- "Manifest creation is broken! PORTAGE_RO_DISTDIRS does not work."
- )
- return False
-
distdirs = ' '.join ( set (
p ['distdir'] for p in package_info_list
) )
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 5b73788..2ea4cb2 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -67,8 +67,8 @@ class PackageDir ( object ):
if shortver in self._packages:
# package exists, check if it existed before script invocation
if self._packages [shortver] ['physical_only']:
- if not skip_if_physical:
- # ignore ebuilds that exist as file
+ if add_if_physical:
+ # else ignore ebuilds that exist as file
self._packages [shortver] = package_info
added = True
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 4766b02..51239e2 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -58,9 +58,6 @@ class Overlay ( object ):
self.scan()
self._init_overlay ( reimport_eclass=True, make_profiles_dir=True )
- for c in self.list_packages ( for_deprules=True ):
- print ( str ( c ) )
-
# --- end of __init__ (...) ---
def scan ( self, **kw ):
@@ -204,9 +201,9 @@ class Overlay ( object ):
def finalize_write_incremental ( self ):
"""Writes metadata + Manifest for all packages."""
+ self._write_categories ( only_active=True )
for cat in self._categories.values():
cat.finalize_write_incremental()
- self._write_categories ( only_active=True )
# --- end of finalize_incremental (...) ---
def generate_metadata ( self, **metadata_kw ):
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/manifest/, roverlay/overlay/
@ 2012-07-30 8:52 André Erdmann
2012-07-24 16:59 ` [gentoo-commits] proj/R_overlay:overlay_wip " André Erdmann
0 siblings, 1 reply; 3+ messages in thread
From: André Erdmann @ 2012-07-30 8:52 UTC (permalink / raw
To: gentoo-commits
commit: ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 15:39:07 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 15:39:07 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ee870b2a
incremental overlay writing: merge write() funcs
* one write function for overlay/{root,category,package}.py
that replaces write_incremental, write, finalize_write_incremental
* also fixed some issues (proper PackageDir cleanup, make threaded writing
controllable)
geändert: roverlay/overlay/category.py
geändert: roverlay/overlay/creator.py
geändert: roverlay/overlay/manifest/__init__.py
geändert: roverlay/overlay/package.py
geändert: roverlay/overlay/root.py
---
roverlay/overlay/category.py | 141 +++++++----------
roverlay/overlay/creator.py | 29 +---
roverlay/overlay/manifest/__init__.py | 4 +-
roverlay/overlay/package.py | 278 +++++++++++++++++++++------------
roverlay/overlay/root.py | 96 +++---------
5 files changed, 266 insertions(+), 282 deletions(-)
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 33149c4..fab6373 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -12,8 +12,6 @@ except ImportError:
from roverlay.overlay.package import PackageDir
-import roverlay.util
-
class Category ( object ):
WRITE_JOBCOUNT = 3
@@ -49,8 +47,6 @@ class Category ( object ):
incremental = self.incremental
)
self._subdirs [pkg_name] = newpkg
- if self.incremental:
- roverlay.util.dodir ( newpkg.physical_location )
finally:
self._lock.release()
@@ -76,26 +72,6 @@ class Category ( object ):
not False in ( d.empty() for d in self._subdirs.values() )
# --- end of empty (...) ---
- def finalize_write_incremental ( self ):
- for subdir in self._subdirs.values():
- if subdir.modified:
- subdir.write_incremental()
- subdir.finalize_write_incremental()
- # --- end of finalize_write_incremental (...) ---
-
- def generate_metadata ( self, **metadata_kw ):
- """Generates metadata for all packages in this category.
- Metadata are automatically generated when calling write().
-
- arguments:
- * **metadata_kw -- see PackageDir.generate_metadata(...)
-
- returns: None (implicit)
- """
- for package in self._subdirs.values():
- package.generate_metadata ( **metadata_kw )
- # --- end of generate_metadata (...) ---
-
def has ( self, subdir ):
return subdir in self._subdirs
# --- end of has (...) ---
@@ -104,13 +80,6 @@ class Category ( object ):
return os.path.isdir ( self.physical_location + os.sep + _dir )
# --- end of has_category (...) ---
- def keep_nth_latest ( self, *args, **kwargs ):
- """See package.py:PackageDir:keep_nth_latest."""
- for subdir in self._subdirs.values():
- subdir.keep_nth_latest ( *args, **kwargs )
- subdir.fs_cleanup()
- # --- end of keep_nth_latest (...) ---
-
def list_packages ( self, for_deprules=False ):
"""Lists all packages in this category.
Yields <category>/<package name> or a dict (see for_deprules below).
@@ -130,6 +99,14 @@ class Category ( object ):
yield self.name + os.sep + name
# --- end of list_packages (...) ---
+ def remove_empty ( self ):
+ """This removes all empty PackageDirs."""
+ with self._lock:
+ for key in tuple ( self._subdirs.keys() ):
+ if self._subdirs [key].check_empty():
+ del self._subdirs [key]
+ # --- end of remove_empty (...) ---
+
def scan ( self, **kw ):
"""Scans this category for existing ebuilds."""
for subdir in os.listdir ( self.physical_location ):
@@ -146,7 +123,7 @@ class Category ( object ):
package.show ( **show_kw )
# --- end of show (...) ---
- def write ( self, **write_kw ):
+ def write ( self, overwrite_ebuilds, keep_n_ebuilds, cautious ):
"""Writes this category to its filesystem location.
returns: None (implicit)
@@ -156,75 +133,75 @@ class Category ( object ):
arguments:
* q -- queue
- * write_kw --
+ * write_kw -- keywords for write(...)
"""
try:
while not q.empty():
- pkg = q.get_nowait()
- pkg.write ( write_manifest=False, **write_kw )
-
+ try:
+ pkg = q.get_nowait()
+ # remove manifest writing from threaded writing since it's
+ # single-threaded
+ pkg.write ( write_manifest=False, **write_kw )
+ #except ( Exception, KeyboardInterrupt ) as e:
+ except Exception as e:
+ # FIXME: reintroduce RERAISE
+ self.logger.exception ( e )
except queue.Empty:
pass
- except ( Exception, KeyboardInterrupt ) as e:
- self.RERAISE_EXCEPTION = e
# --- end of run_write_queue (...) ---
+ if len ( self._subdirs ) == 0: return
+
+ # determine write keyword args
+ write_kwargs = dict (
+ overwrite_ebuilds = overwrite_ebuilds,
+ keep_n_ebuilds = keep_n_ebuilds,
+ cautious = cautious,
+ )
+
+ # start writing:
+
max_jobs = self.__class__.WRITE_JOBCOUNT
- # todo len.. > 3: what's an reasonable number of min package dirs to
- # start threaded writing?
- if max_jobs > 1 and len ( self._subdirs ) > 3:
+ # FIXME/TODO: what's an reasonable number of min package dirs to
+ # start threaded writing?
+ # Ignoring it for now (and expecting enough pkg dirs)
+ if max_jobs > 1:
- # writing 1..self.__class__.WRITE_JOBCOUNT package dirs at once
+ # writing <=max_jobs package dirs at once
- modified_packages = tuple (
- p for p in self._subdirs.values() if p.modified
- )
- if len ( modified_packages ) > 0:
- write_queue = queue.Queue()
- for package in modified_packages:
- roverlay.util.dodir ( package.physical_location )
- write_queue.put_nowait ( package )
+ # don't create more workers than write jobs available
+ max_jobs = min ( max_jobs, len ( self._subdirs ) )
- workers = (
- threading.Thread (
- target=run_write_queue,
- args=( write_queue, write_kw )
- ) for n in range ( max_jobs )
- )
+ write_queue = queue.Queue()
+ for package in self._subdirs.values():
+ write_queue.put_nowait ( package )
- for w in workers: w.start()
- for w in workers: w.join()
+ workers = frozenset (
+ threading.Thread (
+ target=run_write_queue,
+ args=( write_queue, write_kwargs )
+ ) for n in range ( max_jobs )
+ )
+
+ for w in workers: w.start()
+ for w in workers: w.join()
- if hasattr ( self, 'RERAISE_EXCEPTION' ):
- raise self.RERAISE_EXCEPTION
+ self.remove_empty()
- # write manifest files
- for package in modified_packages:
- package.write_manifest()
+ # write manifest files
+ # fixme: debug print
+ #self.logger.info ( "Writing Manifest files for {}".format ( name ) )
+ print ( "Writing Manifest files ..." )
+ for package in self._subdirs.values():
+ package.write_manifest ( ignore_empty=True )
else:
for package in self._subdirs.values():
- if package.modified:
- roverlay.util.dodir ( package.physical_location )
- package.write ( write_manifest=True, **write_kw )
- # --- end of write (...) ---
+ package.write ( **write_kwargs )
- def write_incremental ( self, **write_kw ):
- """Writes this category incrementally."""
- try:
- with self._lock:
- # new package dirs could be added during overlay writing,
- # so collect the list of package dirs before iterating over it
- subdirs = tuple ( self._subdirs.values() )
-
- for subdir in subdirs:
- if subdir.modified:
- roverlay.util.dodir ( subdir.physical_location )
- subdir.write_incremental ( **write_kw )
- except Exception as e:
- self.logger.exception ( e )
- # --- end of write_incremental (...) ---
+ self.remove_empty()
+ # --- end of write (...) ---
def write_manifest ( self, **manifest_kw ):
"""Generates Manifest files for all packages in this category.
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 9de063c..d12e0a5 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -214,32 +214,15 @@ class OverlayCreator ( object ):
self.package_added.inc()
# --- end of add_package (...) ---
- def add_package_file ( self, package_file ):
- """Adds a single R package."""
- raise Exception ( "to be removed" )
- self._pkg_queue.put ( PackageInfo ( filepath=package_file ) )
- self.package_added.inc()
- # --- end of add_package_file (...) ---
-
- def add_package_files ( self, *package_files ):
- """Adds multiple R packages."""
- raise Exception ( "to be removed" )
- for p in package_files: self.add_package_file ( p )
- self.package_added.inc()
- # --- end of add_package_files (...) ---
-
def write_overlay ( self ):
"""Writes the overlay.
arguments:
"""
if self.can_write_overlay:
- if self.write_incremental:
- self.overlay.finalize_write_incremental()
- else:
- start = time.time()
- self.overlay.write()
- self._timestamp ( "overlay written", start )
+ start = time.time()
+ self.overlay.write()
+ self._timestamp ( "overlay written", start )
else:
self.logger.warning ( "Not allowed to write overlay!" )
# --- end of write_overlay (...) ---
@@ -304,7 +287,6 @@ class OverlayCreator ( object ):
self._close_workers()
close_resolver()
- self.overlay.keep_nth_latest ( n=1 )
self.closed = True
# --- end of close (...) ---
@@ -429,8 +411,9 @@ class OverlayCreator ( object ):
if self.NUMTHREADS > 0:
start = time.time()
self.logger.warning (
- "Running in concurrent mode with %i threads." % self.NUMTHREADS
- )
+ "Running in concurrent mode with {num} threads.".format (
+ num=self.NUMTHREADS
+ ) )
self._workers = frozenset (
self._get_worker ( start_now=True ) \
for n in range ( self.NUMTHREADS )
diff --git a/roverlay/overlay/manifest/__init__.py b/roverlay/overlay/manifest/__init__.py
index 9a49c7f..d9f9c0a 100644
--- a/roverlay/overlay/manifest/__init__.py
+++ b/roverlay/overlay/manifest/__init__.py
@@ -11,8 +11,6 @@ _manifest_creation = helpers.ExternalManifestCreation()
# for one directory/overlay
_manifest_lock = threading.Lock()
-
-
def create_manifest ( package_info_list, nofail=False ):
"""Creates a Manifest for package_info, using the <<best>> implementation
available.
@@ -35,4 +33,6 @@ def create_manifest ( package_info_list, nofail=False ):
raise
finally:
_manifest_lock.release()
+
+ return ret
# --- end of create_manifest (...) ---
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index a2e4d9b..d2662bd 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -120,6 +120,21 @@ class PackageDir ( object ):
return False
# --- end of add (...) ---
+ def check_empty ( self ):
+ """Similar to empty(),
+ but also removes the directory of this PackageDir.
+ """
+ if len ( self._packages ) == 0:
+ if os.path.isdir ( self.physical_location ):
+ try:
+ os.rmdir ( self.physical_location )
+ except Exception as e:
+ self.logger.exception ( e )
+ return True
+ else:
+ return False
+ # --- end of check_empty (...) ---
+
def empty ( self ):
"""Returns True if no ebuilds stored, else False.
Note that "not empty" doesn't mean "has ebuilds to write" or "has
@@ -129,25 +144,6 @@ class PackageDir ( object ):
return len ( self._packages ) == 0
# --- end of empty (...) ---
- def finalize_write_incremental ( self ):
- """Method that finalizes incremental writing, i.e. write outstanding
- ebuilds and write metadata.xml, Manifest.
- """
- with self._lock:
- if self.has_ebuilds():
- if self.modified:
- self.write_ebuilds ( overwrite=False )
- if self._need_metadata:
- self.write_metadata()
- if self._need_manifest:
- self.write_manifest()
- else:
- self.logger.critical (
- "<<todo>>: please clean up this dir: {}.".format (
- self.physical_location
- ) )
- # --- end of finalize_write_incremental (...) ---
-
def fs_cleanup ( self ):
"""Cleans up the filesystem location of this package dir.
To be called after keep_nth_latest, calls finalize_write_incremental().
@@ -158,11 +154,9 @@ class PackageDir ( object ):
# --- end of rmtree_error (...) ---
with self._lock:
- if self.has_ebuilds():
- # !!! FIXME this doesn't work if no ebuilds written, but
- # old ones removed -> DISTDIR unknown during Manifest creation
- self.finalize_write_incremental()
- elif os.path.isdir ( self.physical_location ):
+ if os.path.isdir ( self.physical_location ) \
+ and not self.has_ebuilds() \
+ :
# destroy self.physical_location
shutil.rmtree ( self.physical_location, onerror=rmtree_error )
# --- end of fs_cleanup (...) ---
@@ -193,53 +187,58 @@ class PackageDir ( object ):
arguments:
* n -- # of packages/ebuilds to keep
* cautious -- if True: be extra careful, verify that ebuilds exist
+ as file; note that this will ignore all
+ ebuilds that haven't been written to the file-
+ system yet (which implies an extra overhead,
+ you'll have to write all ebuilds first)
"""
-
- # create the list of packages to iterate over,
- # * package has to have an ebuild_file
- # * sort them by version in reverse order (latest package gets index 0)
+ def is_ebuild_cautious ( p_tuple ):
+ # package has to have an ebuild_file that exists
+ efile = p_tuple [1] ['ebuild_file' ]
+ if efile is not None:
+ return os.path.isfile ( efile )
+ else:
+ return False
+ # --- end of is_ebuild_cautious (...) ---
+
+ def is_ebuild ( p_tuple ):
+ # package has to have an ebuild_file or an ebuild entry
+ return (
+ p_tuple [1] ['ebuild_file'] or p_tuple [1] ['ebuild']
+ ) is not None
+ # --- end of is_ebuild (...) ---
+
+ # create the list of packages to iterate over (cautious/non-cautious),
+ # sort them by version in reverse order
packages = reversed ( sorted (
filter (
- lambda p : p [1] ['ebuild_file'] is not None,
- self._packages.items()
+ function=is_ebuild if not cautious else is_ebuild_cautious,
+ iterable=self._packages.items()
),
key=lambda p : p [1] ['version']
) )
+ if n < 1:
+ raise Exception ( "Must keep more than zero ebuilds." )
+
kept = 0
ecount = 0
- if not cautious:
- # could use a slice here, too
- for pvr, pkg in packages:
- ecount += 1
- if kept < n:
- printself.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
- kept += 1
- else:
- self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
- self.purge_package ( pvr )
- else:
- for pvr, pkg in packages:
- ecount += 1
- if os.path.isfile ( pkg ['ebuild_file'] ):
- if kept < n:
- self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
- kept += 1
- else:
- self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
- self.purge_package ( pvr )
- else:
- self.logger.error (
- "{efile} is assumed to exist as file but doesn't!".format (
- efile=pkg ['ebuild_file']
- ) )
- # FIXME/IGNORE: this doesn't count inexistent files as kept
+ for pvr, pkg in packages:
+ ecount += 1
+ if kept < n:
+ self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
+ kept += 1
+ else:
+ self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
+ self.purge_package ( pvr )
+
self.logger.debug (
"Kept {kept}/{total} ebuilds.".format ( kept=kept, total=ecount )
)
# FIXME: Manifest is now invalid and dir could be "empty" (no ebuilds)
+ # FIXME: force metadata regeneration
# --- end of keep_nth_latest (...) ---
def list_versions ( self ):
@@ -252,7 +251,8 @@ class PackageDir ( object ):
self._need_metadata = True
self.modified = True
if self.runtime_incremental:
- return self.write_incremental()
+ with self._lock:
+ return self.write_ebuilds ( overwrite=False )
else:
return True
# --- end of new_ebuild (...) ---
@@ -325,48 +325,104 @@ class PackageDir ( object ):
arguments:
* stream -- stream to use, defaults to sys.stderr
- returns: None (implicit)
+ returns: True
raises:
- * IOError
+ * passes all exceptions (IOError, ..)
"""
- return self.write ( shared_fh=stream )
+ self.write_ebuilds ( overwrite=True, shared_fh=stream )
+ self.write_metadata ( shared_fh=stream )
+ return True
# --- end of show (...) ---
+ def virtual_cleanup ( self ):
+ """Removes all PackageInfos from this structure that don't have an
+ 'ebuild_file' entry.
+ """
+ with self._lock:
+ # keyset may change during this method
+ for pvr in tuple ( self._packages.keys() ):
+ if self._packages [pvr] ['ebuild_file'] is None:
+ del self._packages [pvr]
+ # -- lock
+ # --- end of virtual_cleanup (...) ---
+
def write ( self,
- shared_fh=None, overwrite_ebuilds=True,
- write_ebuilds=True, write_manifest=True, write_metadata=True
+ overwrite_ebuilds=False,
+ write_ebuilds=True, write_manifest=True, write_metadata=True,
+ cleanup=True, keep_n_ebuilds=None, cautious=True
):
"""Writes this directory to its (existent!) filesystem location.
arguments:
- * shared_fh -- if set and not None: write everyting into <fh>
* write_ebuilds -- if set and False: don't write ebuilds
* write_manifest -- if set and False: don't write the Manifest file
* write_metadata -- if set and False: don't write the metadata file
- * overwrite_ebuilds -- if set and False: don't overwrite ebuilds
-
- returns: None (implicit)
+ * overwrite_ebuilds -- whether to overwrite ebuilds,
+ None means autodetect, enable overwriting
+ if not modified since last write
+ Defaults to False
+ * cleanup -- clean up after writing
+ Defaults to True
+ * keep_n_ebuilds -- # of ebuilds to keep (remove all others),
+ Defaults to None (disable) and implies cleanup
+ * cautious -- be cautious when keeping the nth latest ebuilds,
+ this has some overhead
+ Defaults to True
+
+ returns: success (True/False)
raises:
- * IOError (?)
+ * passes IOError
"""
- with self._lock:
- # mkdir not required here, overlay.Category does this
+ # NOTE, replaces:
+ # * old write: overwrite_ebuilds=True
+ # * finalize_write_incremental : no extra args
+ # * write_incremental : write_manifest=False, write_metadata=False,
+ # cleanup=False (or use write_ebuilds)
+ # BREAKS: show(), which has its own method/function now
- # write ebuilds
- if write_ebuilds:
- self.write_ebuilds (
- overwrite=overwrite_ebuilds, shared_fh=shared_fh
- )
+ cleanup = cleanup or ( keep_n_ebuilds is not None )
- # write metadata
- if write_metadata:
- self.write_metadata ( shared_fh=shared_fh )
+ success = True
+ with self._lock:
+ if self.has_ebuilds():
+ # not cautious: remove ebuilds before writing them
+ if not cautious and keep_n_ebuilds is not None:
+ self.keep_nth_latest ( n=keep_n_ebuilds, cautious=False )
+
+ # write ebuilds
+ if self.modified and write_ebuilds:
+ success = self.write_ebuilds (
+ # None ~ not modified
+ overwrite = overwrite_ebuilds \
+ if overwrite_ebuilds is not None \
+ else not self.modified
+ )
- # write manifest (only if shared_fh is None)
- if write_manifest and shared_fh is None:
- self.write_manifest()
+ # cautious: remove ebuilds after writing them
+ if cautious and keep_n_ebuilds is not None:
+ self.keep_nth_latest ( n=keep_n_ebuilds, cautious=True )
+
+ # write metadata
+ if self._need_metadata and write_metadata:
+ # don't mess around with short-circuit bool evaluation
+ if not self.write_metadata():
+ success = False
+
+ # write manifest (only if shared_fh is None)
+ if self._need_manifest and write_manifest:
+ if not self.write_manifest():
+ success = False
+ # -- has_ebuilds?
+
+ if cleanup:
+ self.virtual_cleanup()
+ self.fs_cleanup()
+
+ # FIXME / TODO call fs_cleanup
+ # -- lock
+ return success
# --- end of write (...) ---
def write_ebuilds ( self, overwrite, shared_fh=None ):
@@ -390,7 +446,6 @@ class PackageDir ( object ):
"""
_success = False
try:
- util.dodir ( self.physical_location )
fh = open ( efile, 'w' ) if shared_fh is None else shared_fh
if ebuild_header is not None:
fh.write ( str ( ebuild_header ) )
@@ -426,7 +481,14 @@ class PackageDir ( object ):
all_ebuilds_written = True
+ # don't call dodir if shared_fh is set
+ hasdir = bool ( shared_fh is not None )
+
for efile, p_info in ebuilds_to_write():
+ if not hasdir:
+ util.dodir ( self.physical_location, mkdir_p=True )
+ hasdir = True
+
if write_ebuild ( efile, p_info ['ebuild'] ):
self._need_manifest = True
@@ -452,17 +514,12 @@ class PackageDir ( object ):
return all_ebuilds_written
# --- end of write_ebuilds (...) ---
- def write_incremental ( self ):
- with self._lock:
- return self.write_ebuilds ( overwrite=False )
- # --- end of write_incremental (...) ---
-
- def write_manifest ( self ):
+ def write_manifest ( self, ignore_empty=False ):
"""Generates and writes the Manifest file for this package.
expects: called after writing metadata/ebuilds
- returns: None (implicit)
+ returns: success (True/False)
raises:
* Exception if no ebuild exists
@@ -471,47 +528,64 @@ class PackageDir ( object ):
# it should be sufficient to call create_manifest for one ebuild,
# choosing the latest one that exists in self.physical_location and
# has enough data (DISTDIR, EBUILD_FILE) for this task.
+ # Additionally, all DISTDIRs (multiple repos, sub directories) have
+ # to be collected and passed to Manifest creation.
+ # => collect suitable PackageInfo objects from self._packages
#
- # metadata.xml's full path cannot be used for manifest creation here
- # 'cause DISTDIR would be unknown
- #
-
- # collect suitable PackageInfo instances
pkgs_for_manifest = tuple (
p for p in self._packages.values() \
if p.has ( 'distdir', 'ebuild_file' )
)
if pkgs_for_manifest:
- manifest.create_manifest ( pkgs_for_manifest, nofail=False )
- self._need_manifest = False
+ if manifest.create_manifest ( pkgs_for_manifest, nofail=False ):
+ self._need_manifest = False
+ return True
+ elif ignore_empty:
+ return True
else:
+ # FIXME: debug statements
+ # FIXME: remove excpetion, maybe delete Manifest in this case,..
+ for pvr, p in self._packages.items():
+ print ( "{} {} ebuild={} efile={} has={}".format (
+ pvr, p, p.has ('ebuild'), p ['ebuild_file'], self.has_ebuilds()
+ ) )
+
raise Exception (
- "No ebuild written so far! I really don't know what do to!"
- )
+ 'In {mydir}: No ebuild written so far! '
+ 'I really don\'t know what do to!'.format (
+ mydir=self.physical_location
+ ) )
+
+ return False
# --- end of write_manifest (...) ---
def write_metadata ( self, shared_fh=None ):
- """Writes metadata for this package."""
+ """Writes metadata for this package.
+
+ returns: success (True/False)
+ """
+ success = False
try:
self.generate_metadata ( skip_if_existent=True )
if shared_fh is None:
+ util.dodir ( self.physical_location, mkdir_p=True )
if self._metadata.write():
self._need_metadata = False
self._need_manifest = True
- return True
+ success = True
else:
self.logger.error (
"Failed to write metadata file {}.".format (
self._metadata.filepath
)
)
- return False
else:
self._metadata.show ( shared_fh )
- return True
+ success = True
except Exception as e:
self.logger.exception ( e )
- return False
+
+ return success
# --- end of write_metadata (...) ---
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 363bdb5..219fd70 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -48,10 +48,10 @@ class Overlay ( object ):
self.incremental = incremental
if self.incremental:
-
- self._incremental_write_lock = threading.Lock()
+ # this is multiple-run incremental writing (in contrast to runtime
+ # incremental writing, which writes ebuilds as soon as they're
+ # ready) FIXME: split incremental <-> runtime_incremental
self.scan()
- self._init_overlay ( reimport_eclass=True )
# --- end of __init__ (...) ---
@@ -73,8 +73,6 @@ class Overlay ( object ):
incremental=self.incremental
)
self._categories [category] = newcat
- if self.incremental:
- util.dodir ( newcat.physical_location )
finally:
self._catlock.release()
@@ -226,50 +224,10 @@ class Overlay ( object ):
return cat.add ( package_info )
# --- end of add (...) ---
- def finalize_write_incremental ( self ):
- """Writes metadata + Manifest for all packages."""
- for cat in self._categories.values():
- cat.finalize_write_incremental()
- # --- end of finalize_incremental (...) ---
-
- def generate_manifest ( self, **manifest_kw ):
- """Generates Manifest files for all ebuilds in this overlay that exist
- physically/in filesystem.
- Manifest files are automatically created when calling write().
-
- arguments:
- * **manifest_kw -- see PackageDir.generate_manifest(...)
-
- returns: None (implicit)
- """
- for cat in self._categories.values():
- cat.generate_manifest ( **manifest_kw )
- # --- end of generate_manifest (...) ---
-
- def generate_metadata ( self, **metadata_kw ):
- """Tells the overlay's categories to create metadata.
- You don't have to call this before write()/show() unless you want to use
- special metadata options.
-
- arguments:
- * **metadata_kw -- keywords for package.PackageDir.generate_metadata(...)
-
- returns: None (implicit)
- """
- for cat in self._categories.values():
- cat.generate_metadata ( **metadata_kw )
- # --- end of generate_metadata (...) ---
-
def has_dir ( self, _dir ):
return os.path.isdir ( self.physical_location + os.sep + _dir )
# --- end of has_category (...) ---
- def keep_nth_latest ( self, *args, **kwargs ):
- """See package.py:PackageDir:keep_nth_latest."""
- for cat in self._categories.values():
- cat.keep_nth_latest ( *args, **kwargs )
- # --- end of keep_nth_latest (...) ---
-
def list_packages ( self, for_deprules=True ):
for cat in self._categories.values():
for package in cat.list_packages ( for_deprules=True ):
@@ -313,9 +271,9 @@ class Overlay ( object ):
cat.show ( **show_kw )
# --- end of show (...) ---
- def write ( self, **write_kw ):
+ def write ( self ):
"""Writes the overlay to its physical location (filesystem), including
- metadata and Manifest files.
+ metadata and Manifest files as well as cleanup actions.
arguments:
* **write_kw -- keywords for package.PackageDir.write(...)
@@ -327,35 +285,27 @@ class Overlay ( object ):
! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
when ebuild creation is done.
"""
- raise Exception ( "to be removed/replaced" )
- # writing profiles/ here, rewriting categories/ later
self._init_overlay ( reimport_eclass=True )
for cat in self._categories.values():
- if not cat.empty():
- util.dodir ( cat.physical_location )
- cat.write ( **write_kw )
+ cat.write (
+ overwrite_ebuilds=False,
+ keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
+ cautious=True
+ )
# --- end of write (...) ---
- def write_incremental ( self, **write_kw ):
- """Writes all ebuilds that have been modified since the last write call.
- Note that there are currently two modes of incremental writing:
- (a) per-PackageDir incremental writing triggered by the new_ebuild()
- event method and (b) "batched" incremental writing (this method) which
- writes all modified PackageDirs.
- """
- # FIXME merge with write(), making incremental writing the only option
- # FIXME finalize_write_incremental?
- if not self._incremental_write_lock.acquire():
- # another incremental write is running, drop this request
- return
+ def write_manifest ( self, **manifest_kw ):
+ """Generates Manifest files for all ebuilds in this overlay that exist
+ physically/in filesystem.
+ Manifest files are automatically created when calling write().
- try:
- util.dodir ( self.physical_location )
- cats = tuple ( self._categories.values() )
- for cat in cats:
- util.dodir ( cat.physical_location )
- cat.write_incremental ( **write_kw )
- finally:
- self._incremental_write_lock.release()
- # --- end of write_incremental (...) ---
+ arguments:
+ * **manifest_kw -- see PackageDir.generate_manifest(...)
+
+ returns: None (implicit)
+ """
+ # FIXME: it would be good to ensure that profiles/categories exist
+ for cat in self._categories.values():
+ cat.write_manifest ( **manifest_kw )
+ # --- end of write_manifest (...) ---
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2012-07-30 8:55 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2012-07-30 8:52 [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/manifest/, roverlay/overlay/ André Erdmann
2012-07-24 16:59 ` [gentoo-commits] proj/R_overlay:overlay_wip " André Erdmann
-- strict thread matches above, loose matches on Subject: below --
2012-07-20 16:36 André Erdmann
2012-07-30 8:52 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox