From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 9A32A13800E for ; Mon, 30 Jul 2012 08:55:24 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 8AC67E06D9; Mon, 30 Jul 2012 08:52:47 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 33069E06C5 for ; Mon, 30 Jul 2012 08:52:47 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 9B93E1B400F for ; Mon, 30 Jul 2012 08:52:45 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 5CA03E5457 for ; Mon, 30 Jul 2012 08:52:43 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1343144347.ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/manifest/, roverlay/overlay/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/overlay/category.py roverlay/overlay/creator.py roverlay/overlay/manifest/__init__.py roverlay/overlay/package.py roverlay/overlay/root.py X-VCS-Directories: roverlay/overlay/manifest/ roverlay/overlay/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147 X-VCS-Branch: master Date: Mon, 30 Jul 2012 08:52:43 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: bd975102-4dfa-415a-b15d-f5f33701bf22 X-Archives-Hash: 78eb8f8b537a684ff90538c3c41d3302 commit: ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147 Author: André Erdmann mailerd de> AuthorDate: Tue Jul 24 15:39:07 2012 +0000 Commit: André Erdmann mailerd de> CommitDate: Tue Jul 24 15:39:07 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ee870b2a incremental overlay writing: merge write() funcs * one write function for overlay/{root,category,package}.py that replaces write_incremental, write, finalize_write_incremental * also fixed some issues (proper PackageDir cleanup, make threaded writing controllable) geändert: roverlay/overlay/category.py geändert: roverlay/overlay/creator.py geändert: roverlay/overlay/manifest/__init__.py geändert: roverlay/overlay/package.py geändert: roverlay/overlay/root.py --- roverlay/overlay/category.py | 141 +++++++---------- roverlay/overlay/creator.py | 29 +--- roverlay/overlay/manifest/__init__.py | 4 +- roverlay/overlay/package.py | 278 +++++++++++++++++++++------------ roverlay/overlay/root.py | 96 +++--------- 5 files changed, 266 insertions(+), 282 deletions(-) diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py index 33149c4..fab6373 100644 --- a/roverlay/overlay/category.py +++ b/roverlay/overlay/category.py @@ -12,8 +12,6 @@ except ImportError: from roverlay.overlay.package import PackageDir -import roverlay.util - class Category ( object ): WRITE_JOBCOUNT = 3 @@ -49,8 +47,6 @@ class Category ( object ): incremental = self.incremental ) self._subdirs [pkg_name] = newpkg - if self.incremental: - roverlay.util.dodir ( newpkg.physical_location ) finally: self._lock.release() @@ -76,26 +72,6 @@ class Category ( object ): not False in ( d.empty() for d in self._subdirs.values() ) # --- end of empty (...) --- - def finalize_write_incremental ( self ): - for subdir in self._subdirs.values(): - if subdir.modified: - subdir.write_incremental() - subdir.finalize_write_incremental() - # --- end of finalize_write_incremental (...) --- - - def generate_metadata ( self, **metadata_kw ): - """Generates metadata for all packages in this category. - Metadata are automatically generated when calling write(). - - arguments: - * **metadata_kw -- see PackageDir.generate_metadata(...) - - returns: None (implicit) - """ - for package in self._subdirs.values(): - package.generate_metadata ( **metadata_kw ) - # --- end of generate_metadata (...) --- - def has ( self, subdir ): return subdir in self._subdirs # --- end of has (...) --- @@ -104,13 +80,6 @@ class Category ( object ): return os.path.isdir ( self.physical_location + os.sep + _dir ) # --- end of has_category (...) --- - def keep_nth_latest ( self, *args, **kwargs ): - """See package.py:PackageDir:keep_nth_latest.""" - for subdir in self._subdirs.values(): - subdir.keep_nth_latest ( *args, **kwargs ) - subdir.fs_cleanup() - # --- end of keep_nth_latest (...) --- - def list_packages ( self, for_deprules=False ): """Lists all packages in this category. Yields / or a dict (see for_deprules below). @@ -130,6 +99,14 @@ class Category ( object ): yield self.name + os.sep + name # --- end of list_packages (...) --- + def remove_empty ( self ): + """This removes all empty PackageDirs.""" + with self._lock: + for key in tuple ( self._subdirs.keys() ): + if self._subdirs [key].check_empty(): + del self._subdirs [key] + # --- end of remove_empty (...) --- + def scan ( self, **kw ): """Scans this category for existing ebuilds.""" for subdir in os.listdir ( self.physical_location ): @@ -146,7 +123,7 @@ class Category ( object ): package.show ( **show_kw ) # --- end of show (...) --- - def write ( self, **write_kw ): + def write ( self, overwrite_ebuilds, keep_n_ebuilds, cautious ): """Writes this category to its filesystem location. returns: None (implicit) @@ -156,75 +133,75 @@ class Category ( object ): arguments: * q -- queue - * write_kw -- + * write_kw -- keywords for write(...) """ try: while not q.empty(): - pkg = q.get_nowait() - pkg.write ( write_manifest=False, **write_kw ) - + try: + pkg = q.get_nowait() + # remove manifest writing from threaded writing since it's + # single-threaded + pkg.write ( write_manifest=False, **write_kw ) + #except ( Exception, KeyboardInterrupt ) as e: + except Exception as e: + # FIXME: reintroduce RERAISE + self.logger.exception ( e ) except queue.Empty: pass - except ( Exception, KeyboardInterrupt ) as e: - self.RERAISE_EXCEPTION = e # --- end of run_write_queue (...) --- + if len ( self._subdirs ) == 0: return + + # determine write keyword args + write_kwargs = dict ( + overwrite_ebuilds = overwrite_ebuilds, + keep_n_ebuilds = keep_n_ebuilds, + cautious = cautious, + ) + + # start writing: + max_jobs = self.__class__.WRITE_JOBCOUNT - # todo len.. > 3: what's an reasonable number of min package dirs to - # start threaded writing? - if max_jobs > 1 and len ( self._subdirs ) > 3: + # FIXME/TODO: what's an reasonable number of min package dirs to + # start threaded writing? + # Ignoring it for now (and expecting enough pkg dirs) + if max_jobs > 1: - # writing 1..self.__class__.WRITE_JOBCOUNT package dirs at once + # writing <=max_jobs package dirs at once - modified_packages = tuple ( - p for p in self._subdirs.values() if p.modified - ) - if len ( modified_packages ) > 0: - write_queue = queue.Queue() - for package in modified_packages: - roverlay.util.dodir ( package.physical_location ) - write_queue.put_nowait ( package ) + # don't create more workers than write jobs available + max_jobs = min ( max_jobs, len ( self._subdirs ) ) - workers = ( - threading.Thread ( - target=run_write_queue, - args=( write_queue, write_kw ) - ) for n in range ( max_jobs ) - ) + write_queue = queue.Queue() + for package in self._subdirs.values(): + write_queue.put_nowait ( package ) - for w in workers: w.start() - for w in workers: w.join() + workers = frozenset ( + threading.Thread ( + target=run_write_queue, + args=( write_queue, write_kwargs ) + ) for n in range ( max_jobs ) + ) + + for w in workers: w.start() + for w in workers: w.join() - if hasattr ( self, 'RERAISE_EXCEPTION' ): - raise self.RERAISE_EXCEPTION + self.remove_empty() - # write manifest files - for package in modified_packages: - package.write_manifest() + # write manifest files + # fixme: debug print + #self.logger.info ( "Writing Manifest files for {}".format ( name ) ) + print ( "Writing Manifest files ..." ) + for package in self._subdirs.values(): + package.write_manifest ( ignore_empty=True ) else: for package in self._subdirs.values(): - if package.modified: - roverlay.util.dodir ( package.physical_location ) - package.write ( write_manifest=True, **write_kw ) - # --- end of write (...) --- + package.write ( **write_kwargs ) - def write_incremental ( self, **write_kw ): - """Writes this category incrementally.""" - try: - with self._lock: - # new package dirs could be added during overlay writing, - # so collect the list of package dirs before iterating over it - subdirs = tuple ( self._subdirs.values() ) - - for subdir in subdirs: - if subdir.modified: - roverlay.util.dodir ( subdir.physical_location ) - subdir.write_incremental ( **write_kw ) - except Exception as e: - self.logger.exception ( e ) - # --- end of write_incremental (...) --- + self.remove_empty() + # --- end of write (...) --- def write_manifest ( self, **manifest_kw ): """Generates Manifest files for all packages in this category. diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py index 9de063c..d12e0a5 100644 --- a/roverlay/overlay/creator.py +++ b/roverlay/overlay/creator.py @@ -214,32 +214,15 @@ class OverlayCreator ( object ): self.package_added.inc() # --- end of add_package (...) --- - def add_package_file ( self, package_file ): - """Adds a single R package.""" - raise Exception ( "to be removed" ) - self._pkg_queue.put ( PackageInfo ( filepath=package_file ) ) - self.package_added.inc() - # --- end of add_package_file (...) --- - - def add_package_files ( self, *package_files ): - """Adds multiple R packages.""" - raise Exception ( "to be removed" ) - for p in package_files: self.add_package_file ( p ) - self.package_added.inc() - # --- end of add_package_files (...) --- - def write_overlay ( self ): """Writes the overlay. arguments: """ if self.can_write_overlay: - if self.write_incremental: - self.overlay.finalize_write_incremental() - else: - start = time.time() - self.overlay.write() - self._timestamp ( "overlay written", start ) + start = time.time() + self.overlay.write() + self._timestamp ( "overlay written", start ) else: self.logger.warning ( "Not allowed to write overlay!" ) # --- end of write_overlay (...) --- @@ -304,7 +287,6 @@ class OverlayCreator ( object ): self._close_workers() close_resolver() - self.overlay.keep_nth_latest ( n=1 ) self.closed = True # --- end of close (...) --- @@ -429,8 +411,9 @@ class OverlayCreator ( object ): if self.NUMTHREADS > 0: start = time.time() self.logger.warning ( - "Running in concurrent mode with %i threads." % self.NUMTHREADS - ) + "Running in concurrent mode with {num} threads.".format ( + num=self.NUMTHREADS + ) ) self._workers = frozenset ( self._get_worker ( start_now=True ) \ for n in range ( self.NUMTHREADS ) diff --git a/roverlay/overlay/manifest/__init__.py b/roverlay/overlay/manifest/__init__.py index 9a49c7f..d9f9c0a 100644 --- a/roverlay/overlay/manifest/__init__.py +++ b/roverlay/overlay/manifest/__init__.py @@ -11,8 +11,6 @@ _manifest_creation = helpers.ExternalManifestCreation() # for one directory/overlay _manifest_lock = threading.Lock() - - def create_manifest ( package_info_list, nofail=False ): """Creates a Manifest for package_info, using the <> implementation available. @@ -35,4 +33,6 @@ def create_manifest ( package_info_list, nofail=False ): raise finally: _manifest_lock.release() + + return ret # --- end of create_manifest (...) --- diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py index a2e4d9b..d2662bd 100644 --- a/roverlay/overlay/package.py +++ b/roverlay/overlay/package.py @@ -120,6 +120,21 @@ class PackageDir ( object ): return False # --- end of add (...) --- + def check_empty ( self ): + """Similar to empty(), + but also removes the directory of this PackageDir. + """ + if len ( self._packages ) == 0: + if os.path.isdir ( self.physical_location ): + try: + os.rmdir ( self.physical_location ) + except Exception as e: + self.logger.exception ( e ) + return True + else: + return False + # --- end of check_empty (...) --- + def empty ( self ): """Returns True if no ebuilds stored, else False. Note that "not empty" doesn't mean "has ebuilds to write" or "has @@ -129,25 +144,6 @@ class PackageDir ( object ): return len ( self._packages ) == 0 # --- end of empty (...) --- - def finalize_write_incremental ( self ): - """Method that finalizes incremental writing, i.e. write outstanding - ebuilds and write metadata.xml, Manifest. - """ - with self._lock: - if self.has_ebuilds(): - if self.modified: - self.write_ebuilds ( overwrite=False ) - if self._need_metadata: - self.write_metadata() - if self._need_manifest: - self.write_manifest() - else: - self.logger.critical ( - "<>: please clean up this dir: {}.".format ( - self.physical_location - ) ) - # --- end of finalize_write_incremental (...) --- - def fs_cleanup ( self ): """Cleans up the filesystem location of this package dir. To be called after keep_nth_latest, calls finalize_write_incremental(). @@ -158,11 +154,9 @@ class PackageDir ( object ): # --- end of rmtree_error (...) --- with self._lock: - if self.has_ebuilds(): - # !!! FIXME this doesn't work if no ebuilds written, but - # old ones removed -> DISTDIR unknown during Manifest creation - self.finalize_write_incremental() - elif os.path.isdir ( self.physical_location ): + if os.path.isdir ( self.physical_location ) \ + and not self.has_ebuilds() \ + : # destroy self.physical_location shutil.rmtree ( self.physical_location, onerror=rmtree_error ) # --- end of fs_cleanup (...) --- @@ -193,53 +187,58 @@ class PackageDir ( object ): arguments: * n -- # of packages/ebuilds to keep * cautious -- if True: be extra careful, verify that ebuilds exist + as file; note that this will ignore all + ebuilds that haven't been written to the file- + system yet (which implies an extra overhead, + you'll have to write all ebuilds first) """ - - # create the list of packages to iterate over, - # * package has to have an ebuild_file - # * sort them by version in reverse order (latest package gets index 0) + def is_ebuild_cautious ( p_tuple ): + # package has to have an ebuild_file that exists + efile = p_tuple [1] ['ebuild_file' ] + if efile is not None: + return os.path.isfile ( efile ) + else: + return False + # --- end of is_ebuild_cautious (...) --- + + def is_ebuild ( p_tuple ): + # package has to have an ebuild_file or an ebuild entry + return ( + p_tuple [1] ['ebuild_file'] or p_tuple [1] ['ebuild'] + ) is not None + # --- end of is_ebuild (...) --- + + # create the list of packages to iterate over (cautious/non-cautious), + # sort them by version in reverse order packages = reversed ( sorted ( filter ( - lambda p : p [1] ['ebuild_file'] is not None, - self._packages.items() + function=is_ebuild if not cautious else is_ebuild_cautious, + iterable=self._packages.items() ), key=lambda p : p [1] ['version'] ) ) + if n < 1: + raise Exception ( "Must keep more than zero ebuilds." ) + kept = 0 ecount = 0 - if not cautious: - # could use a slice here, too - for pvr, pkg in packages: - ecount += 1 - if kept < n: - printself.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) ) - kept += 1 - else: - self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) ) - self.purge_package ( pvr ) - else: - for pvr, pkg in packages: - ecount += 1 - if os.path.isfile ( pkg ['ebuild_file'] ): - if kept < n: - self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) ) - kept += 1 - else: - self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) ) - self.purge_package ( pvr ) - else: - self.logger.error ( - "{efile} is assumed to exist as file but doesn't!".format ( - efile=pkg ['ebuild_file'] - ) ) - # FIXME/IGNORE: this doesn't count inexistent files as kept + for pvr, pkg in packages: + ecount += 1 + if kept < n: + self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) ) + kept += 1 + else: + self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) ) + self.purge_package ( pvr ) + self.logger.debug ( "Kept {kept}/{total} ebuilds.".format ( kept=kept, total=ecount ) ) # FIXME: Manifest is now invalid and dir could be "empty" (no ebuilds) + # FIXME: force metadata regeneration # --- end of keep_nth_latest (...) --- def list_versions ( self ): @@ -252,7 +251,8 @@ class PackageDir ( object ): self._need_metadata = True self.modified = True if self.runtime_incremental: - return self.write_incremental() + with self._lock: + return self.write_ebuilds ( overwrite=False ) else: return True # --- end of new_ebuild (...) --- @@ -325,48 +325,104 @@ class PackageDir ( object ): arguments: * stream -- stream to use, defaults to sys.stderr - returns: None (implicit) + returns: True raises: - * IOError + * passes all exceptions (IOError, ..) """ - return self.write ( shared_fh=stream ) + self.write_ebuilds ( overwrite=True, shared_fh=stream ) + self.write_metadata ( shared_fh=stream ) + return True # --- end of show (...) --- + def virtual_cleanup ( self ): + """Removes all PackageInfos from this structure that don't have an + 'ebuild_file' entry. + """ + with self._lock: + # keyset may change during this method + for pvr in tuple ( self._packages.keys() ): + if self._packages [pvr] ['ebuild_file'] is None: + del self._packages [pvr] + # -- lock + # --- end of virtual_cleanup (...) --- + def write ( self, - shared_fh=None, overwrite_ebuilds=True, - write_ebuilds=True, write_manifest=True, write_metadata=True + overwrite_ebuilds=False, + write_ebuilds=True, write_manifest=True, write_metadata=True, + cleanup=True, keep_n_ebuilds=None, cautious=True ): """Writes this directory to its (existent!) filesystem location. arguments: - * shared_fh -- if set and not None: write everyting into * write_ebuilds -- if set and False: don't write ebuilds * write_manifest -- if set and False: don't write the Manifest file * write_metadata -- if set and False: don't write the metadata file - * overwrite_ebuilds -- if set and False: don't overwrite ebuilds - - returns: None (implicit) + * overwrite_ebuilds -- whether to overwrite ebuilds, + None means autodetect, enable overwriting + if not modified since last write + Defaults to False + * cleanup -- clean up after writing + Defaults to True + * keep_n_ebuilds -- # of ebuilds to keep (remove all others), + Defaults to None (disable) and implies cleanup + * cautious -- be cautious when keeping the nth latest ebuilds, + this has some overhead + Defaults to True + + returns: success (True/False) raises: - * IOError (?) + * passes IOError """ - with self._lock: - # mkdir not required here, overlay.Category does this + # NOTE, replaces: + # * old write: overwrite_ebuilds=True + # * finalize_write_incremental : no extra args + # * write_incremental : write_manifest=False, write_metadata=False, + # cleanup=False (or use write_ebuilds) + # BREAKS: show(), which has its own method/function now - # write ebuilds - if write_ebuilds: - self.write_ebuilds ( - overwrite=overwrite_ebuilds, shared_fh=shared_fh - ) + cleanup = cleanup or ( keep_n_ebuilds is not None ) - # write metadata - if write_metadata: - self.write_metadata ( shared_fh=shared_fh ) + success = True + with self._lock: + if self.has_ebuilds(): + # not cautious: remove ebuilds before writing them + if not cautious and keep_n_ebuilds is not None: + self.keep_nth_latest ( n=keep_n_ebuilds, cautious=False ) + + # write ebuilds + if self.modified and write_ebuilds: + success = self.write_ebuilds ( + # None ~ not modified + overwrite = overwrite_ebuilds \ + if overwrite_ebuilds is not None \ + else not self.modified + ) - # write manifest (only if shared_fh is None) - if write_manifest and shared_fh is None: - self.write_manifest() + # cautious: remove ebuilds after writing them + if cautious and keep_n_ebuilds is not None: + self.keep_nth_latest ( n=keep_n_ebuilds, cautious=True ) + + # write metadata + if self._need_metadata and write_metadata: + # don't mess around with short-circuit bool evaluation + if not self.write_metadata(): + success = False + + # write manifest (only if shared_fh is None) + if self._need_manifest and write_manifest: + if not self.write_manifest(): + success = False + # -- has_ebuilds? + + if cleanup: + self.virtual_cleanup() + self.fs_cleanup() + + # FIXME / TODO call fs_cleanup + # -- lock + return success # --- end of write (...) --- def write_ebuilds ( self, overwrite, shared_fh=None ): @@ -390,7 +446,6 @@ class PackageDir ( object ): """ _success = False try: - util.dodir ( self.physical_location ) fh = open ( efile, 'w' ) if shared_fh is None else shared_fh if ebuild_header is not None: fh.write ( str ( ebuild_header ) ) @@ -426,7 +481,14 @@ class PackageDir ( object ): all_ebuilds_written = True + # don't call dodir if shared_fh is set + hasdir = bool ( shared_fh is not None ) + for efile, p_info in ebuilds_to_write(): + if not hasdir: + util.dodir ( self.physical_location, mkdir_p=True ) + hasdir = True + if write_ebuild ( efile, p_info ['ebuild'] ): self._need_manifest = True @@ -452,17 +514,12 @@ class PackageDir ( object ): return all_ebuilds_written # --- end of write_ebuilds (...) --- - def write_incremental ( self ): - with self._lock: - return self.write_ebuilds ( overwrite=False ) - # --- end of write_incremental (...) --- - - def write_manifest ( self ): + def write_manifest ( self, ignore_empty=False ): """Generates and writes the Manifest file for this package. expects: called after writing metadata/ebuilds - returns: None (implicit) + returns: success (True/False) raises: * Exception if no ebuild exists @@ -471,47 +528,64 @@ class PackageDir ( object ): # it should be sufficient to call create_manifest for one ebuild, # choosing the latest one that exists in self.physical_location and # has enough data (DISTDIR, EBUILD_FILE) for this task. + # Additionally, all DISTDIRs (multiple repos, sub directories) have + # to be collected and passed to Manifest creation. + # => collect suitable PackageInfo objects from self._packages # - # metadata.xml's full path cannot be used for manifest creation here - # 'cause DISTDIR would be unknown - # - - # collect suitable PackageInfo instances pkgs_for_manifest = tuple ( p for p in self._packages.values() \ if p.has ( 'distdir', 'ebuild_file' ) ) if pkgs_for_manifest: - manifest.create_manifest ( pkgs_for_manifest, nofail=False ) - self._need_manifest = False + if manifest.create_manifest ( pkgs_for_manifest, nofail=False ): + self._need_manifest = False + return True + elif ignore_empty: + return True else: + # FIXME: debug statements + # FIXME: remove excpetion, maybe delete Manifest in this case,.. + for pvr, p in self._packages.items(): + print ( "{} {} ebuild={} efile={} has={}".format ( + pvr, p, p.has ('ebuild'), p ['ebuild_file'], self.has_ebuilds() + ) ) + raise Exception ( - "No ebuild written so far! I really don't know what do to!" - ) + 'In {mydir}: No ebuild written so far! ' + 'I really don\'t know what do to!'.format ( + mydir=self.physical_location + ) ) + + return False # --- end of write_manifest (...) --- def write_metadata ( self, shared_fh=None ): - """Writes metadata for this package.""" + """Writes metadata for this package. + + returns: success (True/False) + """ + success = False try: self.generate_metadata ( skip_if_existent=True ) if shared_fh is None: + util.dodir ( self.physical_location, mkdir_p=True ) if self._metadata.write(): self._need_metadata = False self._need_manifest = True - return True + success = True else: self.logger.error ( "Failed to write metadata file {}.".format ( self._metadata.filepath ) ) - return False else: self._metadata.show ( shared_fh ) - return True + success = True except Exception as e: self.logger.exception ( e ) - return False + + return success # --- end of write_metadata (...) --- diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py index 363bdb5..219fd70 100644 --- a/roverlay/overlay/root.py +++ b/roverlay/overlay/root.py @@ -48,10 +48,10 @@ class Overlay ( object ): self.incremental = incremental if self.incremental: - - self._incremental_write_lock = threading.Lock() + # this is multiple-run incremental writing (in contrast to runtime + # incremental writing, which writes ebuilds as soon as they're + # ready) FIXME: split incremental <-> runtime_incremental self.scan() - self._init_overlay ( reimport_eclass=True ) # --- end of __init__ (...) --- @@ -73,8 +73,6 @@ class Overlay ( object ): incremental=self.incremental ) self._categories [category] = newcat - if self.incremental: - util.dodir ( newcat.physical_location ) finally: self._catlock.release() @@ -226,50 +224,10 @@ class Overlay ( object ): return cat.add ( package_info ) # --- end of add (...) --- - def finalize_write_incremental ( self ): - """Writes metadata + Manifest for all packages.""" - for cat in self._categories.values(): - cat.finalize_write_incremental() - # --- end of finalize_incremental (...) --- - - def generate_manifest ( self, **manifest_kw ): - """Generates Manifest files for all ebuilds in this overlay that exist - physically/in filesystem. - Manifest files are automatically created when calling write(). - - arguments: - * **manifest_kw -- see PackageDir.generate_manifest(...) - - returns: None (implicit) - """ - for cat in self._categories.values(): - cat.generate_manifest ( **manifest_kw ) - # --- end of generate_manifest (...) --- - - def generate_metadata ( self, **metadata_kw ): - """Tells the overlay's categories to create metadata. - You don't have to call this before write()/show() unless you want to use - special metadata options. - - arguments: - * **metadata_kw -- keywords for package.PackageDir.generate_metadata(...) - - returns: None (implicit) - """ - for cat in self._categories.values(): - cat.generate_metadata ( **metadata_kw ) - # --- end of generate_metadata (...) --- - def has_dir ( self, _dir ): return os.path.isdir ( self.physical_location + os.sep + _dir ) # --- end of has_category (...) --- - def keep_nth_latest ( self, *args, **kwargs ): - """See package.py:PackageDir:keep_nth_latest.""" - for cat in self._categories.values(): - cat.keep_nth_latest ( *args, **kwargs ) - # --- end of keep_nth_latest (...) --- - def list_packages ( self, for_deprules=True ): for cat in self._categories.values(): for package in cat.list_packages ( for_deprules=True ): @@ -313,9 +271,9 @@ class Overlay ( object ): cat.show ( **show_kw ) # --- end of show (...) --- - def write ( self, **write_kw ): + def write ( self ): """Writes the overlay to its physical location (filesystem), including - metadata and Manifest files. + metadata and Manifest files as well as cleanup actions. arguments: * **write_kw -- keywords for package.PackageDir.write(...) @@ -327,35 +285,27 @@ class Overlay ( object ): ! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called when ebuild creation is done. """ - raise Exception ( "to be removed/replaced" ) - # writing profiles/ here, rewriting categories/ later self._init_overlay ( reimport_eclass=True ) for cat in self._categories.values(): - if not cat.empty(): - util.dodir ( cat.physical_location ) - cat.write ( **write_kw ) + cat.write ( + overwrite_ebuilds=False, + keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ), + cautious=True + ) # --- end of write (...) --- - def write_incremental ( self, **write_kw ): - """Writes all ebuilds that have been modified since the last write call. - Note that there are currently two modes of incremental writing: - (a) per-PackageDir incremental writing triggered by the new_ebuild() - event method and (b) "batched" incremental writing (this method) which - writes all modified PackageDirs. - """ - # FIXME merge with write(), making incremental writing the only option - # FIXME finalize_write_incremental? - if not self._incremental_write_lock.acquire(): - # another incremental write is running, drop this request - return + def write_manifest ( self, **manifest_kw ): + """Generates Manifest files for all ebuilds in this overlay that exist + physically/in filesystem. + Manifest files are automatically created when calling write(). - try: - util.dodir ( self.physical_location ) - cats = tuple ( self._categories.values() ) - for cat in cats: - util.dodir ( cat.physical_location ) - cat.write_incremental ( **write_kw ) - finally: - self._incremental_write_lock.release() - # --- end of write_incremental (...) --- + arguments: + * **manifest_kw -- see PackageDir.generate_manifest(...) + + returns: None (implicit) + """ + # FIXME: it would be good to ensure that profiles/categories exist + for cat in self._categories.values(): + cat.write_manifest ( **manifest_kw ) + # --- end of write_manifest (...) --- From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id F1A1313800E for ; Tue, 24 Jul 2012 17:00:09 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 01D42E06C0; Tue, 24 Jul 2012 16:59:52 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id A56F7E06C0 for ; Tue, 24 Jul 2012 16:59:52 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id D74881B4527 for ; Tue, 24 Jul 2012 16:59:51 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 8D9A2E5436 for ; Tue, 24 Jul 2012 16:59:49 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1343144347.ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/manifest/, roverlay/overlay/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/overlay/category.py roverlay/overlay/creator.py roverlay/overlay/manifest/__init__.py roverlay/overlay/package.py roverlay/overlay/root.py X-VCS-Directories: roverlay/overlay/manifest/ roverlay/overlay/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147 X-VCS-Branch: overlay_wip Date: Tue, 24 Jul 2012 16:59:49 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 73def348-ed5f-44b2-9e50-4627d00ee432 X-Archives-Hash: 5e0dad10f09034a5fa52059d264e14a0 Message-ID: <20120724165949.cSSbqGOCWM5tTUYo0T5XgKnt1cgYQtMcon2RS4N-X9k@z> commit: ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147 Author: André Erdmann mailerd de> AuthorDate: Tue Jul 24 15:39:07 2012 +0000 Commit: André Erdmann mailerd de> CommitDate: Tue Jul 24 15:39:07 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ee870b2a incremental overlay writing: merge write() funcs * one write function for overlay/{root,category,package}.py that replaces write_incremental, write, finalize_write_incremental * also fixed some issues (proper PackageDir cleanup, make threaded writing controllable) geändert: roverlay/overlay/category.py geändert: roverlay/overlay/creator.py geändert: roverlay/overlay/manifest/__init__.py geändert: roverlay/overlay/package.py geändert: roverlay/overlay/root.py --- roverlay/overlay/category.py | 141 +++++++---------- roverlay/overlay/creator.py | 29 +--- roverlay/overlay/manifest/__init__.py | 4 +- roverlay/overlay/package.py | 278 +++++++++++++++++++++------------ roverlay/overlay/root.py | 96 +++--------- 5 files changed, 266 insertions(+), 282 deletions(-) diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py index 33149c4..fab6373 100644 --- a/roverlay/overlay/category.py +++ b/roverlay/overlay/category.py @@ -12,8 +12,6 @@ except ImportError: from roverlay.overlay.package import PackageDir -import roverlay.util - class Category ( object ): WRITE_JOBCOUNT = 3 @@ -49,8 +47,6 @@ class Category ( object ): incremental = self.incremental ) self._subdirs [pkg_name] = newpkg - if self.incremental: - roverlay.util.dodir ( newpkg.physical_location ) finally: self._lock.release() @@ -76,26 +72,6 @@ class Category ( object ): not False in ( d.empty() for d in self._subdirs.values() ) # --- end of empty (...) --- - def finalize_write_incremental ( self ): - for subdir in self._subdirs.values(): - if subdir.modified: - subdir.write_incremental() - subdir.finalize_write_incremental() - # --- end of finalize_write_incremental (...) --- - - def generate_metadata ( self, **metadata_kw ): - """Generates metadata for all packages in this category. - Metadata are automatically generated when calling write(). - - arguments: - * **metadata_kw -- see PackageDir.generate_metadata(...) - - returns: None (implicit) - """ - for package in self._subdirs.values(): - package.generate_metadata ( **metadata_kw ) - # --- end of generate_metadata (...) --- - def has ( self, subdir ): return subdir in self._subdirs # --- end of has (...) --- @@ -104,13 +80,6 @@ class Category ( object ): return os.path.isdir ( self.physical_location + os.sep + _dir ) # --- end of has_category (...) --- - def keep_nth_latest ( self, *args, **kwargs ): - """See package.py:PackageDir:keep_nth_latest.""" - for subdir in self._subdirs.values(): - subdir.keep_nth_latest ( *args, **kwargs ) - subdir.fs_cleanup() - # --- end of keep_nth_latest (...) --- - def list_packages ( self, for_deprules=False ): """Lists all packages in this category. Yields / or a dict (see for_deprules below). @@ -130,6 +99,14 @@ class Category ( object ): yield self.name + os.sep + name # --- end of list_packages (...) --- + def remove_empty ( self ): + """This removes all empty PackageDirs.""" + with self._lock: + for key in tuple ( self._subdirs.keys() ): + if self._subdirs [key].check_empty(): + del self._subdirs [key] + # --- end of remove_empty (...) --- + def scan ( self, **kw ): """Scans this category for existing ebuilds.""" for subdir in os.listdir ( self.physical_location ): @@ -146,7 +123,7 @@ class Category ( object ): package.show ( **show_kw ) # --- end of show (...) --- - def write ( self, **write_kw ): + def write ( self, overwrite_ebuilds, keep_n_ebuilds, cautious ): """Writes this category to its filesystem location. returns: None (implicit) @@ -156,75 +133,75 @@ class Category ( object ): arguments: * q -- queue - * write_kw -- + * write_kw -- keywords for write(...) """ try: while not q.empty(): - pkg = q.get_nowait() - pkg.write ( write_manifest=False, **write_kw ) - + try: + pkg = q.get_nowait() + # remove manifest writing from threaded writing since it's + # single-threaded + pkg.write ( write_manifest=False, **write_kw ) + #except ( Exception, KeyboardInterrupt ) as e: + except Exception as e: + # FIXME: reintroduce RERAISE + self.logger.exception ( e ) except queue.Empty: pass - except ( Exception, KeyboardInterrupt ) as e: - self.RERAISE_EXCEPTION = e # --- end of run_write_queue (...) --- + if len ( self._subdirs ) == 0: return + + # determine write keyword args + write_kwargs = dict ( + overwrite_ebuilds = overwrite_ebuilds, + keep_n_ebuilds = keep_n_ebuilds, + cautious = cautious, + ) + + # start writing: + max_jobs = self.__class__.WRITE_JOBCOUNT - # todo len.. > 3: what's an reasonable number of min package dirs to - # start threaded writing? - if max_jobs > 1 and len ( self._subdirs ) > 3: + # FIXME/TODO: what's an reasonable number of min package dirs to + # start threaded writing? + # Ignoring it for now (and expecting enough pkg dirs) + if max_jobs > 1: - # writing 1..self.__class__.WRITE_JOBCOUNT package dirs at once + # writing <=max_jobs package dirs at once - modified_packages = tuple ( - p for p in self._subdirs.values() if p.modified - ) - if len ( modified_packages ) > 0: - write_queue = queue.Queue() - for package in modified_packages: - roverlay.util.dodir ( package.physical_location ) - write_queue.put_nowait ( package ) + # don't create more workers than write jobs available + max_jobs = min ( max_jobs, len ( self._subdirs ) ) - workers = ( - threading.Thread ( - target=run_write_queue, - args=( write_queue, write_kw ) - ) for n in range ( max_jobs ) - ) + write_queue = queue.Queue() + for package in self._subdirs.values(): + write_queue.put_nowait ( package ) - for w in workers: w.start() - for w in workers: w.join() + workers = frozenset ( + threading.Thread ( + target=run_write_queue, + args=( write_queue, write_kwargs ) + ) for n in range ( max_jobs ) + ) + + for w in workers: w.start() + for w in workers: w.join() - if hasattr ( self, 'RERAISE_EXCEPTION' ): - raise self.RERAISE_EXCEPTION + self.remove_empty() - # write manifest files - for package in modified_packages: - package.write_manifest() + # write manifest files + # fixme: debug print + #self.logger.info ( "Writing Manifest files for {}".format ( name ) ) + print ( "Writing Manifest files ..." ) + for package in self._subdirs.values(): + package.write_manifest ( ignore_empty=True ) else: for package in self._subdirs.values(): - if package.modified: - roverlay.util.dodir ( package.physical_location ) - package.write ( write_manifest=True, **write_kw ) - # --- end of write (...) --- + package.write ( **write_kwargs ) - def write_incremental ( self, **write_kw ): - """Writes this category incrementally.""" - try: - with self._lock: - # new package dirs could be added during overlay writing, - # so collect the list of package dirs before iterating over it - subdirs = tuple ( self._subdirs.values() ) - - for subdir in subdirs: - if subdir.modified: - roverlay.util.dodir ( subdir.physical_location ) - subdir.write_incremental ( **write_kw ) - except Exception as e: - self.logger.exception ( e ) - # --- end of write_incremental (...) --- + self.remove_empty() + # --- end of write (...) --- def write_manifest ( self, **manifest_kw ): """Generates Manifest files for all packages in this category. diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py index 9de063c..d12e0a5 100644 --- a/roverlay/overlay/creator.py +++ b/roverlay/overlay/creator.py @@ -214,32 +214,15 @@ class OverlayCreator ( object ): self.package_added.inc() # --- end of add_package (...) --- - def add_package_file ( self, package_file ): - """Adds a single R package.""" - raise Exception ( "to be removed" ) - self._pkg_queue.put ( PackageInfo ( filepath=package_file ) ) - self.package_added.inc() - # --- end of add_package_file (...) --- - - def add_package_files ( self, *package_files ): - """Adds multiple R packages.""" - raise Exception ( "to be removed" ) - for p in package_files: self.add_package_file ( p ) - self.package_added.inc() - # --- end of add_package_files (...) --- - def write_overlay ( self ): """Writes the overlay. arguments: """ if self.can_write_overlay: - if self.write_incremental: - self.overlay.finalize_write_incremental() - else: - start = time.time() - self.overlay.write() - self._timestamp ( "overlay written", start ) + start = time.time() + self.overlay.write() + self._timestamp ( "overlay written", start ) else: self.logger.warning ( "Not allowed to write overlay!" ) # --- end of write_overlay (...) --- @@ -304,7 +287,6 @@ class OverlayCreator ( object ): self._close_workers() close_resolver() - self.overlay.keep_nth_latest ( n=1 ) self.closed = True # --- end of close (...) --- @@ -429,8 +411,9 @@ class OverlayCreator ( object ): if self.NUMTHREADS > 0: start = time.time() self.logger.warning ( - "Running in concurrent mode with %i threads." % self.NUMTHREADS - ) + "Running in concurrent mode with {num} threads.".format ( + num=self.NUMTHREADS + ) ) self._workers = frozenset ( self._get_worker ( start_now=True ) \ for n in range ( self.NUMTHREADS ) diff --git a/roverlay/overlay/manifest/__init__.py b/roverlay/overlay/manifest/__init__.py index 9a49c7f..d9f9c0a 100644 --- a/roverlay/overlay/manifest/__init__.py +++ b/roverlay/overlay/manifest/__init__.py @@ -11,8 +11,6 @@ _manifest_creation = helpers.ExternalManifestCreation() # for one directory/overlay _manifest_lock = threading.Lock() - - def create_manifest ( package_info_list, nofail=False ): """Creates a Manifest for package_info, using the <> implementation available. @@ -35,4 +33,6 @@ def create_manifest ( package_info_list, nofail=False ): raise finally: _manifest_lock.release() + + return ret # --- end of create_manifest (...) --- diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py index a2e4d9b..d2662bd 100644 --- a/roverlay/overlay/package.py +++ b/roverlay/overlay/package.py @@ -120,6 +120,21 @@ class PackageDir ( object ): return False # --- end of add (...) --- + def check_empty ( self ): + """Similar to empty(), + but also removes the directory of this PackageDir. + """ + if len ( self._packages ) == 0: + if os.path.isdir ( self.physical_location ): + try: + os.rmdir ( self.physical_location ) + except Exception as e: + self.logger.exception ( e ) + return True + else: + return False + # --- end of check_empty (...) --- + def empty ( self ): """Returns True if no ebuilds stored, else False. Note that "not empty" doesn't mean "has ebuilds to write" or "has @@ -129,25 +144,6 @@ class PackageDir ( object ): return len ( self._packages ) == 0 # --- end of empty (...) --- - def finalize_write_incremental ( self ): - """Method that finalizes incremental writing, i.e. write outstanding - ebuilds and write metadata.xml, Manifest. - """ - with self._lock: - if self.has_ebuilds(): - if self.modified: - self.write_ebuilds ( overwrite=False ) - if self._need_metadata: - self.write_metadata() - if self._need_manifest: - self.write_manifest() - else: - self.logger.critical ( - "<>: please clean up this dir: {}.".format ( - self.physical_location - ) ) - # --- end of finalize_write_incremental (...) --- - def fs_cleanup ( self ): """Cleans up the filesystem location of this package dir. To be called after keep_nth_latest, calls finalize_write_incremental(). @@ -158,11 +154,9 @@ class PackageDir ( object ): # --- end of rmtree_error (...) --- with self._lock: - if self.has_ebuilds(): - # !!! FIXME this doesn't work if no ebuilds written, but - # old ones removed -> DISTDIR unknown during Manifest creation - self.finalize_write_incremental() - elif os.path.isdir ( self.physical_location ): + if os.path.isdir ( self.physical_location ) \ + and not self.has_ebuilds() \ + : # destroy self.physical_location shutil.rmtree ( self.physical_location, onerror=rmtree_error ) # --- end of fs_cleanup (...) --- @@ -193,53 +187,58 @@ class PackageDir ( object ): arguments: * n -- # of packages/ebuilds to keep * cautious -- if True: be extra careful, verify that ebuilds exist + as file; note that this will ignore all + ebuilds that haven't been written to the file- + system yet (which implies an extra overhead, + you'll have to write all ebuilds first) """ - - # create the list of packages to iterate over, - # * package has to have an ebuild_file - # * sort them by version in reverse order (latest package gets index 0) + def is_ebuild_cautious ( p_tuple ): + # package has to have an ebuild_file that exists + efile = p_tuple [1] ['ebuild_file' ] + if efile is not None: + return os.path.isfile ( efile ) + else: + return False + # --- end of is_ebuild_cautious (...) --- + + def is_ebuild ( p_tuple ): + # package has to have an ebuild_file or an ebuild entry + return ( + p_tuple [1] ['ebuild_file'] or p_tuple [1] ['ebuild'] + ) is not None + # --- end of is_ebuild (...) --- + + # create the list of packages to iterate over (cautious/non-cautious), + # sort them by version in reverse order packages = reversed ( sorted ( filter ( - lambda p : p [1] ['ebuild_file'] is not None, - self._packages.items() + function=is_ebuild if not cautious else is_ebuild_cautious, + iterable=self._packages.items() ), key=lambda p : p [1] ['version'] ) ) + if n < 1: + raise Exception ( "Must keep more than zero ebuilds." ) + kept = 0 ecount = 0 - if not cautious: - # could use a slice here, too - for pvr, pkg in packages: - ecount += 1 - if kept < n: - printself.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) ) - kept += 1 - else: - self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) ) - self.purge_package ( pvr ) - else: - for pvr, pkg in packages: - ecount += 1 - if os.path.isfile ( pkg ['ebuild_file'] ): - if kept < n: - self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) ) - kept += 1 - else: - self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) ) - self.purge_package ( pvr ) - else: - self.logger.error ( - "{efile} is assumed to exist as file but doesn't!".format ( - efile=pkg ['ebuild_file'] - ) ) - # FIXME/IGNORE: this doesn't count inexistent files as kept + for pvr, pkg in packages: + ecount += 1 + if kept < n: + self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) ) + kept += 1 + else: + self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) ) + self.purge_package ( pvr ) + self.logger.debug ( "Kept {kept}/{total} ebuilds.".format ( kept=kept, total=ecount ) ) # FIXME: Manifest is now invalid and dir could be "empty" (no ebuilds) + # FIXME: force metadata regeneration # --- end of keep_nth_latest (...) --- def list_versions ( self ): @@ -252,7 +251,8 @@ class PackageDir ( object ): self._need_metadata = True self.modified = True if self.runtime_incremental: - return self.write_incremental() + with self._lock: + return self.write_ebuilds ( overwrite=False ) else: return True # --- end of new_ebuild (...) --- @@ -325,48 +325,104 @@ class PackageDir ( object ): arguments: * stream -- stream to use, defaults to sys.stderr - returns: None (implicit) + returns: True raises: - * IOError + * passes all exceptions (IOError, ..) """ - return self.write ( shared_fh=stream ) + self.write_ebuilds ( overwrite=True, shared_fh=stream ) + self.write_metadata ( shared_fh=stream ) + return True # --- end of show (...) --- + def virtual_cleanup ( self ): + """Removes all PackageInfos from this structure that don't have an + 'ebuild_file' entry. + """ + with self._lock: + # keyset may change during this method + for pvr in tuple ( self._packages.keys() ): + if self._packages [pvr] ['ebuild_file'] is None: + del self._packages [pvr] + # -- lock + # --- end of virtual_cleanup (...) --- + def write ( self, - shared_fh=None, overwrite_ebuilds=True, - write_ebuilds=True, write_manifest=True, write_metadata=True + overwrite_ebuilds=False, + write_ebuilds=True, write_manifest=True, write_metadata=True, + cleanup=True, keep_n_ebuilds=None, cautious=True ): """Writes this directory to its (existent!) filesystem location. arguments: - * shared_fh -- if set and not None: write everyting into * write_ebuilds -- if set and False: don't write ebuilds * write_manifest -- if set and False: don't write the Manifest file * write_metadata -- if set and False: don't write the metadata file - * overwrite_ebuilds -- if set and False: don't overwrite ebuilds - - returns: None (implicit) + * overwrite_ebuilds -- whether to overwrite ebuilds, + None means autodetect, enable overwriting + if not modified since last write + Defaults to False + * cleanup -- clean up after writing + Defaults to True + * keep_n_ebuilds -- # of ebuilds to keep (remove all others), + Defaults to None (disable) and implies cleanup + * cautious -- be cautious when keeping the nth latest ebuilds, + this has some overhead + Defaults to True + + returns: success (True/False) raises: - * IOError (?) + * passes IOError """ - with self._lock: - # mkdir not required here, overlay.Category does this + # NOTE, replaces: + # * old write: overwrite_ebuilds=True + # * finalize_write_incremental : no extra args + # * write_incremental : write_manifest=False, write_metadata=False, + # cleanup=False (or use write_ebuilds) + # BREAKS: show(), which has its own method/function now - # write ebuilds - if write_ebuilds: - self.write_ebuilds ( - overwrite=overwrite_ebuilds, shared_fh=shared_fh - ) + cleanup = cleanup or ( keep_n_ebuilds is not None ) - # write metadata - if write_metadata: - self.write_metadata ( shared_fh=shared_fh ) + success = True + with self._lock: + if self.has_ebuilds(): + # not cautious: remove ebuilds before writing them + if not cautious and keep_n_ebuilds is not None: + self.keep_nth_latest ( n=keep_n_ebuilds, cautious=False ) + + # write ebuilds + if self.modified and write_ebuilds: + success = self.write_ebuilds ( + # None ~ not modified + overwrite = overwrite_ebuilds \ + if overwrite_ebuilds is not None \ + else not self.modified + ) - # write manifest (only if shared_fh is None) - if write_manifest and shared_fh is None: - self.write_manifest() + # cautious: remove ebuilds after writing them + if cautious and keep_n_ebuilds is not None: + self.keep_nth_latest ( n=keep_n_ebuilds, cautious=True ) + + # write metadata + if self._need_metadata and write_metadata: + # don't mess around with short-circuit bool evaluation + if not self.write_metadata(): + success = False + + # write manifest (only if shared_fh is None) + if self._need_manifest and write_manifest: + if not self.write_manifest(): + success = False + # -- has_ebuilds? + + if cleanup: + self.virtual_cleanup() + self.fs_cleanup() + + # FIXME / TODO call fs_cleanup + # -- lock + return success # --- end of write (...) --- def write_ebuilds ( self, overwrite, shared_fh=None ): @@ -390,7 +446,6 @@ class PackageDir ( object ): """ _success = False try: - util.dodir ( self.physical_location ) fh = open ( efile, 'w' ) if shared_fh is None else shared_fh if ebuild_header is not None: fh.write ( str ( ebuild_header ) ) @@ -426,7 +481,14 @@ class PackageDir ( object ): all_ebuilds_written = True + # don't call dodir if shared_fh is set + hasdir = bool ( shared_fh is not None ) + for efile, p_info in ebuilds_to_write(): + if not hasdir: + util.dodir ( self.physical_location, mkdir_p=True ) + hasdir = True + if write_ebuild ( efile, p_info ['ebuild'] ): self._need_manifest = True @@ -452,17 +514,12 @@ class PackageDir ( object ): return all_ebuilds_written # --- end of write_ebuilds (...) --- - def write_incremental ( self ): - with self._lock: - return self.write_ebuilds ( overwrite=False ) - # --- end of write_incremental (...) --- - - def write_manifest ( self ): + def write_manifest ( self, ignore_empty=False ): """Generates and writes the Manifest file for this package. expects: called after writing metadata/ebuilds - returns: None (implicit) + returns: success (True/False) raises: * Exception if no ebuild exists @@ -471,47 +528,64 @@ class PackageDir ( object ): # it should be sufficient to call create_manifest for one ebuild, # choosing the latest one that exists in self.physical_location and # has enough data (DISTDIR, EBUILD_FILE) for this task. + # Additionally, all DISTDIRs (multiple repos, sub directories) have + # to be collected and passed to Manifest creation. + # => collect suitable PackageInfo objects from self._packages # - # metadata.xml's full path cannot be used for manifest creation here - # 'cause DISTDIR would be unknown - # - - # collect suitable PackageInfo instances pkgs_for_manifest = tuple ( p for p in self._packages.values() \ if p.has ( 'distdir', 'ebuild_file' ) ) if pkgs_for_manifest: - manifest.create_manifest ( pkgs_for_manifest, nofail=False ) - self._need_manifest = False + if manifest.create_manifest ( pkgs_for_manifest, nofail=False ): + self._need_manifest = False + return True + elif ignore_empty: + return True else: + # FIXME: debug statements + # FIXME: remove excpetion, maybe delete Manifest in this case,.. + for pvr, p in self._packages.items(): + print ( "{} {} ebuild={} efile={} has={}".format ( + pvr, p, p.has ('ebuild'), p ['ebuild_file'], self.has_ebuilds() + ) ) + raise Exception ( - "No ebuild written so far! I really don't know what do to!" - ) + 'In {mydir}: No ebuild written so far! ' + 'I really don\'t know what do to!'.format ( + mydir=self.physical_location + ) ) + + return False # --- end of write_manifest (...) --- def write_metadata ( self, shared_fh=None ): - """Writes metadata for this package.""" + """Writes metadata for this package. + + returns: success (True/False) + """ + success = False try: self.generate_metadata ( skip_if_existent=True ) if shared_fh is None: + util.dodir ( self.physical_location, mkdir_p=True ) if self._metadata.write(): self._need_metadata = False self._need_manifest = True - return True + success = True else: self.logger.error ( "Failed to write metadata file {}.".format ( self._metadata.filepath ) ) - return False else: self._metadata.show ( shared_fh ) - return True + success = True except Exception as e: self.logger.exception ( e ) - return False + + return success # --- end of write_metadata (...) --- diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py index 363bdb5..219fd70 100644 --- a/roverlay/overlay/root.py +++ b/roverlay/overlay/root.py @@ -48,10 +48,10 @@ class Overlay ( object ): self.incremental = incremental if self.incremental: - - self._incremental_write_lock = threading.Lock() + # this is multiple-run incremental writing (in contrast to runtime + # incremental writing, which writes ebuilds as soon as they're + # ready) FIXME: split incremental <-> runtime_incremental self.scan() - self._init_overlay ( reimport_eclass=True ) # --- end of __init__ (...) --- @@ -73,8 +73,6 @@ class Overlay ( object ): incremental=self.incremental ) self._categories [category] = newcat - if self.incremental: - util.dodir ( newcat.physical_location ) finally: self._catlock.release() @@ -226,50 +224,10 @@ class Overlay ( object ): return cat.add ( package_info ) # --- end of add (...) --- - def finalize_write_incremental ( self ): - """Writes metadata + Manifest for all packages.""" - for cat in self._categories.values(): - cat.finalize_write_incremental() - # --- end of finalize_incremental (...) --- - - def generate_manifest ( self, **manifest_kw ): - """Generates Manifest files for all ebuilds in this overlay that exist - physically/in filesystem. - Manifest files are automatically created when calling write(). - - arguments: - * **manifest_kw -- see PackageDir.generate_manifest(...) - - returns: None (implicit) - """ - for cat in self._categories.values(): - cat.generate_manifest ( **manifest_kw ) - # --- end of generate_manifest (...) --- - - def generate_metadata ( self, **metadata_kw ): - """Tells the overlay's categories to create metadata. - You don't have to call this before write()/show() unless you want to use - special metadata options. - - arguments: - * **metadata_kw -- keywords for package.PackageDir.generate_metadata(...) - - returns: None (implicit) - """ - for cat in self._categories.values(): - cat.generate_metadata ( **metadata_kw ) - # --- end of generate_metadata (...) --- - def has_dir ( self, _dir ): return os.path.isdir ( self.physical_location + os.sep + _dir ) # --- end of has_category (...) --- - def keep_nth_latest ( self, *args, **kwargs ): - """See package.py:PackageDir:keep_nth_latest.""" - for cat in self._categories.values(): - cat.keep_nth_latest ( *args, **kwargs ) - # --- end of keep_nth_latest (...) --- - def list_packages ( self, for_deprules=True ): for cat in self._categories.values(): for package in cat.list_packages ( for_deprules=True ): @@ -313,9 +271,9 @@ class Overlay ( object ): cat.show ( **show_kw ) # --- end of show (...) --- - def write ( self, **write_kw ): + def write ( self ): """Writes the overlay to its physical location (filesystem), including - metadata and Manifest files. + metadata and Manifest files as well as cleanup actions. arguments: * **write_kw -- keywords for package.PackageDir.write(...) @@ -327,35 +285,27 @@ class Overlay ( object ): ! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called when ebuild creation is done. """ - raise Exception ( "to be removed/replaced" ) - # writing profiles/ here, rewriting categories/ later self._init_overlay ( reimport_eclass=True ) for cat in self._categories.values(): - if not cat.empty(): - util.dodir ( cat.physical_location ) - cat.write ( **write_kw ) + cat.write ( + overwrite_ebuilds=False, + keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ), + cautious=True + ) # --- end of write (...) --- - def write_incremental ( self, **write_kw ): - """Writes all ebuilds that have been modified since the last write call. - Note that there are currently two modes of incremental writing: - (a) per-PackageDir incremental writing triggered by the new_ebuild() - event method and (b) "batched" incremental writing (this method) which - writes all modified PackageDirs. - """ - # FIXME merge with write(), making incremental writing the only option - # FIXME finalize_write_incremental? - if not self._incremental_write_lock.acquire(): - # another incremental write is running, drop this request - return + def write_manifest ( self, **manifest_kw ): + """Generates Manifest files for all ebuilds in this overlay that exist + physically/in filesystem. + Manifest files are automatically created when calling write(). - try: - util.dodir ( self.physical_location ) - cats = tuple ( self._categories.values() ) - for cat in cats: - util.dodir ( cat.physical_location ) - cat.write_incremental ( **write_kw ) - finally: - self._incremental_write_lock.release() - # --- end of write_incremental (...) --- + arguments: + * **manifest_kw -- see PackageDir.generate_manifest(...) + + returns: None (implicit) + """ + # FIXME: it would be good to ensure that profiles/categories exist + for cat in self._categories.values(): + cat.write_manifest ( **manifest_kw ) + # --- end of write_manifest (...) ---