From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 86B3113800E for ; Mon, 30 Jul 2012 08:54:06 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id ABC40E06D6; Mon, 30 Jul 2012 08:52:47 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 6BE02E06D6 for ; Mon, 30 Jul 2012 08:52:47 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 958071B400C for ; Mon, 30 Jul 2012 08:52:46 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id A3F6AE545A for ; Mon, 30 Jul 2012 08:52:43 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1343146867.daed3bde53ce1fec42ad806813bb2798321b76d4.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/overlay/category.py roverlay/overlay/creator.py roverlay/overlay/package.py roverlay/overlay/root.py X-VCS-Directories: roverlay/overlay/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: daed3bde53ce1fec42ad806813bb2798321b76d4 X-VCS-Branch: master Date: Mon, 30 Jul 2012 08:52:43 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: e438d073-66e5-40cb-9bef-f6422f4e8bc6 X-Archives-Hash: 2876cb46b710a006843e98993fb797b0 Message-ID: <20120730085243.xAuu3zDBNKA8RFe0PSwUUPpJD6uP59gmp0zyCSFePbQ@z> commit: daed3bde53ce1fec42ad806813bb2798321b76d4 Author: André Erdmann mailerd de> AuthorDate: Tue Jul 24 16:21:07 2012 +0000 Commit: André Erdmann mailerd de> CommitDate: Tue Jul 24 16:21:07 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=daed3bde overlay: split incremental/runtime_incremental Also added a _writeable bool to Overlay which controls whether writing is allowed or not. geändert: roverlay/overlay/category.py geändert: roverlay/overlay/creator.py geändert: roverlay/overlay/package.py geändert: roverlay/overlay/root.py --- roverlay/overlay/category.py | 36 +++++++++----- roverlay/overlay/creator.py | 20 +++---- roverlay/overlay/package.py | 23 ++++++-- roverlay/overlay/root.py | 113 +++++++++++++++++++++++++++++------------ 4 files changed, 129 insertions(+), 63 deletions(-) diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py index fab6373..a0b1607 100644 --- a/roverlay/overlay/category.py +++ b/roverlay/overlay/category.py @@ -16,25 +16,35 @@ class Category ( object ): WRITE_JOBCOUNT = 3 - def __init__ ( self, name, logger, directory, get_header, incremental ): + def __init__ ( self, + name, logger, directory, get_header, runtime_incremental + ): """Initializes a overlay/portage category (such as 'app-text', 'sci-R'). arguments: - * name -- name of the category - * logger -- parent logger - * directory -- filesystem location - * get_header -- function that returns an ebuild header + * name -- name of the category + * logger -- parent logger + * directory -- filesystem location + * get_header -- function that returns an ebuild header + * runtime_incremental -- enable/disable runtime incremental writing + for this category (and all created PackageDirs) """ - self.logger = logger.getChild ( name ) - self.name = name - self._lock = threading.RLock() - self._subdirs = dict() - self.physical_location = directory - self.get_header = get_header - self.incremental = incremental + self.logger = logger.getChild ( name ) + self.name = name + self._lock = threading.RLock() + self._subdirs = dict() + self.physical_location = directory + self.get_header = get_header + self.runtime_incremental = runtime_incremental # --- end of __init__ (...) --- def _get_package_dir ( self, pkg_name ): + """Returns a PackageDir object for pkg_name. + (so that .name == pkg_name and pkg_name in self._subdirs) + + arguments: + * pkg_name -- + """ if not pkg_name in self._subdirs: self._lock.acquire() try: @@ -44,7 +54,7 @@ class Category ( object ): logger = self.logger, directory = self.physical_location + os.sep + pkg_name, get_header = self.get_header, - incremental = self.incremental + runtime_incremental = self.runtime_incremental ) self._subdirs [pkg_name] = newpkg finally: diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py index d12e0a5..60e9517 100644 --- a/roverlay/overlay/creator.py +++ b/roverlay/overlay/creator.py @@ -82,18 +82,16 @@ class OverlayCreator ( object ): # this queue is used to propagate exceptions from threads self._err_queue = errorqueue.ErrorQueue() - self.can_write_overlay = allow_write - self.write_incremental = allow_write and USE_INCREMENTAL_WRITE - # init overlay using config values self.overlay = Overlay ( - name=config.get_or_fail ( 'OVERLAY.name' ), - logger=self.logger, - directory=config.get_or_fail ( 'OVERLAY.dir' ), - default_category= config.get_or_fail ( 'OVERLAY.category' ), - eclass_files=config.get ( 'OVERLAY.eclass_files', None ), - ebuild_header=config.get ( 'EBUILD.default_header', None ), - incremental=self.write_incremental + name = config.get_or_fail ( 'OVERLAY.name' ), + logger = self.logger, + directory = config.get_or_fail ( 'OVERLAY.dir' ), + default_category = config.get_or_fail ( 'OVERLAY.category' ), + eclass_files = config.get ( 'OVERLAY.eclass_files', None ), + ebuild_header = config.get ( 'EBUILD.default_header', None ), + incremental = USE_INCREMENTAL_WRITE, + write_allowed = allow_write ) self.depresolver = easyresolver.setup ( self._err_queue ) @@ -219,7 +217,7 @@ class OverlayCreator ( object ): arguments: """ - if self.can_write_overlay: + if self.overlay.writeable(): start = time.time() self.overlay.write() self._timestamp ( "overlay written", start ) diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py index d2662bd..bedbce3 100644 --- a/roverlay/overlay/package.py +++ b/roverlay/overlay/package.py @@ -13,15 +13,25 @@ SUPPRESS_EXCEPTIONS = True class PackageDir ( object ): EBUILD_SUFFIX = '.ebuild' - def __init__ ( self, name, logger, directory, get_header, incremental ): + def __init__ ( self, + name, logger, directory, get_header, runtime_incremental + ): """Initializes a PackageDir which contains ebuilds, metadata and a Manifest file. arguments: - * name -- name of the directory (${PN} in ebuilds) - * logger -- parent logger - * directory -- filesystem location of this PackageDir - * get_header -- function that returns an ebuild header + * name -- name of the directory (${PN} in ebuilds) + * logger -- parent logger + * directory -- filesystem location of this PackageDir + * get_header -- function that returns an ebuild header + * runtime_incremental -- enable/disable runtime incremental ebuild + writing. This trades speed (disabled) for + memory consumption (enabled) 'cause it will + write _all_ successfully created ebuilds + directly after they've been created. + Writing all ebuilds at once is generally faster + (+threading), but all PackageInfos must be + kept in memory for that. """ self.logger = logger.getChild ( name ) self.name = name @@ -30,7 +40,7 @@ class PackageDir ( object ): self._packages = dict() self.physical_location = directory self.get_header = get_header - self.runtime_incremental = incremental + self.runtime_incremental = runtime_incremental self._metadata = MetadataJob ( filepath = self.physical_location + os.sep + 'metadata.xml', @@ -250,6 +260,7 @@ class PackageDir ( object ): self._need_manifest = True self._need_metadata = True self.modified = True + # FIXME: delete PackageInfo if ebuild creation did not succeed if self.runtime_incremental: with self._lock: return self.write_ebuilds ( overwrite=False ) diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py index 219fd70..ae3267f 100644 --- a/roverlay/overlay/root.py +++ b/roverlay/overlay/root.py @@ -22,23 +22,49 @@ class Overlay ( object ): def __init__ ( self, - name, logger, directory, - default_category, eclass_files, + name, + logger, + directory, + default_category, + eclass_files, ebuild_header, - incremental + write_allowed, + incremental, + runtime_incremental=True ): - self.name = name - self.logger = logger.getChild ( 'overlay' ) - self.physical_location = directory - self.default_category = default_category - self.eclass_files = eclass_files + """Initializes an overlay. - self.ignore_existing_ebuilds = False - - self._profiles_dir = self.physical_location + os.sep + 'profiles' - self._catlock = threading.Lock() - self._categories = dict() - self._header = EbuildHeader ( ebuild_header ) + arguments: + * name -- name of this overlay + * logger -- parent logger to use + * directory -- filesystem location of this overlay + * default_category -- category of packages being added without a + specific category + * eclass_files -- eclass files to import and + inherit in all ebuilds + * ebuild_header -- the header text included in all created ebuilds + * write_allowed -- whether writing is allowed + * incremental -- enable/disable incremental writing: + use already existing ebuilds (don't recreate + them) + * runtime_incremental -- see package.py:PackageDir.__init__ (...), + Defaults to ?FIXME? + """ + self.name = name + self.logger = logger.getChild ( 'overlay' ) + self.physical_location = directory + self.default_category = default_category + + self._eclass_files = eclass_files + #self._incremental = incremental + # disable runtime_incremental if writing not allowed + self._runtime_incremental = write_allowed and runtime_incremental + self._writeable = write_allowed + + self._profiles_dir = self.physical_location + os.sep + 'profiles' + self._catlock = threading.Lock() + self._categories = dict() + self._header = EbuildHeader ( ebuild_header ) # fixme or ignore: calculating eclass names twice, # once here and another time when calling _init_overlay @@ -46,13 +72,12 @@ class Overlay ( object ): self._get_eclass_import_info ( only_eclass_names=True ) ) ) - self.incremental = incremental - if self.incremental: + #if self._incremental: + if incremental: # this is multiple-run incremental writing (in contrast to runtime # incremental writing, which writes ebuilds as soon as they're - # ready) FIXME: split incremental <-> runtime_incremental + # ready) self.scan() - # --- end of __init__ (...) --- def _get_category ( self, category ): @@ -70,7 +95,7 @@ class Overlay ( object ): self.logger, self.physical_location + os.sep + category, get_header=self._header.get, - incremental=self.incremental + runtime_incremental=self._runtime_incremental ) self._categories [category] = newcat finally: @@ -89,9 +114,9 @@ class Overlay ( object ): raises: AssertionError if a file does not end with '.eclass'. """ - if self.eclass_files: + if self._eclass_files: - for eclass in self.eclass_files: + for eclass in self._eclass_files: dest = os.path.splitext ( os.path.basename ( eclass ) ) if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ): @@ -116,7 +141,7 @@ class Overlay ( object ): * Exception if copying fails """ - if self.eclass_files: + if self._eclass_files: # import eclass files eclass_dir = self.physical_location + os.sep + 'eclass' try: @@ -240,6 +265,10 @@ class Overlay ( object ): yield kwargs # --- end of list_rule_kwargs (...) --- + def readonly ( self ): + return not self._writeable + # --- end of readonly (...) --- + def scan ( self, **kw ): def scan_categories(): for x in os.listdir ( self.physical_location ): @@ -271,6 +300,10 @@ class Overlay ( object ): cat.show ( **show_kw ) # --- end of show (...) --- + def writeable ( self ): + return self._writeable + # --- end of writeable (...) --- + def write ( self ): """Writes the overlay to its physical location (filesystem), including metadata and Manifest files as well as cleanup actions. @@ -285,14 +318,21 @@ class Overlay ( object ): ! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called when ebuild creation is done. """ - self._init_overlay ( reimport_eclass=True ) - - for cat in self._categories.values(): - cat.write ( - overwrite_ebuilds=False, - keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ), - cautious=True - ) + if self._writeable: + self._init_overlay ( reimport_eclass=True ) + + for cat in self._categories.values(): + cat.write ( + overwrite_ebuilds=False, + keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ), + cautious=True + ) + else: + # FIXME debug print + print ( + "Dropped write request for readonly overlay {}!".format ( + self.name + ) ) # --- end of write (...) --- def write_manifest ( self, **manifest_kw ): @@ -305,7 +345,14 @@ class Overlay ( object ): returns: None (implicit) """ - # FIXME: it would be good to ensure that profiles/categories exist - for cat in self._categories.values(): - cat.write_manifest ( **manifest_kw ) + if self._writeable: + # FIXME: it would be good to ensure that profiles/categories exist + for cat in self._categories.values(): + cat.write_manifest ( **manifest_kw ) + else: + # FIXME debug print + print ( + "Dropped write_manifest request for readonly overlay {}!".format ( + self.name + ) ) # --- end of write_manifest (...) ---