[gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/

[gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/

[André Erdmann]

commit:     daed3bde53ce1fec42ad806813bb2798321b76d4
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:21:07 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:21:07 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=daed3bde

overlay: split incremental/runtime_incremental

Also added a _writeable bool to Overlay which controls
whether writing is allowed or not.

	geändert:   roverlay/overlay/category.py
	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/package.py
	geändert:   roverlay/overlay/root.py

---
 roverlay/overlay/category.py |   36 +++++++++-----
 roverlay/overlay/creator.py  |   20 +++----
 roverlay/overlay/package.py  |   23 ++++++--
 roverlay/overlay/root.py     |  113 +++++++++++++++++++++++++++++------------
 4 files changed, 129 insertions(+), 63 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index fab6373..a0b1607 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -16,25 +16,35 @@ class Category ( object ):
 
 	WRITE_JOBCOUNT = 3
 
-	def __init__ ( self, name, logger, directory, get_header, incremental ):
+	def __init__ ( self,
+		name, logger, directory, get_header, runtime_incremental
+	):
 		"""Initializes a overlay/portage category (such as 'app-text', 'sci-R').
 
 		arguments:
-		* name       -- name of the category
-		* logger     -- parent logger
-		* directory  -- filesystem location
-		* get_header -- function that returns an ebuild header
+		* name                -- name of the category
+		* logger              -- parent logger
+		* directory           -- filesystem location
+		* get_header          -- function that returns an ebuild header
+		* runtime_incremental -- enable/disable runtime incremental writing
+		                         for this category (and all created PackageDirs)
 		"""
-		self.logger            = logger.getChild ( name )
-		self.name              = name
-		self._lock             = threading.RLock()
-		self._subdirs          = dict()
-		self.physical_location = directory
-		self.get_header        = get_header
-		self.incremental       = incremental
+		self.logger              = logger.getChild ( name )
+		self.name                = name
+		self._lock               = threading.RLock()
+		self._subdirs            = dict()
+		self.physical_location   = directory
+		self.get_header          = get_header
+		self.runtime_incremental = runtime_incremental
 	# --- end of __init__ (...) ---
 
 	def _get_package_dir ( self, pkg_name ):
+		"""Returns a PackageDir object for pkg_name.
+		(so that <new object>.name == pkg_name and pkg_name in self._subdirs)
+
+		arguments:
+		* pkg_name --
+		"""
 		if not pkg_name in self._subdirs:
 			self._lock.acquire()
 			try:
@@ -44,7 +54,7 @@ class Category ( object ):
 						logger      = self.logger,
 						directory   = self.physical_location + os.sep + pkg_name,
 						get_header  = self.get_header,
-						incremental = self.incremental
+						runtime_incremental = self.runtime_incremental
 					)
 					self._subdirs [pkg_name] = newpkg
 			finally:

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index d12e0a5..60e9517 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -82,18 +82,16 @@ class OverlayCreator ( object ):
 		# this queue is used to propagate exceptions from threads
 		self._err_queue = errorqueue.ErrorQueue()
 
-		self.can_write_overlay = allow_write
-		self.write_incremental = allow_write and USE_INCREMENTAL_WRITE
-
 		# init overlay using config values
 		self.overlay = Overlay (
-			name=config.get_or_fail ( 'OVERLAY.name' ),
-			logger=self.logger,
-			directory=config.get_or_fail ( 'OVERLAY.dir' ),
-			default_category= config.get_or_fail ( 'OVERLAY.category' ),
-			eclass_files=config.get ( 'OVERLAY.eclass_files', None ),
-			ebuild_header=config.get ( 'EBUILD.default_header', None ),
-			incremental=self.write_incremental
+			name             = config.get_or_fail ( 'OVERLAY.name' ),
+			logger           = self.logger,
+			directory        = config.get_or_fail ( 'OVERLAY.dir' ),
+			default_category = config.get_or_fail ( 'OVERLAY.category' ),
+			eclass_files     = config.get ( 'OVERLAY.eclass_files', None ),
+			ebuild_header    = config.get ( 'EBUILD.default_header', None ),
+			incremental      = USE_INCREMENTAL_WRITE,
+			write_allowed    = allow_write
 		)
 
 		self.depresolver = easyresolver.setup ( self._err_queue )
@@ -219,7 +217,7 @@ class OverlayCreator ( object ):
 
 		arguments:
 		"""
-		if self.can_write_overlay:
+		if self.overlay.writeable():
 			start = time.time()
 			self.overlay.write()
 			self._timestamp ( "overlay written", start )

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index d2662bd..bedbce3 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -13,15 +13,25 @@ SUPPRESS_EXCEPTIONS = True
 class PackageDir ( object ):
 	EBUILD_SUFFIX = '.ebuild'
 
-	def __init__ ( self, name, logger, directory, get_header, incremental ):
+	def __init__ ( self,
+		name, logger, directory, get_header, runtime_incremental
+	):
 		"""Initializes a PackageDir which contains ebuilds, metadata and
 		a Manifest file.
 
 		arguments:
-		* name       -- name of the directory (${PN} in ebuilds)
-		* logger     -- parent logger
-		* directory  -- filesystem location of this PackageDir
-		* get_header -- function that returns an ebuild header
+		* name                -- name of the directory (${PN} in ebuilds)
+		* logger              -- parent logger
+		* directory           -- filesystem location of this PackageDir
+		* get_header          -- function that returns an ebuild header
+		* runtime_incremental -- enable/disable runtime incremental ebuild
+		                         writing. This trades speed (disabled) for
+		                         memory consumption (enabled) 'cause it will
+		                         write _all_ successfully created ebuilds
+		                         directly after they've been created.
+		                         Writing all ebuilds at once is generally faster
+		                         (+threading), but all PackageInfos must be
+		                         kept in memory for that.
 		"""
 		self.logger              = logger.getChild ( name )
 		self.name                = name
@@ -30,7 +40,7 @@ class PackageDir ( object ):
 		self._packages           = dict()
 		self.physical_location   = directory
 		self.get_header          = get_header
-		self.runtime_incremental = incremental
+		self.runtime_incremental = runtime_incremental
 
 		self._metadata = MetadataJob (
 			filepath = self.physical_location + os.sep + 'metadata.xml',
@@ -250,6 +260,7 @@ class PackageDir ( object ):
 		self._need_manifest = True
 		self._need_metadata = True
 		self.modified       = True
+		# FIXME: delete PackageInfo if ebuild creation did not succeed
 		if self.runtime_incremental:
 			with self._lock:
 				return self.write_ebuilds ( overwrite=False )

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 219fd70..ae3267f 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -22,23 +22,49 @@ class Overlay ( object ):
 
 	def __init__ (
 		self,
-		name, logger, directory,
-		default_category, eclass_files,
+		name,
+		logger,
+		directory,
+		default_category,
+		eclass_files,
 		ebuild_header,
-		incremental
+		write_allowed,
+		incremental,
+		runtime_incremental=True
 	):
-		self.name              = name
-		self.logger            = logger.getChild ( 'overlay' )
-		self.physical_location = directory
-		self.default_category  = default_category
-		self.eclass_files      = eclass_files
+		"""Initializes an overlay.
 
-		self.ignore_existing_ebuilds = False
-
-		self._profiles_dir     = self.physical_location + os.sep + 'profiles'
-		self._catlock          = threading.Lock()
-		self._categories       = dict()
-		self._header           = EbuildHeader ( ebuild_header )
+		arguments:
+		* name                -- name of this overlay
+		* logger              -- parent logger to use
+		* directory           -- filesystem location of this overlay
+		* default_category    -- category of packages being added without a
+		                          specific category
+		* eclass_files        -- eclass files to import and
+		                          inherit in all ebuilds
+		* ebuild_header       -- the header text included in all created ebuilds
+		* write_allowed       -- whether writing is allowed
+		* incremental         -- enable/disable incremental writing:
+		                         use already existing ebuilds (don't recreate
+		                         them)
+		* runtime_incremental -- see package.py:PackageDir.__init__ (...),
+		                          Defaults to ?FIXME?
+		"""
+		self.name                 = name
+		self.logger               = logger.getChild ( 'overlay' )
+		self.physical_location    = directory
+		self.default_category     = default_category
+
+		self._eclass_files        = eclass_files
+		#self._incremental         = incremental
+		# disable runtime_incremental if writing not allowed
+		self._runtime_incremental = write_allowed and runtime_incremental
+		self._writeable           = write_allowed
+
+		self._profiles_dir        = self.physical_location + os.sep + 'profiles'
+		self._catlock             = threading.Lock()
+		self._categories          = dict()
+		self._header              = EbuildHeader ( ebuild_header )
 
 		# fixme or ignore: calculating eclass names twice,
 		# once here and another time when calling _init_overlay
@@ -46,13 +72,12 @@ class Overlay ( object ):
 			self._get_eclass_import_info ( only_eclass_names=True )
 		) )
 
-		self.incremental = incremental
-		if self.incremental:
+		#if self._incremental:
+		if incremental:
 			# this is multiple-run incremental writing (in contrast to runtime
 			# incremental writing, which writes ebuilds as soon as they're
-			# ready) FIXME: split incremental <-> runtime_incremental
+			# ready)
 			self.scan()
-
 	# --- end of __init__ (...) ---
 
 	def _get_category ( self, category ):
@@ -70,7 +95,7 @@ class Overlay ( object ):
 						self.logger,
 						self.physical_location + os.sep + category,
 						get_header=self._header.get,
-						incremental=self.incremental
+						runtime_incremental=self._runtime_incremental
 					)
 					self._categories [category] = newcat
 			finally:
@@ -89,9 +114,9 @@ class Overlay ( object ):
 
 		raises: AssertionError if a file does not end with '.eclass'.
 		"""
-		if self.eclass_files:
+		if self._eclass_files:
 
-			for eclass in self.eclass_files:
+			for eclass in self._eclass_files:
 				dest = os.path.splitext ( os.path.basename ( eclass ) )
 
 				if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ):
@@ -116,7 +141,7 @@ class Overlay ( object ):
 		* Exception if copying fails
 		"""
 
-		if self.eclass_files:
+		if self._eclass_files:
 			# import eclass files
 			eclass_dir = self.physical_location + os.sep +  'eclass'
 			try:
@@ -240,6 +265,10 @@ class Overlay ( object ):
 				yield kwargs
 	# --- end of list_rule_kwargs (...) ---
 
+	def readonly ( self ):
+		return not self._writeable
+	# --- end of readonly (...) ---
+
 	def scan ( self, **kw ):
 		def scan_categories():
 			for x in os.listdir ( self.physical_location ):
@@ -271,6 +300,10 @@ class Overlay ( object ):
 			cat.show ( **show_kw )
 	# --- end of show (...) ---
 
+	def writeable ( self ):
+		return self._writeable
+	# --- end of writeable (...) ---
+
 	def write ( self ):
 		"""Writes the overlay to its physical location (filesystem), including
 		metadata and Manifest files as well as cleanup actions.
@@ -285,14 +318,21 @@ class Overlay ( object ):
 		! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
 		when ebuild creation is done.
 		"""
-		self._init_overlay ( reimport_eclass=True )
-
-		for cat in self._categories.values():
-			cat.write (
-				overwrite_ebuilds=False,
-				keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
-				cautious=True
-			)
+		if self._writeable:
+			self._init_overlay ( reimport_eclass=True )
+
+			for cat in self._categories.values():
+				cat.write (
+					overwrite_ebuilds=False,
+					keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
+					cautious=True
+				)
+		else:
+			# FIXME debug print
+			print (
+				"Dropped write request for readonly overlay {}!".format (
+					self.name
+			) )
 	# --- end of write (...) ---
 
 	def write_manifest ( self, **manifest_kw ):
@@ -305,7 +345,14 @@ class Overlay ( object ):
 
 		returns: None (implicit)
 		"""
-		# FIXME: it would be good to ensure that profiles/categories exist
-		for cat in self._categories.values():
-			cat.write_manifest ( **manifest_kw )
+		if self._writeable:
+			# FIXME: it would be good to ensure that profiles/categories exist
+			for cat in self._categories.values():
+				cat.write_manifest ( **manifest_kw )
+		else:
+			# FIXME debug print
+			print (
+				"Dropped write_manifest request for readonly overlay {}!".format (
+					self.name
+			) )
 	# --- end of write_manifest (...) ---

[gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/

[André Erdmann]

commit:     b568e1b218a2ed83dc0ae36bb4a3326b19078034
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:58:09 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:58:09 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b568e1b2

stop thread when queue is empty

---
 roverlay/overlay/category.py |   25 ++++++++++++-------------
 1 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index a0b1607..2d66246 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -145,19 +145,18 @@ class Category ( object ):
 			* q        -- queue
 			* write_kw -- keywords for write(...)
 			"""
-			try:
-				while not q.empty():
-					try:
-						pkg = q.get_nowait()
-						# remove manifest writing from threaded writing since it's
-						# single-threaded
-						pkg.write ( write_manifest=False, **write_kw )
-					#except ( Exception, KeyboardInterrupt ) as e:
-					except Exception as e:
-						# FIXME: reintroduce RERAISE
-						self.logger.exception ( e )
-			except queue.Empty:
-				pass
+			while not q.empty():
+				try:
+					pkg = q.get_nowait()
+					# remove manifest writing from threaded writing since it's
+					# single-threaded
+					pkg.write ( write_manifest=False, **write_kw )
+				except queue.Empty:
+					break
+				#except ( Exception, KeyboardInterrupt ) as e:
+				except Exception as e:
+					# FIXME: reintroduce RERAISE
+					self.logger.exception ( e )
 		# --- end of run_write_queue (...) ---
 
 		if len ( self._subdirs ) == 0: return

[gentoo-commits] proj/R_overlay:overlay_wip commit in: roverlay/overlay/

[André Erdmann]

commit:     c456433d70dae2e58fb0297ab1b82822b2405b2d
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:58:40 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:58:40 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=c456433d

add "ebuild uncreatable" event

Immediately removes the PackageInfo object from the overlay
structure to save some memory.

	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/package.py

---
 roverlay/overlay/creator.py |    1 +
 roverlay/overlay/package.py |   22 ++++++++++++++++++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 60e9517..5b0a973 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -382,6 +382,7 @@ class OverlayCreator ( object ):
 			if package_info.overlay_package_ref.new_ebuild():
 				self.overlay_added.inc()
 		else:
+			package_info.overlay_package_ref.ebuild_uncreateable ( package_info )
 			self.create_fail.inc()
 
 	# --- end of _add_to_overlay (...) ---

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index bedbce3..cb5cf72 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -145,6 +145,25 @@ class PackageDir ( object ):
 			return False
 	# --- end of check_empty (...) ---
 
+	def ebuild_uncreateable ( self, package_info ):
+		"""Called when ebuild creation (finally) failed for a PackageInfo
+		object of this PackageDir.
+
+		arguments:
+		* package_info --
+		"""
+		try:
+			self._lock.acquire()
+			pvr = package_info ['ebuild_verstr']
+			# FIXME debug print
+			print ( "removing {PVR} from {PN}".format ( PVR=pvr, PN=self.name ) )
+			del self._packages [pvr]
+		except KeyError:
+			pass
+		finally:
+			self._lock.release()
+	# --- end of uncreateable_ebuild (...) ---
+
 	def empty ( self ):
 		"""Returns True if no ebuilds stored, else False.
 		Note that "not empty" doesn't mean "has ebuilds to write" or "has
@@ -255,12 +274,11 @@ class PackageDir ( object ):
 		return self._packages.keys()
 	# --- end of list_versions (...) ---
 
-	def new_ebuild ( self, write=False ):
+	def new_ebuild ( self ):
 		"""Called when a new ebuild has been created for this PackageDir."""
 		self._need_manifest = True
 		self._need_metadata = True
 		self.modified       = True
-		# FIXME: delete PackageInfo if ebuild creation did not succeed
 		if self.runtime_incremental:
 			with self._lock:
 				return self.write_ebuilds ( overwrite=False )

[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/

[André Erdmann]

commit:     b568e1b218a2ed83dc0ae36bb4a3326b19078034
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 16:58:09 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 16:58:09 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b568e1b2

stop thread when queue is empty

---
 roverlay/overlay/category.py |   25 ++++++++++++-------------
 1 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index a0b1607..2d66246 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -145,19 +145,18 @@ class Category ( object ):
 			* q        -- queue
 			* write_kw -- keywords for write(...)
 			"""
-			try:
-				while not q.empty():
-					try:
-						pkg = q.get_nowait()
-						# remove manifest writing from threaded writing since it's
-						# single-threaded
-						pkg.write ( write_manifest=False, **write_kw )
-					#except ( Exception, KeyboardInterrupt ) as e:
-					except Exception as e:
-						# FIXME: reintroduce RERAISE
-						self.logger.exception ( e )
-			except queue.Empty:
-				pass
+			while not q.empty():
+				try:
+					pkg = q.get_nowait()
+					# remove manifest writing from threaded writing since it's
+					# single-threaded
+					pkg.write ( write_manifest=False, **write_kw )
+				except queue.Empty:
+					break
+				#except ( Exception, KeyboardInterrupt ) as e:
+				except Exception as e:
+					# FIXME: reintroduce RERAISE
+					self.logger.exception ( e )
 		# --- end of run_write_queue (...) ---
 
 		if len ( self._subdirs ) == 0: return