[gentoo-commits] proj/catalyst:master commit in: catalyst/

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     02b002e091994999abd74d9c34e5ad9a34d41e17
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Mar  2 16:43:16 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Mar  2 22:54:04 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=02b002e0

catalyst/support.py: Update for snakeoil API change.

---
 catalyst/support.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 316dfa3..5e7ce92 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -618,8 +618,8 @@ def read_makeconf(mymakeconffile):
 	if os.path.exists(mymakeconffile):
 		try:
 			try:
-				import snakeoil.fileutils
-				return snakeoil.fileutils.read_bash_dict(mymakeconffile, sourcing_command="source")
+				import snakeoil.bash #import snakeoil.fileutils
+				return snakeoil.bash.read_bash_dict(mymakeconffile, sourcing_command="source")
 			except ImportError:
 				try:
 					import portage.util

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     380085b1b64a80cd6fc8518309e8ca9e31246ba3
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 14 03:32:29 2013 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Apr  2 20:04:23 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=380085b1

Fix undefined variable: RLIMIT_NOFILE

It was not imported from resource, it was also not used correctly.

---
 catalyst/support.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index e25394e..5abf614 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -6,14 +6,16 @@ selinux_capable = False
 #fakeroot_capable = False
 BASH_BINARY             = "/bin/bash"
 
+# set it to 0 for the soft limit, 1 for the hard limit
+DESIRED_RLIMIT = 0
 try:
-        import resource
-        max_fd_limit=resource.getrlimit(RLIMIT_NOFILE)
+	import resource
+	max_fd_limit=resource.getrlimit(resource.RLIMIT_NOFILE)[DESIRED_RLIMIT]
 except SystemExit, e:
-        raise
+	raise
 except:
-        # hokay, no resource module.
-        max_fd_limit=256
+	# hokay, no resource module.
+	max_fd_limit=256
 
 # pids this process knows of.
 spawned_pids = []

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     876edf3306f3d1c739f60199bef6082e4ac96459
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Jan  3 18:48:31 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Apr  2 20:04:23 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=876edf33

main.py: print the output of an ImportError to help in debugging.

---
 catalyst/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 4146bca..bba3cba 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -227,9 +227,10 @@ def import_modules():
 				raise CatalystError,"Can't find " + x + ".py plugin in " + \
 					module_dir
 
-	except ImportError:
+	except ImportError as e:
 		print "!!! catalyst: Python modules not found in "+\
 			module_dir + "; exiting."
+		print e
 		sys.exit(1)
 
 	return targetmap

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     e072dadbba8758c67bc83ee284af3453c734d7bd
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Jan  4 01:02:09 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Apr  2 20:04:24 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=e072dadb

Creation of a new defaults.py

catalyst/support.py: Move some defaults to a new defaults file.

---
 catalyst/defaults.py | 23 +++++++++++++++++++++++
 catalyst/main.py     |  5 ++---
 catalyst/support.py  | 35 +++--------------------------------
 3 files changed, 28 insertions(+), 35 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
new file mode 100644
index 0000000..b1dbda4
--- /dev/null
+++ b/catalyst/defaults.py
@@ -0,0 +1,23 @@
+
+
+# these should never be touched
+required_build_targets = ["generic_target", "generic_stage_target"]
+
+# new build types should be added here
+valid_build_targets = ["stage1_target", "stage2_target", "stage3_target",
+	"stage4_target", "grp_target", "livecd_stage1_target", "livecd_stage2_target",
+	"embedded_target", "tinderbox_target", "snapshot_target", "netboot_target",
+	"netboot2_target"
+	]
+
+required_config_file_values = ["storedir", "sharedir", "distdir", "portdir"]
+
+valid_config_file_values = required_config_file_values[:]
+valid_config_file_values.extend(["PKGCACHE", "KERNCACHE", "CCACHE", "DISTCC",
+	"ICECREAM", "ENVSCRIPT", "AUTORESUME", "FETCH", "CLEAR_AUTORESUME",
+	"options", "DEBUG", "VERBOSE", "PURGE", "PURGEONLY", "SNAPCACHE",
+	"snapshot_cache", "hash_function", "digests", "contents", "SEEDCACHE"
+	])
+
+verbosity = 1
+

diff --git a/catalyst/main.py b/catalyst/main.py
index bba3cba..e2ef976 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -21,9 +21,8 @@ sys.path.append(__selfpath__ + "/modules")
 from . import __version__
 import catalyst.config
 import catalyst.util
-from catalyst.support import (required_build_targets,
-	valid_build_targets, CatalystError, find_binary, LockInUse)
-
+from catalyst.support import CatalystError, find_binary, LockInUse
+from catalyst.defaults import required_build_targets, valid_build_targets
 from hash_utils import HashMap, HASH_DEFINITIONS
 from contents import ContentsMap, CONTENTS_DEFINITIONS
 

diff --git a/catalyst/support.py b/catalyst/support.py
index 5abf614..4fe4603 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -1,6 +1,9 @@
 
 import sys,string,os,types,re,signal,traceback,time
 #import md5,sha
+
+from catalyst.defaults import verbosity, valid_config_file_values
+
 selinux_capable = False
 #userpriv_capable = (os.getuid() == 0)
 #fakeroot_capable = False
@@ -80,38 +83,6 @@ def read_from_clst(file):
 	return myline
 # read_from_clst
 
-# these should never be touched
-required_build_targets=["generic_target","generic_stage_target"]
-
-# new build types should be added here
-valid_build_targets=["stage1_target","stage2_target","stage3_target","stage4_target","grp_target",
-			"livecd_stage1_target","livecd_stage2_target","embedded_target",
-			"tinderbox_target","snapshot_target","netboot_target","netboot2_target"]
-
-required_config_file_values=["storedir","sharedir","distdir","portdir"]
-valid_config_file_values=required_config_file_values[:]
-valid_config_file_values.append("PKGCACHE")
-valid_config_file_values.append("KERNCACHE")
-valid_config_file_values.append("CCACHE")
-valid_config_file_values.append("DISTCC")
-valid_config_file_values.append("ICECREAM")
-valid_config_file_values.append("ENVSCRIPT")
-valid_config_file_values.append("AUTORESUME")
-valid_config_file_values.append("FETCH")
-valid_config_file_values.append("CLEAR_AUTORESUME")
-valid_config_file_values.append("options")
-valid_config_file_values.append("DEBUG")
-valid_config_file_values.append("VERBOSE")
-valid_config_file_values.append("PURGE")
-valid_config_file_values.append("PURGEONLY")
-valid_config_file_values.append("SNAPCACHE")
-valid_config_file_values.append("snapshot_cache")
-valid_config_file_values.append("hash_function")
-valid_config_file_values.append("digests")
-valid_config_file_values.append("contents")
-valid_config_file_values.append("SEEDCACHE")
-
-verbosity=1
 
 def list_bashify(mylist):
 	if type(mylist)==types.StringType:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     69c1839e501cbbf05c1790b267ed1b5b356091c0
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Jan 19 05:51:35 2013 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Apr  2 20:04:24 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=69c1839e

Move confdefaults out of main.py

---
 catalyst/defaults.py | 17 +++++++++++++++++
 catalyst/main.py     | 20 +++-----------------
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 748d1dd..b83e4f5 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -21,6 +21,22 @@ valid_config_file_values.extend(["PKGCACHE", "KERNCACHE", "CCACHE", "DISTCC",
 
 verbosity = 1
 
+confdefaults={
+	"distdir": "/usr/portage/distfiles",
+	"hash_function": "crc32",
+	"icecream": "/var/cache/icecream",
+	"local_overlay": "/usr/local/portage",
+	"options": "",
+	"packagedir": "/usr/portage/packages",
+	"portdir": "/usr/portage",
+	"port_tmpdir": "/var/tmp/portage",
+	"repo_name": "portage",
+	"sharedir": "/usr/lib/catalyst",
+	"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
+	"snapshot_name": "portage-",
+	"storedir": "/var/tmp/catalyst",
+	}
+
 PORT_LOGDIR_CLEAN = \
 	'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +30 -delete'
 
@@ -48,3 +64,4 @@ SOURCE_MOUNT_DEFAULTS = {
 	"proc": "/proc",
 	"shm": "shmfs",
 	}
+

diff --git a/catalyst/main.py b/catalyst/main.py
index e2ef976..5748d31 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -22,7 +22,8 @@ from . import __version__
 import catalyst.config
 import catalyst.util
 from catalyst.support import CatalystError, find_binary, LockInUse
-from catalyst.defaults import required_build_targets, valid_build_targets
+from catalyst.defaults import (required_build_targets, valid_build_targets,
+	confdefaults)
 from hash_utils import HashMap, HASH_DEFINITIONS
 from contents import ContentsMap, CONTENTS_DEFINITIONS
 
@@ -70,21 +71,6 @@ def parse_config(myconfig):
 	myconf={}
 	config_file=""
 
-	confdefaults = {
-		"distdir": "/usr/portage/distfiles",
-		"hash_function": "crc32",
-		"icecream": "/var/cache/icecream",
-		"local_overlay": "/usr/local/portage",
-		"options": "",
-		"packagedir": "/usr/portage/packages",
-		"portdir": "/usr/portage",
-		"repo_name": "portage",
-		"sharedir": "/usr/share/catalyst",
-		"snapshot_name": "portage-",
-		"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
-		"storedir": "/var/tmp/catalyst",
-		}
-
 	# first, try the one passed (presumably from the cmdline)
 	if myconfig:
 		if os.path.exists(myconfig):
@@ -117,7 +103,7 @@ def parse_config(myconfig):
 		sys.exit(1)
 
 	# now, load up the values into conf_values so that we can use them
-	for x in confdefaults.keys():
+	for x in list(confdefaults):
 		if x in myconf:
 			print "Setting",x,"to config file value \""+myconf[x]+"\""
 			conf_values[x]=myconf[x]

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     c71831578f53d2551cd845b25bee3aaee3baa6f4
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 12 02:49:19 2013 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Jun 14 03:51:36 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=c7183157

Remove unused urllib import.

This urllib import was added in commit 64c16cae70da13de3c55d8555a2e4c5dcdf2fcad
to fix an issue, but it is not used.  So must have covered up the real bug.  Removing it now
that I've completed some testing and haven't come across anything I can attribute to it.

---
 catalyst/support.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 4fe4603..bc24130 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -23,10 +23,6 @@ except:
 # pids this process knows of.
 spawned_pids = []
 
-try:
-        import urllib
-except SystemExit, e:
-        raise
 
 def cleanup(pids,block_exceptions=True):
         """function to go through and reap the list of pids passed to it"""

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     c43de87222e118738e21afe8ea0cfafb866c7044
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 12 02:54:40 2013 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Jun 14 03:57:22 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=c43de872

Remove unused variable new and an undefined variable s.

Commit b3475906d5f51a21ecaf4ff048002a2f44face52 introduced an
undefined variable "s".  The code must always be hitting the
general except: pass block.  Seems useless, if not maybe it'll
spit out a true error, so the real problem can be fixed.  Removed it all.

---
 catalyst/support.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index bc24130..d5dbfec 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -292,15 +292,7 @@ def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,retur
 				if trg_fd[x] == src_fd[x]:
 					continue
 				if trg_fd[x] in src_fd[x+1:]:
-					new=os.dup2(trg_fd[x],max(src_fd) + 1)
 					os.close(trg_fd[x])
-					try:
-						while True:
-							src_fd[s.index(trg_fd[x])]=new
-					except SystemExit, e:
-						raise
-					except:
-						pass
 
 			# transfer the fds to their final pre-exec position.
 			for x in range(0,len(trg_fd)):

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     cd9ed84add5379f07f46faea102f24ec51754f54
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 20 01:50:26 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan  1 05:58:05 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=cd9ed84a

Move LockInUse from support.py to lock.py, fix bad execption raising,  pyflakes cleanup

---
 catalyst/lock.py    | 55 ++++++++++++++++++++++++++++++-----------------------
 catalyst/main.py    |  3 ++-
 catalyst/support.py | 11 -----------
 3 files changed, 33 insertions(+), 36 deletions(-)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index 2d10d2f..78968cb 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -3,14 +3,26 @@ import os
 import fcntl
 import errno
 import sys
-import string
 import time
-from catalyst.support import *
+from catalyst.support import CatalystError, normpath
 
 def writemsg(mystr):
 	sys.stderr.write(mystr)
 	sys.stderr.flush()
 
+
+class LockInUse(Exception):
+	def __init__(self, message):
+		if message:
+			#(type,value)=sys.exc_info()[:2]
+			#if value!=None:
+			    #print
+			    #kprint traceback.print_exc(file=sys.stdout)
+			print
+			print "!!! catalyst lock file in use: "+message
+			print
+
+
 class LockDir:
 	locking_method=fcntl.flock
 	lock_dirs_in_use=[]
@@ -109,7 +121,8 @@ class LockDir:
 	def fcntl_lock(self,locktype):
 		if self.myfd==None:
 			if not os.path.exists(os.path.dirname(self.lockdir)):
-				raise DirectoryNotFound, os.path.dirname(self.lockdir)
+				raise CatalystError("DirectoryNotFound: %s"
+					% os.path.dirname(self.lockdir))
 			if not os.path.exists(self.lockfile):
 				old_mask=os.umask(000)
 				self.myfd = os.open(self.lockfile, os.O_CREAT|os.O_RDWR,0660)
@@ -168,7 +181,7 @@ class LockDir:
 			print "lockfile does not exist '%s'" % self.lockfile
 			if (self.myfd != None):
 				try:
-					os.close(myfd)
+					os.close(self.myfd)
 					self.myfd=None
 				except:
 					pass
@@ -251,12 +264,13 @@ class LockDir:
 
 			self.add_hardlock_file_to_cleanup()
 			if not os.path.exists(self.myhardlock):
-				raise FileNotFound, "Created lockfile is missing: %(filename)s" % {"filename":self.myhardlock}
+				raise CatalystError("FileNotFound: Created lockfile is missing: "
+					"%(filename)s" % {"filename":self.myhardlock})
 			try:
-				res = os.link(self.myhardlock, self.lockfile)
-			except SystemExit, e:
+				os.link(self.myhardlock, self.lockfile)
+			except SystemExit:
 				raise
-			except Exception, e:
+			except Exception:
 #				if "DEBUG" in self.settings:
 #					print "lockfile(): Hardlink: Link failed."
 #					print "Exception: ",e
@@ -286,7 +300,7 @@ class LockDir:
 				os.unlink(self.myhardlock)
 			if os.path.exists(self.lockfile):
 				os.unlink(self.lockfile)
-		except SystemExit, e:
+		except SystemExit:
 			raise
 		except:
 			writemsg("Something strange happened to our hardlink locks.\n")
@@ -314,7 +328,7 @@ class LockDir:
 		try:
 			myhls = os.stat(link)
 			mylfs = os.stat(lock)
-		except SystemExit, e:
+		except SystemExit:
 			raise
 		except:
 			myhls = None
@@ -340,7 +354,7 @@ class LockDir:
 			pass
 
 	def hardlock_cleanup(self,path):
-		mypid  = str(os.getpid())
+		#mypid  = str(os.getpid())
 		myhost = os.uname()[1]
 		mydl = os.listdir(path)
 		results = []
@@ -384,26 +398,26 @@ class LockDir:
 								# We're sweeping through, unlinking everyone's locks.
 								os.unlink(filename)
 								results.append("Unlinked: " + filename)
-							except SystemExit, e:
+							except SystemExit:
 								raise
-							except Exception,e:
+							except Exception:
 								pass
 					try:
 						os.unlink(x)
 						results.append("Unlinked: " + x)
 						os.unlink(mylockname)
 						results.append("Unlinked: " + mylockname)
-					except SystemExit, e:
+					except SystemExit:
 						raise
-					except Exception,e:
+					except Exception:
 						pass
 				else:
 					try:
 						os.unlink(mylockname)
 						results.append("Unlinked: " + mylockname)
-					except SystemExit, e:
+					except SystemExit:
 						raise
-					except Exception,e:
+					except Exception:
 						pass
 		return results
 
@@ -415,15 +429,8 @@ if __name__ == "__main__":
 			print i,time.time()
 			time.sleep(1)
 		print
-	def normpath(mypath):
-		newpath = os.path.normpath(mypath)
-		if len(newpath) > 1:
-			if newpath[1] == "/":
-				newpath = "/"+newpath.lstrip("/")
-		return newpath
 
 	print "Lock 5 starting"
-	import time
 	Lock1=LockDir("/tmp/lock_path")
 	Lock1.write_lock()
 	print "Lock1 write lock"

diff --git a/catalyst/main.py b/catalyst/main.py
index 81d48b5..15664de 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -21,7 +21,8 @@ sys.path.append(__selfpath__ + "/modules")
 from . import __version__
 import catalyst.config
 import catalyst.util
-from catalyst.support import CatalystError, find_binary, LockInUse
+from catalyst.lock import LockInUse
+from catalyst.support import CatalystError, find_binary
 from catalyst.defaults import (confdefaults, option_messages,
 	required_build_targets, valid_build_targets)
 from hash_utils import HashMap, HASH_DEFINITIONS

diff --git a/catalyst/support.py b/catalyst/support.py
index d5dbfec..feaa645 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -115,17 +115,6 @@ class CatalystError(Exception):
 			print "!!! catalyst: "+message
 			print
 
-class LockInUse(Exception):
-	def __init__(self, message):
-		if message:
-			#(type,value)=sys.exc_info()[:2]
-			#if value!=None:
-			    #print
-			    #kprint traceback.print_exc(file=sys.stdout)
-			print
-			print "!!! catalyst lock file in use: "+message
-			print
-
 def die(msg=None):
 	warn(msg)
 	sys.exit(1)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     9944ba53ababcf245d75f5d9c9efce07a4753033
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 23 04:57:05 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan  1 05:58:06 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=9944ba53

reduce 2 operations into one simpler one

---
 catalyst/config.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/catalyst/config.py b/catalyst/config.py
index 460bbd5..8b23342 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -54,11 +54,7 @@ class ParserBase:
 			# Skip any blank lines
 			if not myline: continue
 
-			# Look for separator
-			msearch = myline.find(self.key_value_separator)
-
-			# If separator found assume its a new key
-			if msearch != -1:
+			if self.key_value_separator in myline:
 				# Split on the first occurence of the separator creating two strings in the array mobjs
 				mobjs = myline.split(self.key_value_separator, 1)
 				mobjs[1] = mobjs[1].strip().strip('"')

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     0b3a26d733ea8e8b3fce044be1d2c872138d74d5
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 20 19:22:27 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan  1 05:58:05 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=0b3a26d7

Some spacing, comment and indent cleanup

---
 catalyst/support.py | 405 +++++++++++++++++++++++++++-------------------------
 1 file changed, 213 insertions(+), 192 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index feaa645..e2d64a1 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -1,12 +1,19 @@
 
-import sys,string,os,types,re,signal,traceback,time
-#import md5,sha
+import sys
+import string
+import os
+import types
+import re
+import signal
+import traceback
+import time
 
 from catalyst.defaults import verbosity, valid_config_file_values
 
 selinux_capable = False
 #userpriv_capable = (os.getuid() == 0)
 #fakeroot_capable = False
+
 BASH_BINARY             = "/bin/bash"
 
 # set it to 0 for the soft limit, 1 for the hard limit
@@ -25,35 +32,35 @@ spawned_pids = []
 
 
 def cleanup(pids,block_exceptions=True):
-        """function to go through and reap the list of pids passed to it"""
-        global spawned_pids
-        if type(pids) == int:
-                pids = [pids]
-        for x in pids:
-                try:
-                        os.kill(x,signal.SIGTERM)
-                        if os.waitpid(x,os.WNOHANG)[1] == 0:
-                                # feisty bugger, still alive.
-                                os.kill(x,signal.SIGKILL)
-                                os.waitpid(x,0)
-
-                except OSError, oe:
-                        if block_exceptions:
-                                pass
-                        if oe.errno not in (10,3):
-                                raise oe
-                except SystemExit:
-                        raise
-                except Exception:
-                        if block_exceptions:
-                                pass
-                try:                    spawned_pids.remove(x)
-                except IndexError:      pass
-
-
-
-# a function to turn a string of non-printable characters into a string of
-# hex characters
+	"""function to go through and reap the list of pids passed to it"""
+	global spawned_pids
+	if type(pids) == int:
+		pids = [pids]
+	for x in pids:
+		try:
+			os.kill(x,signal.SIGTERM)
+			if os.waitpid(x,os.WNOHANG)[1] == 0:
+				# feisty bugger, still alive.
+				os.kill(x,signal.SIGKILL)
+				os.waitpid(x,0)
+		except OSError, oe:
+			if block_exceptions:
+				pass
+			if oe.errno not in (10,3):
+				raise oe
+		except SystemExit:
+					raise
+		except Exception:
+			if block_exceptions:
+				pass
+		try:
+			spawned_pids.remove(x)
+		except IndexError:
+			pass
+
+
+# a function to turn a string of non-printable characters
+# into a string of hex characters
 def hexify(str):
 	hexStr = string.hexdigits
 	r = ''
@@ -61,7 +68,6 @@ def hexify(str):
 		i = ord(ch)
 		r = r + hexStr[(i >> 4) & 0xF] + hexStr[i & 0xF]
 	return r
-# hexify()
 
 
 def read_from_clst(file):
@@ -77,7 +83,6 @@ def read_from_clst(file):
 	    myline = myline + line
 	myf.close()
 	return myline
-# read_from_clst
 
 
 def list_bashify(mylist):
@@ -92,6 +97,7 @@ def list_bashify(mylist):
 	mypack=string.join(mypack)
 	return mypack
 
+
 def list_to_string(mylist):
 	if type(mylist)==types.StringType:
 		mypack=[mylist]
@@ -104,6 +110,7 @@ def list_to_string(mylist):
 	mypack=string.join(mypack)
 	return mypack
 
+
 class CatalystError(Exception):
 	def __init__(self, message):
 		if message:
@@ -115,84 +122,83 @@ class CatalystError(Exception):
 			print "!!! catalyst: "+message
 			print
 
+
 def die(msg=None):
 	warn(msg)
 	sys.exit(1)
 
+
 def warn(msg):
 	print "!!! catalyst: "+msg
 
+
 def find_binary(myc):
 	"""look through the environmental path for an executable file named whatever myc is"""
-        # this sucks. badly.
-        p=os.getenv("PATH")
-        if p == None:
-                return None
-        for x in p.split(":"):
-                #if it exists, and is executable
-                if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
-                        return "%s/%s" % (x,myc)
-        return None
+	# this sucks. badly.
+	p=os.getenv("PATH")
+	if p == None:
+		return None
+	for x in p.split(":"):
+		#if it exists, and is executable
+		if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
+			return "%s/%s" % (x,myc)
+	return None
+
 
 def spawn_bash(mycommand,env={},debug=False,opt_name=None,**keywords):
 	"""spawn mycommand as an arguement to bash"""
 	args=[BASH_BINARY]
 	if not opt_name:
-	    opt_name=mycommand.split()[0]
+		opt_name=mycommand.split()[0]
 	if "BASH_ENV" not in env:
-	    env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
+		env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
 	if debug:
-	    args.append("-x")
+		args.append("-x")
 	args.append("-c")
 	args.append(mycommand)
 	return spawn(args,env=env,opt_name=opt_name,**keywords)
 
-#def spawn_get_output(mycommand,spawn_type=spawn,raw_exit_code=False,emulate_gso=True, \
-#        collect_fds=[1],fd_pipes=None,**keywords):
 
 def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \
-        collect_fds=[1],fd_pipes=None,**keywords):
-        """call spawn, collecting the output to fd's specified in collect_fds list
-        emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the
-        requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the
-        'lets let log only stdin and let stderr slide by'.
-
-        emulate_gso was deprecated from the day it was added, so convert your code over.
-        spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot"""
-        global selinux_capable
-        pr,pw=os.pipe()
-
-        #if type(spawn_type) not in [types.FunctionType, types.MethodType]:
-        #        s="spawn_type must be passed a function, not",type(spawn_type),spawn_type
-        #        raise Exception,s
-
-        if fd_pipes==None:
-                fd_pipes={}
-                fd_pipes[0] = 0
-
-        for x in collect_fds:
-                fd_pipes[x] = pw
-        keywords["returnpid"]=True
-
-        mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
-        os.close(pw)
-        if type(mypid) != types.ListType:
-                os.close(pr)
-                return [mypid, "%s: No such file or directory" % mycommand.split()[0]]
-
-        fd=os.fdopen(pr,"r")
-        mydata=fd.readlines()
-        fd.close()
-        if emulate_gso:
-                mydata=string.join(mydata)
-                if len(mydata) and mydata[-1] == "\n":
-                        mydata=mydata[:-1]
-        retval=os.waitpid(mypid[0],0)[1]
-        cleanup(mypid)
-        if raw_exit_code:
-                return [retval,mydata]
-        retval=process_exit_code(retval)
-        return [retval, mydata]
+	collect_fds=[1],fd_pipes=None,**keywords):
+	"""call spawn, collecting the output to fd's specified in collect_fds list
+	emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the
+	requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the
+	'lets let log only stdin and let stderr slide by'.
+
+	emulate_gso was deprecated from the day it was added, so convert your code over.
+	spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot"""
+	global selinux_capable
+	pr,pw=os.pipe()
+
+	if fd_pipes==None:
+			fd_pipes={}
+			fd_pipes[0] = 0
+
+	for x in collect_fds:
+			fd_pipes[x] = pw
+	keywords["returnpid"]=True
+
+	mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
+	os.close(pw)
+	if type(mypid) != types.ListType:
+			os.close(pr)
+			return [mypid, "%s: No such file or directory" % mycommand.split()[0]]
+
+	fd=os.fdopen(pr,"r")
+	mydata=fd.readlines()
+	fd.close()
+	if emulate_gso:
+			mydata=string.join(mydata)
+			if len(mydata) and mydata[-1] == "\n":
+					mydata=mydata[:-1]
+	retval=os.waitpid(mypid[0],0)[1]
+	cleanup(mypid)
+	if raw_exit_code:
+			return [retval,mydata]
+	retval=process_exit_code(retval)
+	return [retval, mydata]
+
 
 # base spawn function
 def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,returnpid=False,\
@@ -230,8 +236,8 @@ def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,retur
 				return None
 			myc = find_binary(myc)
 			if myc == None:
-			    return None
-        mypid=[]
+				return None
+	mypid=[]
 	if logfile:
 		pr,pw=os.pipe()
 		mypid.extend(spawn(('tee','-i','-a',logfile),returnpid=True,fd_pipes={0:pr,1:1,2:2}))
@@ -295,77 +301,77 @@ def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,retur
 			if x not in trg_fd:
 				try:
 					os.close(x)
-                                except SystemExit, e:
-                                        raise
-                                except:
-                                        pass
-
-                # note this order must be preserved- can't change gid/groups if you change uid first.
-                if selinux_capable and selinux_context:
-                        import selinux
-                        selinux.setexec(selinux_context)
-                if gid:
-                        os.setgid(gid)
-                if groups:
-                        os.setgroups(groups)
-                if uid:
-                        os.setuid(uid)
-                if umask:
-                        os.umask(umask)
-                else:
-                        os.umask(022)
-
-                try:
-                        #print "execing", myc, myargs
-                        if func_call:
-                                # either use a passed in func for interpretting the results, or return if no exception.
-                                # note the passed in list, and dict are expanded.
-                                if len(mycommand) == 4:
-                                        os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2])))
-                                try:
-                                        mycommand[0](*mycommand[1],**mycommand[2])
-                                except Exception,e:
-                                        print "caught exception",e," in forked func",mycommand[0]
-                                sys.exit(0)
-
-			#os.execvp(myc,myargs)
-                        os.execve(myc,myargs,env)
-                except SystemExit, e:
-                        raise
-                except Exception, e:
-                        if not func_call:
-                                raise str(e)+":\n   "+myc+" "+string.join(myargs)
-                        print "func call failed"
-
-                # If the execve fails, we need to report it, and exit
-                # *carefully* --- report error here
-                os._exit(1)
-                sys.exit(1)
-                return # should never get reached
-
-        # if we were logging, kill the pipes.
-        if logfile:
-                os.close(pr)
-                os.close(pw)
-
-        if returnpid:
-                return mypid
-
-        # loop through pids (typically one, unless logging), either waiting on their death, or waxing them
-        # if the main pid (mycommand) returned badly.
-        while len(mypid):
-                retval=os.waitpid(mypid[-1],0)[1]
-                if retval != 0:
-                        cleanup(mypid[0:-1],block_exceptions=False)
-                        # at this point we've killed all other kid pids generated via this call.
-                        # return now.
-                        if raw_exit_code:
-                                return retval
-                        return process_exit_code(retval,throw_signals=raise_signals)
-                else:
-                        mypid.pop(-1)
-        cleanup(mypid)
-        return 0
+				except SystemExit, e:
+					raise
+				except:
+					pass
+
+		# note this order must be preserved- can't change gid/groups if you change uid first.
+		if selinux_capable and selinux_context:
+			import selinux
+			selinux.setexec(selinux_context)
+		if gid:
+			os.setgid(gid)
+		if groups:
+			os.setgroups(groups)
+		if uid:
+			os.setuid(uid)
+		if umask:
+			os.umask(umask)
+		else:
+			os.umask(022)
+
+		try:
+			#print "execing", myc, myargs
+			if func_call:
+				# either use a passed in func for interpretting the results, or return if no exception.
+				# note the passed in list, and dict are expanded.
+				if len(mycommand) == 4:
+					os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2])))
+				try:
+					mycommand[0](*mycommand[1],**mycommand[2])
+				except Exception,e:
+					print "caught exception",e," in forked func",mycommand[0]
+				sys.exit(0)
+
+			os.execve(myc,myargs,env)
+		except SystemExit, e:
+			raise
+		except Exception, e:
+			if not func_call:
+				raise str(e)+":\n   "+myc+" "+string.join(myargs)
+			print "func call failed"
+
+		# If the execve fails, we need to report it, and exit
+		# *carefully* --- report error here
+		os._exit(1)
+		sys.exit(1)
+		return # should never get reached
+
+	# if we were logging, kill the pipes.
+	if logfile:
+			os.close(pr)
+			os.close(pw)
+
+	if returnpid:
+			return mypid
+
+	# loop through pids (typically one, unless logging), either waiting on their death, or waxing them
+	# if the main pid (mycommand) returned badly.
+	while len(mypid):
+		retval=os.waitpid(mypid[-1],0)[1]
+		if retval != 0:
+			cleanup(mypid[0:-1],block_exceptions=False)
+			# at this point we've killed all other kid pids generated via this call.
+			# return now.
+			if raw_exit_code:
+				return retval
+			return process_exit_code(retval,throw_signals=raise_signals)
+		else:
+			mypid.pop(-1)
+	cleanup(mypid)
+	return 0
+
 
 def cmd(mycmd,myexc="",env={}):
 	try:
@@ -376,19 +382,21 @@ def cmd(mycmd,myexc="",env={}):
 	except:
 		raise
 
+
 def process_exit_code(retval,throw_signals=False):
-        """process a waitpid returned exit code, returning exit code if it exit'd, or the
-        signal if it died from signalling
-        if throw_signals is on, it raises a SystemExit if the process was signaled.
-        This is intended for usage with threads, although at the moment you can't signal individual
-        threads in python, only the master thread, so it's a questionable option."""
-        if (retval & 0xff)==0:
-                return retval >> 8 # return exit code
-        else:
-                if throw_signals:
-                        #use systemexit, since portage is stupid about exception catching.
-                        raise SystemExit()
-                return (retval & 0xff) << 8 # interrupted by signal
+	"""process a waitpid returned exit code, returning exit code if it exit'd, or the
+	signal if it died from signalling
+	if throw_signals is on, it raises a SystemExit if the process was signaled.
+	This is intended for usage with threads, although at the moment you can't signal individual
+	threads in python, only the master thread, so it's a questionable option."""
+	if (retval & 0xff)==0:
+		return retval >> 8 # return exit code
+	else:
+		if throw_signals:
+			#use systemexit, since portage is stupid about exception catching.
+			raise SystemExit()
+		return (retval & 0xff) << 8 # interrupted by signal
+
 
 def file_locate(settings,filelist,expand=1):
 	#if expand=1, non-absolute paths will be accepted and
@@ -398,15 +406,18 @@ def file_locate(settings,filelist,expand=1):
 			#filenames such as cdtar are optional, so we don't assume the variable is defined.
 			pass
 		else:
-		    if len(settings[myfile])==0:
-			    raise CatalystError, "File variable \""+myfile+"\" has a length of zero (not specified.)"
-		    if settings[myfile][0]=="/":
-			    if not os.path.exists(settings[myfile]):
-				    raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]
-		    elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
-			    settings[myfile]=os.getcwd()+"/"+settings[myfile]
-		    else:
-			    raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]+" (2nd try)"
+			if len(settings[myfile])==0:
+				raise CatalystError("File variable \"" + myfile +
+					"\" has a length of zero (not specified.)")
+			if settings[myfile][0]=="/":
+				if not os.path.exists(settings[myfile]):
+					raise CatalystError("Cannot locate specified " + myfile +
+						": "+settings[myfile])
+			elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
+				settings[myfile]=os.getcwd()+"/"+settings[myfile]
+			else:
+				raise CatalystError("Cannot locate specified " + myfile +
+					": "+settings[myfile]+" (2nd try)" +
 """
 Spec file format:
 
@@ -427,6 +438,8 @@ that the order of multiple-value items is preserved, but the order that the item
 defined are not preserved. In other words, "foo", "bar", "oni" ordering is preserved but "item1"
 "item2" "item3" ordering is not, as the item strings are stored in a dictionary (hash).
 """
+					)
+
 
 def parse_makeconf(mylines):
 	mymakeconf={}
@@ -450,6 +463,7 @@ def parse_makeconf(mylines):
 			    mymakeconf[mobj.group(1)]=clean_string
 	return mymakeconf
 
+
 def read_makeconf(mymakeconffile):
 	if os.path.exists(mymakeconffile):
 		try:
@@ -475,10 +489,12 @@ def read_makeconf(mymakeconffile):
 		makeconf={}
 		return makeconf
 
+
 def msg(mymsg,verblevel=1):
 	if verbosity>=verblevel:
 		print mymsg
 
+
 def pathcompare(path1,path2):
 	# Change double slashes to slash
 	path1 = re.sub(r"//",r"/",path1)
@@ -491,6 +507,7 @@ def pathcompare(path1,path2):
 		return 1
 	return 0
 
+
 def ismount(path):
 	"enhanced to handle bind mounts"
 	if os.path.ismount(path):
@@ -504,6 +521,7 @@ def ismount(path):
 			return 1
 	return 0
 
+
 def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 	"helper function to help targets parse additional arguments"
 	global valid_config_file_values
@@ -522,6 +540,7 @@ def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 	if messages:
 		raise CatalystError, '\n\tAlso: '.join(messages)
 
+
 def touch(myfile):
 	try:
 		myf=open(myfile,"w")
@@ -529,8 +548,9 @@ def touch(myfile):
 	except IOError:
 		raise CatalystError, "Could not touch "+myfile+"."
 
+
 def countdown(secs=5, doing="Starting"):
-        if secs:
+	if secs:
 		print ">>> Waiting",secs,"seconds before starting..."
 		print ">>> (Control-C to abort)...\n"+doing+" in: ",
 		ticks=range(secs)
@@ -541,14 +561,15 @@ def countdown(secs=5, doing="Starting"):
 			time.sleep(1)
 		print
 
+
 def normpath(mypath):
 	TrailingSlash=False
-        if mypath[-1] == "/":
-	    TrailingSlash=True
-        newpath = os.path.normpath(mypath)
-        if len(newpath) > 1:
-                if newpath[:2] == "//":
-                        newpath = newpath[1:]
+	if mypath[-1] == "/":
+		TrailingSlash=True
+	newpath = os.path.normpath(mypath)
+	if len(newpath) > 1:
+		if newpath[:2] == "//":
+			newpath = newpath[1:]
 	if TrailingSlash:
-	    newpath=newpath+'/'
-        return newpath
+		newpath=newpath+'/'
+	return newpath

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     c9f2dd369d57ab4b963753de125fbcad3e3582fc
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 22 08:34:41 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan  1 05:58:06 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=c9f2dd36

FIXME! Comment out a small code block causing TypeError.

This was also short circuiting another large code block.  FIXME!!!! This
whole class seems overly complicated with TOO MANY nested try:excepts:

---
 catalyst/lock.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index 78968cb..c90f5b8 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -27,11 +27,17 @@ class LockDir:
 	locking_method=fcntl.flock
 	lock_dirs_in_use=[]
 	die_on_failed_lock=True
+
 	def __del__(self):
+		#print "Lock.__del__() 1"
 		self.clean_my_hardlocks()
+		#print "Lock.__del__() 2"
 		self.delete_lock_from_path_list()
+		#print "Lock.__del__() 3"
 		if self.islocked():
+			#print "Lock.__del__() 4"
 			self.fcntl_unlock()
+		#print "Lock.__del__() finnished"
 
 	def __init__(self,lockdir):
 		self.locked=False
@@ -179,7 +185,9 @@ class LockDir:
 		unlinkfile = 1
 		if not os.path.exists(self.lockfile):
 			print "lockfile does not exist '%s'" % self.lockfile
+			#print "fcntl_unlock() , self.myfd:", self.myfd, type(self.myfd)
 			if (self.myfd != None):
+				#print "fcntl_unlock() trying to close it "
 				try:
 					os.close(self.myfd)
 					self.myfd=None
@@ -193,11 +201,13 @@ class LockDir:
 					unlinkfile = 1
 					self.locking_method(self.myfd,fcntl.LOCK_UN)
 			except SystemExit, e:
-				raise
+				raise e
 			except Exception, e:
-				os.close(self.myfd)
-				self.myfd=None
-				raise IOError, "Failed to unlock file '%s'\n" % self.lockfile
+				#if self.myfd is not None:
+					#print "fcntl_unlock() trying to close", self.myfd
+					#os.close(self.myfd)
+					#self.myfd=None
+				#raise IOError, "Failed to unlock file '%s'\n%s" % (self.lockfile, str(e))
 				try:
 					# This sleep call was added to allow other processes that are
 					# waiting for a lock to be able to grab it before it is deleted.
@@ -224,7 +234,7 @@ class LockDir:
 #						if "DEBUG" in self.settings:
 #							print "Unlinked lockfile..."
 				except SystemExit, e:
-					raise
+					raise e
 				except Exception, e:
 					# We really don't care... Someone else has the lock.
 					# So it is their problem now.
@@ -236,7 +246,7 @@ class LockDir:
 					# open fd closed automatically on them.
 					#if type(lockfilename) == types.StringType:
 					#        os.close(myfd)
-
+		#print "fcntl_unlock() trying a last ditch close", self.myfd
 		if (self.myfd != None):
 			os.close(self.myfd)
 			self.myfd=None
@@ -421,6 +431,7 @@ class LockDir:
 						pass
 		return results
 
+
 if __name__ == "__main__":
 
 	def lock_work():

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     fa577bb409c36c3b38a90f65cb7a665c5474d2c9
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 25 04:00:24 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan  1 05:58:06 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=fa577bb4

Extend ParserBase to do variable substitution.

Also add embedded variable substitiution to default settings.
Why are we not using python's built-in ConfigParser?
We'll migrate it later.

---
 catalyst/config.py | 3 +++
 catalyst/main.py   | 6 +++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/catalyst/config.py b/catalyst/config.py
index 8b23342..bf3c7c6 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -66,6 +66,9 @@ class ParserBase:
 				# Start a new array using the first element of mobjs
 				cur_array = [mobjs[0]]
 				if mobjs[1]:
+					# do any variable substitiution embeded in it with
+					# the values already obtained
+					mobjs[1] = mobjs[1] % values
 					if self.multiple_values:
 						# split on white space creating additional array elements
 #						subarray = white_space.split(mobjs[1])

diff --git a/catalyst/main.py b/catalyst/main.py
index 1446cf9..5fa3f1b 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -116,7 +116,6 @@ def parse_config(myconfig):
 	# add our python base directory to use for loading target arch's
 	conf_values["PythonDir"] = __selfpath__
 
-
 	# print out any options messages
 	for opt in conf_values['options']:
 		if opt in option_messages:
@@ -133,6 +132,11 @@ def parse_config(myconfig):
 	if "envscript" in myconf:
 		print "Envscript support enabled."
 
+	# take care of any variable substitutions that may be left
+	for x in list(conf_values):
+		if isinstance(conf_values[x], str):
+			conf_values[x] = conf_values[x] % conf_values
+
 
 def import_module(target):
 	"""

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     89beddf2ccfb320bac1f8d5b2e5964bf4890ac78
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu May 30 19:34:06 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan  1 05:58:07 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=89beddf2

Migrate version to use snakeoil's format_version() to append git commit info.

This will make tagging releases easy as well as providing better debug info while running live versions of the software.

---
 catalyst/main.py    | 11 +++++++----
 catalyst/version.py | 23 +++++++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 7fea4e7..ae0abae 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -14,7 +14,6 @@ import os.path
 __selfpath__ = os.path.abspath(os.path.dirname(__file__))
 
 
-from . import __version__
 import catalyst.config
 import catalyst.util
 from catalyst.contents import ContentsMap, CONTENTS_DEFINITIONS
@@ -22,6 +21,7 @@ from catalyst.defaults import confdefaults, option_messages
 from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
 from catalyst.lock import LockInUse
 from catalyst.support import CatalystError, find_binary
+from catalyst.version import get_version
 
 
 conf_values={}
@@ -57,7 +57,7 @@ catalyst -f stage1-specfile.spec
 
 
 def version():
-	print "Catalyst, version "+__version__
+	print get_version()
 	print "Copyright 2003-2008 Gentoo Foundation"
 	print "Copyright 2008-2012 various authors"
 	print "Distributed under the GNU General Public License version 2.1\n"
@@ -169,8 +169,8 @@ def build_target(addlargs):
 
 def main():
 
-	version()
 	if os.getuid() != 0:
+		version()
 		# catalyst cannot be run as a normal user due to chroots, mounts, etc
 		print "!!! catalyst: This script requires root privileges to operate"
 		sys.exit(2)
@@ -204,11 +204,12 @@ def main():
 	run = False
 	for o, a in opts:
 		if o in ("-h", "--help"):
+			version()
 			usage()
 			sys.exit(1)
 
 		if o in ("-V", "--version"):
-			print "Catalyst version "+__version__
+			print get_version()
 			sys.exit(1)
 
 		if o in ("-d", "--debug"):
@@ -264,6 +265,8 @@ def main():
 		usage()
 		sys.exit(2)
 
+	# made it this far so start by outputting our version info
+	version()
 	# import configuration file and import our main module using those settings
 	parse_config(myconfig)
 

diff --git a/catalyst/version.py b/catalyst/version.py
new file mode 100644
index 0000000..03c77e4
--- /dev/null
+++ b/catalyst/version.py
@@ -0,0 +1,23 @@
+#!/usr/bin/python -OO
+
+# Maintained in full by:
+# Catalyst Team <catalyst@gentoo.org>
+# Release Engineering Team <releng@gentoo.org>
+# Copyright: 2011 Brian Harring <ferringb@gmail.com>
+# License: BSD/GPL2
+# Copied & edited by: Brian Dolbec <dolsen@gentoo.org>
+
+'''Version information and/or git version information
+'''
+
+from snakeoil.version import format_version
+
+__version__="rewrite-git"
+_ver = None
+
+def get_version():
+	"""Return: a string describing our version."""
+	global _ver
+	if _ver is None:
+		_ver = format_version('catalyst',__file__, __version__)
+	return _ver

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     32df09409b29b93bb89f930452a4f0ad52e8fcc5
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jun 11 16:07:03 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Feb 26 19:15:21 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=32df0940

fix '-' in stage names for python imports

---
 catalyst/main.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index ae0abae..fc1681c 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -153,11 +153,12 @@ def import_module(target):
 
 def build_target(addlargs):
 	try:
-		module = import_module(addlargs["target"])
-		target = getattr(module, addlargs["target"])(conf_values, addlargs)
+		target = addlargs["target"].replace('-', '_')
+		module = import_module(target)
+		target = getattr(module, target)(conf_values, addlargs)
 	except AttributeError:
 		raise CatalystError(
-			"Target \"%s\" not available." % addlargs["target"],
+			"Target \"%s\" not available." % target,
 			print_traceback=True)
 
 	try:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     83c4337f59102245d6018810db5bb79f471b002f
Author:     W. Trevor King <wking <AT> tremily <DOT> us>
AuthorDate: Tue Jun 11 21:41:18 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Feb 26 19:56:51 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=83c4337f

version: Remove shebang (Python modules are not scripts)

Nobody should be executing this file directly.

---
 catalyst/version.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index 03c77e4..ea12761 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python -OO
-
 # Maintained in full by:
 # Catalyst Team <catalyst@gentoo.org>
 # Release Engineering Team <releng@gentoo.org>

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     5a09558cbbeca6fda075525103a43d44e5cd5871
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Jun 12 04:07:08 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Feb 26 19:57:22 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=5a09558c

main.py, lock.py, fileops.py  remove shebangs

---
 catalyst/fileops.py | 1 -
 catalyst/lock.py    | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 009bf1c..e3a4ead 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -1,4 +1,3 @@
-#!/usr/bin/python
 
 # Maintained in full by:
 # Catalyst Team <catalyst@gentoo.org>

diff --git a/catalyst/lock.py b/catalyst/lock.py
index 12d0754..a6f9d3a 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -1,4 +1,5 @@
-#!/usr/bin/python
+
+
 import os
 import fcntl
 import errno

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     24ec1400aa8c1b9aedae9a498e172d63cfe3705d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 28 03:58:10 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu May 21 15:28:18 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=24ec1400

catalyst/support.py: Fix some leading whitespace rebase errors

Change a CatalystError to not print a traceback.

 catalyst/support.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 7957f08..2ac4816 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -152,17 +152,17 @@ def file_locate(settings,filelist,expand=1):
 			#filenames such as cdtar are optional, so we don't assume the variable is defined.
 			pass
 		else:
-		    if len(settings[myfile])==0:
-			    raise CatalystError("File variable \"" + myfile +
+			if len(settings[myfile])==0:
+				raise CatalystError("File variable \"" + myfile +
 					"\" has a length of zero (not specified.)", print_traceback=True)
-		    if settings[myfile][0]=="/":
-			    if not os.path.exists(settings[myfile]):
-				    raise CatalystError("Cannot locate specified " + myfile +
-						": "+settings[myfile], print_traceback=True)
-		    elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
-			    settings[myfile]=os.getcwd()+"/"+settings[myfile]
-		    else:
-			    raise CatalystError("Cannot locate specified " + myfile +
+			if settings[myfile][0]=="/":
+				if not os.path.exists(settings[myfile]):
+					raise CatalystError("Cannot locate specified " + myfile +
+						": " + settings[myfile], print_traceback=False)
+			elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
+				settings[myfile]=os.getcwd()+"/"+settings[myfile]
+			else:
+				raise CatalystError("Cannot locate specified " + myfile +
 					": "+settings[myfile]+" (2nd try)" +
 """
 Spec file format:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     a1cb9a08ad98a046e1bedc8f3418894691923174
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 29 14:39:10 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Aug 29 14:39:22 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=a1cb9a08

Adjust for the snakeoil api change

 catalyst/version.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index 8060bff..8c0b267 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -10,7 +10,7 @@
 
 import os
 
-from snakeoil.version import format_version
+from snakeoil.version import get_version as get_ver
 
 __version__="rewrite-git"
 _ver = None
@@ -19,7 +19,7 @@ _ver = None
 def get_git_version(version=__version__):
 	"""Return: a string describing our version."""
 	global _ver
-	_ver = format_version('catalyst',__file__, version)
+	_ver = get_ver('catalyst-%s' % version, __file__)
 	return _ver
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     83d1bd1a2b01e3b89caca3ff48e4308176c208f0
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 29 16:10:36 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Aug 29 16:10:36 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=83d1bd1a

version.py: Copy what was needed of the old format_version code to fix versioning

The new get_version code from snakeoil is not compatible with teh rest of
the catlyst code.  It will need a lot more conversion to use the new system.
I do think it could be better once completed.

 catalyst/version.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index 8c0b267..c274a11 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -10,16 +10,28 @@
 
 import os
 
-from snakeoil.version import get_version as get_ver
+from snakeoil.version import get_git_version as get_ver
 
-__version__="rewrite-git"
+__version__="git"
 _ver = None
 
 
 def get_git_version(version=__version__):
 	"""Return: a string describing our version."""
 	global _ver
-	_ver = get_ver('catalyst-%s' % version, __file__)
+	cwd = os.path.dirname(os.path.abspath(__file__))
+	version_info = get_ver(cwd)
+
+	if not version_info:
+		s = "extended version info unavailable"
+	elif version_info['tag'] == api_version:
+		s = 'released %s' % (version_info['date'],)
+	else:
+		s = ('vcs version %s, date %s' %
+			 (version_info['rev'], version_info['date']))
+
+	_ver = '%s %s\n%s' % (project, api_version, s)
+
 	return _ver
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     98ffc8dc8d236ba18a992c632cc29a1935757057
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 29 16:17:38 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Aug 29 16:20:13 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=98ffc8dc

version.py: Fix missed edits

 catalyst/version.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index c274a11..d5c5cde 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__="git"
+__version__= "git"
 _ver = None
 
 
@@ -24,13 +24,13 @@ def get_git_version(version=__version__):
 
 	if not version_info:
 		s = "extended version info unavailable"
-	elif version_info['tag'] == api_version:
+	elif version_info['tag'] == __version__:
 		s = 'released %s' % (version_info['date'],)
 	else:
 		s = ('vcs version %s, date %s' %
 			 (version_info['rev'], version_info['date']))
 
-	_ver = '%s %s\n%s' % (project, api_version, s)
+	_ver = 'Catalyst %s\n%s' % (__version__, s)
 
 	return _ver
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     348e9759220ef02cefaa80e5b763a32572b15bc3
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Aug 30 02:15:20 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Aug 30 02:15:20 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=348e9759

catalyst/config.py: Fix an error for x is an integer

 catalyst/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/config.py b/catalyst/config.py
index c952648..67d4e77 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -85,7 +85,7 @@ class ParserBase:
 #					cur_array += mobjs
 					cur_array += myline.split()
 				else:
-					raise CatalystError("Syntax error: " + x, print_traceback=True)
+					raise CatalystError("Syntax error: %s" % x, print_traceback=True)
 
 			# XXX: Do we really still need this "single value is a string" behavior?
 			if len(cur_array) == 2:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     4328dcd693ae02a77c519e5186e5430171c16ac5
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Aug 30 20:55:52 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Aug 30 20:56:09 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4328dcd6

catalyst/main.py: Fix variables not being substituted with correct values

 catalyst/main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index a222209..2335495 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -107,8 +107,8 @@ def parse_config(myconfig):
 			else:
 				conf_values[x]=myconf[x]
 		else:
-			print "Setting",x,"to default value \""+confdefaults[x]+"\""
-			conf_values[x]=confdefaults[x]
+			conf_values[x] = confdefaults[x] % confdefaults
+			print "Setting",x,"to default value \"" + conf_values[x] + "\""
 
 	# add our python base directory to use for loading target arch's
 	conf_values["PythonDir"] = __selfpath__

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Richard Farina]

commit:     f499da75fe51ed8ae2830a20fef5a27bce633173
Author:     Rick Farina (Zero_Chaos) <zerochaos <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 31 03:15:50 2015 +0000
Commit:     Richard Farina <zerochaos <AT> gentoo <DOT> org>
CommitDate: Mon Aug 31 03:15:50 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f499da75

Revert "catalyst/main.py: Fix variables not being substituted with correct values"

This reverts commit 4328dcd693ae02a77c519e5186e5430171c16ac5.

Reverting due to :

Envscript support enabled.
Traceback (most recent call last):
  File "/usr/lib/python-exec/python2.7/catalyst", line 39, in <module>
    main()
  File "/usr/lib64/python2.7/site-packages/catalyst/main.py", line 343, in main
    build_target(addlargs)
  File "/usr/lib64/python2.7/site-packages/catalyst/main.py", line 158, in build_target
    target = getattr(module, target)(conf_values, addlargs)
  File "/usr/lib64/python2.7/site-packages/catalyst/targets/stage1.py", line 21, in __init__
    StageBase.__init__(self,spec,addlargs)
  File "/usr/lib64/python2.7/site-packages/catalyst/base/stagebase.py", line 72, in __init__
    for x in [x[:-3] for x in os.listdir(arch_dir) if x.endswith(".py") and x != "__init__.py"]:
OSError: [Errno 2] No such file or directory: './catalyst/arch/'

 catalyst/main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 2335495..a222209 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -107,8 +107,8 @@ def parse_config(myconfig):
 			else:
 				conf_values[x]=myconf[x]
 		else:
-			conf_values[x] = confdefaults[x] % confdefaults
-			print "Setting",x,"to default value \"" + conf_values[x] + "\""
+			print "Setting",x,"to default value \""+confdefaults[x]+"\""
+			conf_values[x]=confdefaults[x]
 
 	# add our python base directory to use for loading target arch's
 	conf_values["PythonDir"] = __selfpath__

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     b0e860c4b5c218729d344a821814071b4392ed61
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Sep  1 04:48:45 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Sep  1 04:49:39 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b0e860c4

main.py: Add 'local_overlay' to the overridable defaults in the catalyst.conf file

 catalyst/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index dd3c517..3ab4166 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -116,7 +116,8 @@ def parse_config(myconfig):
 		if opt in option_messages:
 			print option_messages[opt]
 
-	for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir"]:
+	for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir",
+				"local_overlay"]:
 		if key in myconf:
 			conf_values[key] = myconf[key]
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     7a036cf0fb918d95498d84577463db35c2eccc97
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Sep  1 04:47:45 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Sep  1 04:49:14 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=7a036cf0

main.py: Remove 2 debug prints causing misleading information

In this part if the code, these inital defaults are not yet updated.
The valuse are updated later in the code.

 catalyst/main.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index a222209..dd3c517 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -101,13 +101,11 @@ def parse_config(myconfig):
 	# now, load up the values into conf_values so that we can use them
 	for x in list(confdefaults):
 		if x in myconf:
-			print "Setting",x,"to config file value \""+myconf[x]+"\""
 			if x == 'options':
 				conf_values[x] = set(myconf[x].split())
 			else:
 				conf_values[x]=myconf[x]
 		else:
-			print "Setting",x,"to default value \""+confdefaults[x]+"\""
 			conf_values[x]=confdefaults[x]
 
 	# add our python base directory to use for loading target arch's

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     8035e2073b3bb7ca3125627a9671b63239f0537f
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Sep  1 04:39:38 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Sep  1 04:40:55 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=8035e207

fileops.py: Fix a bug where the mode was not being passed 

Change the default mode to 0o755

 catalyst/fileops.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index fd98db2..129d967 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -21,7 +21,7 @@ from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs,
 from catalyst.support import CatalystError
 
 
-def ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True,
+def ensure_dirs(path, gid=-1, uid=-1, mode=0o755, minimal=True,
 		failback=None, fatal=False):
 	'''Wrapper to snakeoil.osutil's ensure_dirs()
 	This additionally allows for failures to run
@@ -40,7 +40,7 @@ def ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True,
 	:return: True if the directory could be created/ensured to have those
 		permissions, False if not.
 	'''
-	succeeded = snakeoil_ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True)
+	succeeded = snakeoil_ensure_dirs(path, gid=-1, uid=-1, mode=mode, minimal=True)
 	if not succeeded:
 		if failback:
 			failback()
@@ -50,7 +50,7 @@ def ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True,
 	return succeeded
 
 
-def clear_dir(target, mode=0755, chg_flags=False, remove=False):
+def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
 	'''Universal directory clearing function
 
 	@target: string, path to be cleared or removed

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     1039b4beef11da6dfd8d54b1ab3724b1e817bcdf
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Sep  5 16:23:30 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Sep  6 21:18:00 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1039b4be

defaults.py: Update for the new xattr variations of the definitions

This should allow to not define the decompressor_search_order in the spec file.

 catalyst/defaults.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 5ce2858..5db7ab6 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -1,5 +1,8 @@
 
 
+from DeComp.definitions import DECOMPRESSOR_XATTR_SEARCH_ORDER
+
+
 # these should never be touched
 required_build_targets = ["targetbase", "generic_stage_target"]
 
@@ -23,8 +26,8 @@ verbosity = 1
 
 confdefaults={
 	"archdir": "%(PythonDir)s/arch",
-	"compression_mode": 'lbzip2',
-	"decompressor_search_order": 'lbzip2 bzip2 tar pixz xz gzip squashfs',
+	"compression_mode": 'lbzip2_x',
+	"decompressor_search_order": DECOMPRESSOR_XATTR_SEARCH_ORDER,
 	"distdir": "/usr/portage/distfiles",
 	"hash_function": "crc32",
 	"icecream": "/var/cache/icecream",

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     8ebde1b8598e34caa83f8bcae6185ee4f6e38264
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun May 24 07:38:32 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Sep  6 21:18:00 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=8ebde1b8

catalyst/main.py: Comment out the try: except: pair for target.run()

 catalyst/main.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 0ba81b9..7e3b55e 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -163,12 +163,12 @@ def build_target(addlargs):
 			"Target \"%s\" not available." % target,
 			print_traceback=True)
 
-	try:
-		target.run()
-	except:
-		print "!!! catalyst: Error encountered during run of target " + \
-			addlargs["target"]
-		sys.exit(1)
+	#try:
+	target.run()
+	#except:
+		#print "!!! catalyst: Error encountered during run of target " + \
+		#	addlargs["target"]
+		#sys.exit(1)
 
 def main():
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     ce64d85fdd92920cae60cd93039f6cae35de51ad
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Sep  8 06:13:30 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Sep  8 14:11:48 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ce64d85f

main.py:  Add an exception for CatalystError which would have been already handled 

Cleanup some dead code.

 catalyst/main.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index dc9342b..2a5bdd3 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -163,13 +163,10 @@ def build_target(addlargs):
 		raise CatalystError(
 			"Target \"%s\" not available." % target,
 			print_traceback=True)
-
-	#try:
+	except CatalystError:
+		return
 	target.run()
-	#except:
-		#print "!!! catalyst: Error encountered during run of target " + \
-		#	addlargs["target"]
-		#sys.exit(1)
+
 
 def main():
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     ce11f7e38d2e309359a6bf8963f4f7a0569fdc52
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 03:29:19 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 03:29:19 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ce11f7e3

lock: fix hardlink_active definition

This is a member function of a class, so it must have a |self| argument
first.  The current code is clearly broken otherwise.  Not that it seems
to matter as there aren't any callers in the current codebase.

 catalyst/lock.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index a6f9d3a..9f954eb 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -354,7 +354,7 @@ class LockDir:
 				return True
 		return False
 
-	def hardlink_active(lock):
+	def hardlink_active(self, lock):
 		if not os.path.exists(lock):
 			return False
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     6c2de744bc8eb511c937c7735d144bd5d40063ed
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 03:42:11 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 03:42:11 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=6c2de744

lint: stop using old-style classes

Everyone has moved away from these in the python world.

 catalyst/builder.py | 2 +-
 catalyst/config.py  | 2 +-
 catalyst/lock.py    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/catalyst/builder.py b/catalyst/builder.py
index ad27d78..3ba86c8 100644
--- a/catalyst/builder.py
+++ b/catalyst/builder.py
@@ -1,5 +1,5 @@
 
-class generic:
+class generic(object):
 	def __init__(self,myspec):
 		self.settings=myspec
 

diff --git a/catalyst/config.py b/catalyst/config.py
index 5d74dce..48d9c12 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -2,7 +2,7 @@
 import re
 from catalyst.support import CatalystError
 
-class ParserBase:
+class ParserBase(object):
 
 	filename = ""
 	lines = None

diff --git a/catalyst/lock.py b/catalyst/lock.py
index 9f954eb..c031258 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -24,7 +24,7 @@ class LockInUse(Exception):
 			print
 
 
-class LockDir:
+class LockDir(object):
 	locking_method=fcntl.flock
 	lock_dirs_in_use=[]
 	die_on_failed_lock=True

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     ecbe93c9794d714e6fb05f3c06a30699667371fd
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 03:27:01 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 03:27:01 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ecbe93c9

lint: fix bad env dict handling

By using a kwarg default of {}, the value is retained across multiple
calls (and the linter warns about it).  Use the standard "if None"
style to avoid that.

Also fix the write to the dict passed in by creating a local copy
before we insert BASH_ENV into it.

 catalyst/support.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index b6705c9..78942a7 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -120,11 +120,14 @@ def find_binary(myc):
 	return None
 
 
-def cmd(mycmd, myexc="", env={}, debug=False, fail_func=None):
+def cmd(mycmd, myexc="", env=None, debug=False, fail_func=None):
+	if env is None:
+		env = {}
 	#print "***** cmd()"
 	sys.stdout.flush()
 	args=[BASH_BINARY]
 	if "BASH_ENV" not in env:
+		env = env.copy()
 		env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
 	if debug:
 		args.append("-x")

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     f4e85c25e4092b5616db44b24a34986731a21d7a
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 03:23:45 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 03:23:45 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f4e85c25

lint: fix anomalous-backslash-in-string

All strings using escape sequences should be raw strings.
This way we don't end up with accidental misbehavior.

 catalyst/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/config.py b/catalyst/config.py
index 67d4e77..5d74dce 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -42,7 +42,7 @@ class ParserBase:
 		values = {}
 		cur_array = []
 
-		trailing_comment=re.compile('\s*#.*$')
+		trailing_comment=re.compile(r'\s*#.*$')
 		#white_space=re.compile('\s+')
 
 		for x, myline in enumerate(self.lines):

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     50994b728212ae42d3f16f7081d972aa405ac008
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 03:59:48 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 03:59:48 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=50994b72

lock: rewrite delete_lock_from_path_list

This code looks like it attempts to remove self.lockdir from the
lock_dirs_in_use list, but it doesn't seem to quite work:
 - it uses "for x in lock_dirs_in_use", but then indexes lock_dirs_in_use
   via the "i" variable
 - the "i" variable is set to 0, but then never incremented (looks like
   due to incorrect indentation)
 - the only way an entry would be deleted is if it happened to be in the
   first location of the list

In the end, this looks like an ad-hoc implementation of the list .remove
builtin, so just delete the code and use that.

 catalyst/lock.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index c031258..25d2aa2 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -57,15 +57,9 @@ class LockDir(object):
 		self.hardlock_paths={}
 
 	def delete_lock_from_path_list(self):
-		i=0
 		try:
-			if LockDir.lock_dirs_in_use:
-				for x in LockDir.lock_dirs_in_use:
-					if LockDir.lock_dirs_in_use[i] == self.lockdir:
-						del LockDir.lock_dirs_in_use[i]
-						break
-						i=i+1
-		except AttributeError:
+			LockDir.lock_dirs_in_use.remove(self.lockdir)
+		except ValueError:
 			pass
 
 	def islocked(self):

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     b71beb16153c2fddb4972fb4f727e8b6e9fbcf8c
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 13:46:32 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 13:46:32 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b71beb16

lint: fix duplicate fcntl import

This module is already imported at the top, so no point in doing it again.

 catalyst/lock.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index 25d2aa2..71918d6 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -176,7 +176,6 @@ class LockDir(object):
 			#writemsg("Lockfile obtained\n")
 
 	def fcntl_unlock(self):
-		import fcntl
 		unlinkfile = 1
 		if not os.path.exists(self.lockfile):
 			print "lockfile does not exist '%s'" % self.lockfile

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     4bc5a844fd340a3bd9b842678702fa6abf1b6b18
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 14:12:05 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 14:12:05 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4bc5a844

fileops: fix passing of gid/uid/minimal args

This func accepted these kwargs but forgot to pass them along to the
snakeoil layer.

 catalyst/fileops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 8a05985..5a1d0f3 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -42,7 +42,7 @@ def ensure_dirs(path, gid=-1, uid=-1, mode=0o755, minimal=True,
 	:return: True if the directory could be created/ensured to have those
 		permissions, False if not.
 	'''
-	succeeded = snakeoil_ensure_dirs(path, gid=-1, uid=-1, mode=mode, minimal=True)
+	succeeded = snakeoil_ensure_dirs(path, gid=gid, uid=uid, mode=mode, minimal=minimal)
 	if not succeeded:
 		if failback:
 			failback()

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     d0a0de7291f68220daded2dee8cb6e8fbbc2fee0
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 14:37:09 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 14:37:09 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d0a0de72

version: use the passed in value

The get_git_version function takes a |version| argument, but doesn't
actually use it, which makes calls to the function with a version do
nothing.  Tweak the function to use the arg in the way it looks like
it intended.

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index 5deb4d7..3b82988 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -31,7 +31,7 @@ def get_git_version(version=__version__):
 		s = ('vcs version %s, date %s' %
 			 (version_info['rev'], version_info['date']))
 
-	_ver = 'Catalyst %s\n%s' % (__version__, s)
+	_ver = 'Catalyst %s\n%s' % (version, s)
 
 	return _ver
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     49437c0c128d8632aa15d1d7bbfbb0e0a1f81905
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 14:07:44 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 14:07:44 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=49437c0c

lint: avoid relative imports

We import from catalyst.xxx elsewhere, so be consistent.

 catalyst/contents.py   | 2 +-
 catalyst/hash_utils.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/catalyst/contents.py b/catalyst/contents.py
index a06b2db..73eda61 100644
--- a/catalyst/contents.py
+++ b/catalyst/contents.py
@@ -2,7 +2,7 @@
 from collections import namedtuple
 from subprocess import Popen, PIPE
 
-from support import CatalystError, warn
+from catalyst.support import CatalystError, warn
 
 
 # use ContentsMap.fields for the value legend

diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py
index 39f489b..0262422 100644
--- a/catalyst/hash_utils.py
+++ b/catalyst/hash_utils.py
@@ -3,7 +3,7 @@ import os
 from collections import namedtuple
 from subprocess import Popen, PIPE
 
-from support import CatalystError
+from catalyst.support import CatalystError
 
 
 # Use HashMap.fields for the value legend

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     24b8bcbb89103b77865a7b21cc94450d25c4bc7c
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 16:00:34 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 16:00:34 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=24b8bcbb

main: delay root check until before we run

This way we can run things like --help and verify argument/config parsing
all as non-root.  Only actual building requires root.

 catalyst/main.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 4e83414..604c6ab 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -168,13 +168,6 @@ def build_target(addlargs):
 
 
 def main():
-
-	if os.getuid() != 0:
-		version()
-		# catalyst cannot be run as a normal user due to chroots, mounts, etc
-		print "!!! catalyst: This script requires root privileges to operate"
-		sys.exit(2)
-
 	# we need some options in order to work correctly
 	if len(sys.argv) < 2:
 		usage()
@@ -342,6 +335,11 @@ def main():
 	if "target" not in addlargs:
 		raise CatalystError("Required value \"target\" not specified.")
 
+	if os.getuid() != 0:
+		# catalyst cannot be run as a normal user due to chroots, mounts, etc
+		print "!!! catalyst: This script requires root privileges to operate"
+		sys.exit(2)
+
 	# everything is setup, so the build is a go
 	try:
 		success = build_target(addlargs)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     5d42738fb733ecb3fb92b69bb85f17a36bef60b7
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 16:33:50 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 16:33:50 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5d42738f

main: switch to argparse

Switch from ad-hoc getopt parsing to the more powerful/standard argparse.

 catalyst/main.py | 191 +++++++++++++++++++++++++------------------------------
 1 file changed, 85 insertions(+), 106 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 604c6ab..2a25df4 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -6,9 +6,9 @@
 # Chris Gianelloni <wolf31o2@wolf31o2.org>
 # $Id$
 
+import argparse
 import os
 import sys
-import getopt
 import os.path
 
 __selfpath__ = os.path.abspath(os.path.dirname(__file__))
@@ -28,35 +28,6 @@ from catalyst.version import get_version
 conf_values={}
 
 
-def usage():
-	print """Usage catalyst [options] [-C variable=value...] [ -s identifier]
- -a --clear-autoresume  clear autoresume flags
- -c --config            use specified configuration file
- -C --cli               catalyst commandline (MUST BE LAST OPTION)
- -d --debug             enable debugging
- -f --file              read specfile
- -F --fetchonly         fetch files only
- -h --help              print this help message
- -p --purge             clear tmp dirs,package cache, autoresume flags
- -P --purgeonly         clear tmp dirs,package cache, autoresume flags and exit
- -T --purgetmponly      clear tmp dirs and autoresume flags and exit
- -s --snapshot          generate a release snapshot
- -V --version           display version information
- -v --verbose           verbose output
-
-Usage examples:
-
-Using the commandline option (-C, --cli) to build a Portage snapshot:
-catalyst -C target=snapshot version_stamp=my_date
-
-Using the snapshot option (-s, --snapshot) to build a release snapshot:
-catalyst -s 20071121"
-
-Using the specfile option (-f, --file) to build a stage target:
-catalyst -f stage1-specfile.spec
-"""
-
-
 def version():
 	print get_version()
 	print "Copyright 2003-2008 Gentoo Foundation"
@@ -167,96 +138,104 @@ def build_target(addlargs):
 	return target.run()
 
 
-def main():
-	# we need some options in order to work correctly
-	if len(sys.argv) < 2:
-		usage()
-		sys.exit(2)
-
-	# parse out the command line arguments
-	try:
-		opts, _args = getopt.getopt(sys.argv[1:], "apPThvdc:C:f:FVs:", ["purge", "purgeonly", "purgetmponly", "help", "version", "debug",
-			"clear-autoresume", "config=", "cli=", "file=", "fetch", "verbose","snapshot="])
-
-	except getopt.GetoptError:
-		usage()
-		sys.exit(2)
-
-	myconfig=""
-	myspecfile=""
-	mycmdline=[]
-
-	# check preconditions
-	if len(opts) == 0:
-		print "!!! catalyst: please specify one of either -f or -C\n"
-		usage()
-		sys.exit(2)
-
-	options = set()
-
-	run = False
-	for o, a in opts:
-		if o in ("-h", "--help"):
-			version()
-			usage()
-			sys.exit(1)
+class FilePath(object):
+	"""Argparse type for getting a path to a file."""
 
-		if o in ("-V", "--version"):
-			print get_version()
-			sys.exit(1)
+	def __init__(self, exists=True):
+		self.exists = exists
 
-		if o in ("-d", "--debug"):
-			conf_values["DEBUG"] = True
-			conf_values["VERBOSE"] = True
+	def __call__(self, string):
+		if not os.path.exists(string):
+			raise argparse.ArgumentTypeError('file does not exist: %s' % string)
+		return string
 
-		if o in ("-c", "--config"):
-			myconfig=a
+	def __repr__(self):
+		return '%s(exists=%s)' % (type(self).__name__, self.exists)
 
-		if o in ("-C", "--cli"):
-			run = True
-			x=sys.argv.index(o)+1
-			while x < len(sys.argv):
-				mycmdline.append(sys.argv[x])
-				x=x+1
 
-		if o in ("-f", "--file"):
-			run = True
-			myspecfile=a
+def get_parser():
+	"""Return an argument parser"""
+	epilog = """Usage examples:
 
-		if o in ("-F", "--fetchonly"):
-			options.add("fetch")
+Using the commandline option (-C, --cli) to build a Portage snapshot:
+$ catalyst -C target=snapshot version_stamp=my_date
 
-		if o in ("-v", "--verbose"):
-			conf_values["VERBOSE"]="1"
+Using the snapshot option (-s, --snapshot) to build a release snapshot:
+$ catalyst -s 20071121
 
-		if o in ("-s", "--snapshot"):
-			if len(sys.argv) < 3:
-				print "!!! catalyst: missing snapshot identifier\n"
-				usage()
-				sys.exit(2)
-			else:
-				run = True
-				mycmdline.append("target=snapshot")
-				mycmdline.append("version_stamp="+a)
+Using the specfile option (-f, --file) to build a stage target:
+$ catalyst -f stage1-specfile.spec"""
+
+	parser = argparse.ArgumentParser(epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
+	parser.add_argument('-d', '--debug',
+		default=False, action='store_true',
+		help='enable debugging')
+	parser.add_argument('-v', '--verbose',
+		default=False, action='store_true',
+		help='verbose output')
+	parser.add_argument('-c', '--config',
+		type=FilePath(),
+		help='use specified configuration file')
+	parser.add_argument('-f', '--file',
+		type=FilePath(),
+		help='read specfile')
+	parser.add_argument('-F', '--fetchonly',
+		default=False, action='store_true',
+		help='fetch files only')
+	parser.add_argument('-a', '--clear-autoresume',
+		default=False, action='store_true',
+		help='clear autoresume flags')
+	parser.add_argument('-p', '--purge',
+		default=False, action='store_true',
+		help='clear tmp dirs, package cache, autoresume flags')
+	parser.add_argument('-P', '--purgeonly',
+		default=False, action='store_true',
+		help='clear tmp dirs, package cache, autoresume flags and exit')
+	parser.add_argument('-T', '--purgetmponly',
+		default=False, action='store_true',
+		help='clear tmp dirs and autoresume flags and exit')
+	parser.add_argument('-s', '--snapshot',
+		help='generate a release snapshot')
+	parser.add_argument('-V', '--version',
+		action='version', version=get_version(),
+		help='display version information')
+	parser.add_argument('-C', '--cli',
+		default=[], nargs=argparse.REMAINDER,
+		help='catalyst commandline (MUST BE LAST OPTION)')
+	return parser
 
-		if o in ("-p", "--purge"):
-			options.add("purge")
 
-		if o in ("-P", "--purgeonly"):
-			options.add("purgeonly")
+def main():
+	parser = get_parser()
+	opts = parser.parse_args(sys.argv[1:])
 
-		if o in ("-T", "--purgetmponly"):
-			options.add("purgetmponly")
+	# Parse the command line options.
+	myconfig = opts.config
+	myspecfile = opts.file
+	mycmdline = opts.cli[:]
 
-		if o in ("-a", "--clear-autoresume"):
-			options.add("clear-autoresume")
+	if opts.snapshot:
+		mycmdline.append('target=snapshot')
+		mycmdline.append('version_stamp=' + opts.snapshot)
 
-	#print "MAIN: cli options =", options
+	conf_values['DEBUG'] = opts.debug
+	conf_values['VERBOSE'] = opts.debug or opts.verbose
 
-	if not run:
-		print "!!! catalyst: please specify one of either -f or -C\n"
-		usage()
-		sys.exit(2)
+	options = set()
+	if opts.fetchonly:
+		options.add('fetch')
+	if opts.purge:
+		options.add('purge')
+	if opts.purgeonly:
+		options.add('purgeonly')
+	if opts.purgetmponly:
+		options.add('purgetmponly')
+	if opts.clear_autoresume:
+		options.add('clear-autoresume')
+
+	# Make sure we have some work before moving further.
+	if not myspecfile and not mycmdline:
+		parser.error('please specify one of either -f or -C or -s')
 
 	# made it this far so start by outputting our version info
 	version()

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     5c93c16d1df9ab1512ec4444aa994f97c44c831d
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 19:49:29 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 19:49:29 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5c93c16d

main: drop pointless os.path import

You can import os and then use os.path directly, so there's no need to
import this thing.

 catalyst/main.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 2a25df4..ebec4d3 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -9,7 +9,6 @@
 import argparse
 import os
 import sys
-import os.path
 
 __selfpath__ = os.path.abspath(os.path.dirname(__file__))
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     12b695116a39a64188e8e9af8501f7d6154fcf46
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 21:41:55 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Thu Oct  8 22:07:57 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=12b69511

support: delete unused code

Using the vulture tool (and verifying a bit with `git grep`),
delete various bits of code from this module that aren't used.

 catalyst/support.py | 28 ----------------------------
 1 file changed, 28 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index f184ed7..62be63a 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -1,7 +1,6 @@
 
 import glob
 import sys
-import string  # pylint: disable=deprecated-module
 import os
 import types
 import re
@@ -11,35 +10,8 @@ from subprocess import Popen
 
 from catalyst.defaults import verbosity, valid_config_file_values
 
-selinux_capable = False
-#userpriv_capable = (os.getuid() == 0)
-#fakeroot_capable = False
-
 BASH_BINARY             = "/bin/bash"
 
-# set it to 0 for the soft limit, 1 for the hard limit
-DESIRED_RLIMIT = 0
-try:
-	import resource
-	max_fd_limit=resource.getrlimit(resource.RLIMIT_NOFILE)[DESIRED_RLIMIT]
-except Exception:
-	# hokay, no resource module.
-	max_fd_limit=256
-
-# pids this process knows of.
-spawned_pids = []
-
-
-# a function to turn a string of non-printable characters
-# into a string of hex characters
-def hexify(s):
-	hexStr = string.hexdigits
-	r = ''
-	for ch in s:
-		i = ord(ch)
-		r = r + hexStr[(i >> 4) & 0xF] + hexStr[i & 0xF]
-	return r
-
 
 def read_from_clst(path):
 	line = ''

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     206b5aafefcd4aaf25df6abae88963889639ff72
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 22:22:29 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Thu Oct  8 22:07:57 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=206b5aaf

main: use find_binary from snakeoil

No point in duplicating our own version when snakeoil provides it.

 catalyst/main.py    | 12 +++++++++---
 catalyst/support.py | 13 -------------
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 65e34ef..03c13c0 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -10,6 +10,8 @@ import argparse
 import os
 import sys
 
+from snakeoil import process
+
 __selfpath__ = os.path.abspath(os.path.dirname(__file__))
 
 from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
@@ -20,7 +22,7 @@ import catalyst.config
 import catalyst.util
 from catalyst.defaults import confdefaults, option_messages
 from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
-from catalyst.support import CatalystError, find_binary
+from catalyst.support import CatalystError
 from catalyst.version import get_version
 
 
@@ -283,7 +285,9 @@ def main():
 
 		# Then check for any programs that the hash func requires.
 		for digest in digests:
-			if find_binary(hash_map.hash_map[digest].cmd) == None:
+			try:
+				process.find_binary(hash_map.hash_map[digest].cmd)
+			except process.CommandNotFound:
 				# In auto mode, just ignore missing support.
 				if skip_missing:
 					digests.remove(digest)
@@ -309,7 +313,9 @@ def main():
 			print
 			print "Catalyst aborting...."
 			sys.exit(2)
-		if find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd) == None:
+		try:
+			process.find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd)
+		except process.CommandNotFound:
 			print
 			print "hash_function="+conf_values["hash_function"]
 			print "\tThe "+hash_map.hash_map[conf_values["hash_function"]].cmd + \

diff --git a/catalyst/support.py b/catalyst/support.py
index a26e49c..1207d36 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -64,19 +64,6 @@ def warn(msg):
 	print "!!! catalyst: "+msg
 
 
-def find_binary(myc):
-	"""look through the environmental path for an executable file named whatever myc is"""
-	# this sucks. badly.
-	p=os.getenv("PATH")
-	if p == None:
-		return None
-	for x in p.split(":"):
-		#if it exists, and is executable
-		if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
-			return "%s/%s" % (x,myc)
-	return None
-
-
 def cmd(mycmd, myexc="", env=None, debug=False, fail_func=None):
 	if env is None:
 		env = {}

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     fd78adf1f2155a76af380a5294954b253f4e1053
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Fri Oct  9 05:37:56 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Oct  9 05:37:56 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=fd78adf1

main: convert to new logging module

 catalyst/main.py | 104 +++++++++++++++++++++++--------------------------------
 1 file changed, 43 insertions(+), 61 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index adae9a6..f33369b 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -7,6 +7,7 @@
 # $Id$
 
 import argparse
+import datetime
 import os
 import sys
 
@@ -31,36 +32,36 @@ conf_values={}
 
 
 def version():
-	print get_version()
-	print "Copyright 2003-2008 Gentoo Foundation"
-	print "Copyright 2008-2012 various authors"
-	print "Distributed under the GNU General Public License version 2.1\n"
+	log.info(get_version())
+	log.info('Copyright 2003-%s Gentoo Foundation', datetime.datetime.now().year)
+	log.info('Copyright 2008-2012 various authors')
+	log.info('Distributed under the GNU General Public License version 2.1')
 
 def parse_config(myconfig):
 	# search a couple of different areas for the main config file
 	myconf={}
 	config_file=""
+	default_config_file = '/etc/catalyst/catalyst.conf'
 
 	# first, try the one passed (presumably from the cmdline)
 	if myconfig:
 		if os.path.exists(myconfig):
-			print "Using command line specified Catalyst configuration file, "+myconfig
+			log.notice('Using command line specified Catalyst configuration file: %s',
+				myconfig)
 			config_file=myconfig
 
 		else:
-			print "!!! catalyst: Could not use specified configuration file "+\
-				myconfig
-			sys.exit(1)
+			log.critical('Specified configuration file does not exist: %s', myconfig)
 
 	# next, try the default location
-	elif os.path.exists("/etc/catalyst/catalyst.conf"):
-		print "Using default Catalyst configuration file, /etc/catalyst/catalyst.conf"
-		config_file="/etc/catalyst/catalyst.conf"
+	elif os.path.exists(default_config_file):
+		log.notice('Using default Catalyst configuration file: %s',
+			default_config_file)
+		config_file = default_config_file
 
 	# can't find a config file (we are screwed), so bail out
 	else:
-		print "!!! catalyst: Could not find a suitable configuration file"
-		sys.exit(1)
+		log.critical('Could not find a suitable configuration file')
 
 	# now, try and parse the config file "config_file"
 	try:
@@ -69,8 +70,7 @@ def parse_config(myconfig):
 		myconf.update(myconfig.get_values())
 
 	except Exception:
-		print "!!! catalyst: Unable to parse configuration file, "+myconfig
-		sys.exit(1)
+		log.critical('Could not find parse configuration file: %s', myconfig)
 
 	# now, load up the values into conf_values so that we can use them
 	for x in list(confdefaults):
@@ -90,7 +90,7 @@ def parse_config(myconfig):
 	# print out any options messages
 	for opt in conf_values['options']:
 		if opt in option_messages:
-			print option_messages[opt]
+			log.info(option_messages[opt])
 
 	for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir",
 				"local_overlay"]:
@@ -102,7 +102,7 @@ def parse_config(myconfig):
 		conf_values["contents"] = myconf["contents"].replace("-", '_')
 
 	if "envscript" in myconf:
-		print "Envscript support enabled."
+		log.info('Envscript support enabled.')
 
 	# take care of any variable substitutions that may be left
 	for x in list(conf_values):
@@ -118,11 +118,8 @@ def import_module(target):
 	try:
 		mod_name = "catalyst.targets." + target
 		module = __import__(mod_name, [],[], ["not empty"])
-	except ImportError as e:
-		print "!!! catalyst: Python module import error: %s " % target + \
-			"in catalyst/targets/ ... exiting."
-		print "ERROR was: ", e
-		sys.exit(1)
+	except ImportError:
+		log.critical('Python module import error: %s', target, exc_info=True)
 	return module
 
 
@@ -278,7 +275,7 @@ def main():
 	parse_config(myconfig)
 
 	conf_values["options"].update(options)
-	#print "MAIN: conf_values['options'] =", conf_values["options"]
+	log.debug('conf_values[options] = %s', conf_values['options'])
 
 	# initialize our contents generator
 	contents_map = ContentsMap(CONTENTS_DEFINITIONS)
@@ -308,14 +305,13 @@ def main():
 
 		# First validate all the requested digests are valid keys.
 		if digests - valid_digests:
-			print
-			print "These are not a valid digest entries:"
-			print ', '.join(digests - valid_digests)
-			print "Valid digest entries:"
-			print ', '.join(sorted(valid_digests))
-			print
-			print "Catalyst aborting...."
-			sys.exit(2)
+			log.critical(
+				'These are not valid digest entries:\n'
+				'%s\n'
+				'Valid digest entries:\n'
+				'%s',
+				', '.join(digests - valid_digests),
+				', '.join(sorted(valid_digests)))
 
 		# Then check for any programs that the hash func requires.
 		for digest in digests:
@@ -326,37 +322,28 @@ def main():
 				if skip_missing:
 					digests.remove(digest)
 					continue
-				print
-				print "digest=" + digest
-				print "\tThe " + hash_map.hash_map[digest].cmd + \
-					" binary was not found. It needs to be in your system path"
-				print
-				print "Catalyst aborting...."
-				sys.exit(2)
+				log.critical(
+					'The "%s" binary needed by digest "%s" was not found. '
+					'It needs to be in your system path.',
+					hash_map.hash_map[digest].cmd, digest)
 
 		# Now reload the config with our updated value.
 		conf_values['digests'] = ' '.join(digests)
 
 	if "hash_function" in conf_values:
 		if conf_values["hash_function"] not in HASH_DEFINITIONS:
-			print
-			print conf_values["hash_function"]+\
-				" is not a valid hash_function entry"
-			print "Valid hash_function entries:"
-			print HASH_DEFINITIONS.keys()
-			print
-			print "Catalyst aborting...."
-			sys.exit(2)
+			log.critical(
+				'%s is not a valid hash_function entry\n'
+				'Valid hash_function entries:\n'
+				'%s', HASH_DEFINITIONS.keys())
 		try:
 			process.find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd)
 		except process.CommandNotFound:
-			print
-			print "hash_function="+conf_values["hash_function"]
-			print "\tThe "+hash_map.hash_map[conf_values["hash_function"]].cmd + \
-				" binary was not found. It needs to be in your system path"
-			print
-			print "Catalyst aborting...."
-			sys.exit(2)
+			log.critical(
+				'The "%s" binary needed by hash_function "%s" was not found. '
+				'It needs to be in your system path.',
+				hash_map.hash_map[conf_values['hash_function']].cmd,
+				conf_values['hash_function'])
 
 	addlargs={}
 
@@ -370,25 +357,20 @@ def main():
 			cmdline.parse_lines(mycmdline)
 			addlargs.update(cmdline.get_values())
 		except CatalystError:
-			print "!!! catalyst: Could not parse commandline, exiting."
-			sys.exit(1)
+			log.critical('Could not parse commandline')
 
 	if "target" not in addlargs:
 		raise CatalystError("Required value \"target\" not specified.")
 
 	if os.getuid() != 0:
 		# catalyst cannot be run as a normal user due to chroots, mounts, etc
-		print "!!! catalyst: This script requires root privileges to operate"
-		sys.exit(2)
+		log.critical('This script requires root privileges to operate')
 
 	# everything is setup, so the build is a go
 	try:
 		success = build_target(addlargs)
 	except KeyboardInterrupt:
-		print "\nCatalyst build aborted due to user interrupt ( Ctrl-C )"
-		print
-		print "Catalyst aborting...."
-		sys.exit(2)
+		log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
 	if not success:
 		sys.exit(2)
 	sys.exit(0)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     0c931533d9ad82c4c724e7de25db07851cea2f66
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Fri Oct  9 03:31:03 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Oct  9 03:31:03 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=0c931533

main: group related command line flags

This makes the --help output more manageable so people can quickly
scan and skip options that they don't care about.

 catalyst/main.py | 48 ++++++++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 03c13c0..e6b6447 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -168,41 +168,49 @@ Using the specfile option (-f, --file) to build a stage target:
 $ catalyst -f stage1-specfile.spec"""
 
 	parser = argparse.ArgumentParser(epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
-	parser.add_argument('-d', '--debug',
+
+	parser.add_argument('-V', '--version',
+		action='version', version=get_version(),
+		help='display version information')
+
+	group = parser.add_argument_group('Program output options')
+	group.add_argument('-d', '--debug',
 		default=False, action='store_true',
 		help='enable debugging')
-	parser.add_argument('-v', '--verbose',
+	group.add_argument('-v', '--verbose',
 		default=False, action='store_true',
 		help='verbose output')
-	parser.add_argument('-c', '--config',
-		type=FilePath(),
-		help='use specified configuration file')
-	parser.add_argument('-f', '--file',
-		type=FilePath(),
-		help='read specfile')
-	parser.add_argument('-F', '--fetchonly',
-		default=False, action='store_true',
-		help='fetch files only')
-	parser.add_argument('-a', '--clear-autoresume',
+
+	group = parser.add_argument_group('Temporary file management')
+	group.add_argument('-a', '--clear-autoresume',
 		default=False, action='store_true',
 		help='clear autoresume flags')
-	parser.add_argument('-p', '--purge',
+	group.add_argument('-p', '--purge',
 		default=False, action='store_true',
 		help='clear tmp dirs, package cache, autoresume flags')
-	parser.add_argument('-P', '--purgeonly',
+	group.add_argument('-P', '--purgeonly',
 		default=False, action='store_true',
 		help='clear tmp dirs, package cache, autoresume flags and exit')
-	parser.add_argument('-T', '--purgetmponly',
+	group.add_argument('-T', '--purgetmponly',
 		default=False, action='store_true',
 		help='clear tmp dirs and autoresume flags and exit')
-	parser.add_argument('-s', '--snapshot',
+
+	group = parser.add_argument_group('Target/config file management')
+	group.add_argument('-F', '--fetchonly',
+		default=False, action='store_true',
+		help='fetch files only')
+	group.add_argument('-c', '--config',
+		type=FilePath(),
+		help='use specified configuration file')
+	group.add_argument('-f', '--file',
+		type=FilePath(),
+		help='read specfile')
+	group.add_argument('-s', '--snapshot',
 		help='generate a release snapshot')
-	parser.add_argument('-V', '--version',
-		action='version', version=get_version(),
-		help='display version information')
-	parser.add_argument('-C', '--cli',
+	group.add_argument('-C', '--cli',
 		default=[], nargs=argparse.REMAINDER,
 		help='catalyst commandline (MUST BE LAST OPTION)')
+
 	return parser
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     3c5ef2e76dd5c57b64a160fbfc4b441569d1401e
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Fri Oct  9 05:29:27 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Oct  9 05:29:27 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3c5ef2e7

log: new logging module to standardize catalyst output

This has everything you could ever want:
- control where output is sent (stdout or a file)
- control over log level
- automatic exit when CRITICAL is used
- automatic colorization of critical/error/warning messages
- explicit control over use of colors
- automatic handling of embedded newlines
- standardized output format
- all logging routed through a single logger (not the root)
- additional log level between warning & info: notice

This just lays the groundwork -- no code is actually converted over
to using this.  That will be done in follow up commit(s).

Note: The default output level has changed from "info" to "notice".
That means the default display won't spam w/irrelevant details.

Example output (only the main.py module is converted):
$ ./bin/wrapper.py -s test
09 Oct 2015 01:34:56 EDT: NOTICE  : Using default Catalyst configuration file: /etc/catalyst/catalyst.conf
ContentsMap: __init__(), search_order = ['pixz', 'lbzip2', 'isoinfo_l', 'squashfs', 'gzip', 'xz', 'bzip2', 'tar', 'isoinfo_f']
Creating Portage tree snapshot test from /usr/portage...
^C

$ ./bin/wrapper.py -s test --debug
09 Oct 2015 01:35:43 EDT: INFO    : main.py:version: Catalyst git
09 Oct 2015 01:35:43 EDT: INFO    : main.py:version: vcs version 4440e8908c677d8764e29b0f127e2dd6c02b7621, date Fri Oct 9 01:28:19 2015 -0400
09 Oct 2015 01:35:43 EDT: INFO    : main.py:version: Copyright 2003-2015 Gentoo Foundation
09 Oct 2015 01:35:43 EDT: INFO    : main.py:version: Copyright 2008-2012 various authors
09 Oct 2015 01:35:43 EDT: INFO    : main.py:version: Distributed under the GNU General Public License version 2.1
09 Oct 2015 01:35:43 EDT: NOTICE  : log.py:notice: Using default Catalyst configuration file: /etc/catalyst/catalyst.conf
09 Oct 2015 01:35:43 EDT: INFO    : main.py:parse_config: Snapshot cache support enabled.
09 Oct 2015 01:35:43 EDT: INFO    : main.py:parse_config: Kernel cache support enabled.
09 Oct 2015 01:35:43 EDT: INFO    : main.py:parse_config: Autoresuming support enabled.
09 Oct 2015 01:35:43 EDT: INFO    : main.py:parse_config: Package cache support enabled.
09 Oct 2015 01:35:43 EDT: INFO    : main.py:parse_config: Seed cache support enabled.
09 Oct 2015 01:35:43 EDT: INFO    : main.py:parse_config: Envscript support enabled.
09 Oct 2015 01:35:43 EDT: DEBUG   : main.py:main: conf_values[options] = set(['snapcache', 'kerncache', 'autoresume', 'pkgcache', 'bindist', 'seedcache'])
^C

$ ./bin/wrapper.py -s test -C /asdf
09 Oct 2015 01:36:59 EDT: NOTICE  : Using default Catalyst configuration file: /etc/catalyst/catalyst.conf
ContentsMap: __init__(), search_order = ['pixz', 'lbzip2', 'isoinfo_l', 'squashfs', 'gzip', 'xz', 'bzip2', 'tar', 'isoinfo_f']

!!! catalyst: Syntax error: 0

09 Oct 2015 01:36:59 EDT: CRITICAL: Could not parse commandline

 catalyst/log.py  | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 catalyst/main.py |  30 ++++++++++++-
 2 files changed, 155 insertions(+), 2 deletions(-)

diff --git a/catalyst/log.py b/catalyst/log.py
new file mode 100644
index 0000000..871c53c
--- /dev/null
+++ b/catalyst/log.py
@@ -0,0 +1,127 @@
+# Copyright 2003-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+"""Logging related code
+
+This largely exposes the same interface as the logging module except we add
+another level "notice" between warning & info, and all output goes through
+the "catalyst" logger.
+"""
+
+from __future__ import print_function
+
+import logging
+import logging.handlers
+import os
+import sys
+import time
+
+
+class CatalystLogger(logging.Logger):
+	"""Override the _log member to autosplit on new lines"""
+
+	def _log(self, level, msg, args, **kwargs):
+		"""If given a multiline message, split it"""
+		# We have to interpolate it first in case they spread things out
+		# over multiple lines like: Bad Thing:\n%s\nGoodbye!
+		msg %= args
+		for line in msg.splitlines():
+			super(CatalystLogger, self)._log(level, line, (), **kwargs)
+
+
+# The logger that all output should go through.
+# This is ugly because we want to not perturb the logging module state.
+_klass = logging.getLoggerClass()
+logging.setLoggerClass(CatalystLogger)
+logger = logging.getLogger('catalyst')
+logging.setLoggerClass(_klass)
+del _klass
+
+
+# Set the notice level between warning and info.
+NOTICE = (logging.WARNING + logging.INFO) / 2
+logging.addLevelName(NOTICE, 'NOTICE')
+
+
+# The API we expose to consumers.
+def notice(msg, *args, **kwargs):
+	"""Log a notice message"""
+	logger.log(NOTICE, msg, *args, **kwargs)
+
+def critical(msg, *args, **kwargs):
+	"""Log a critical message and then exit"""
+	status = kwargs.pop('status', 1)
+	logger.critical(msg, *args, **kwargs)
+	sys.exit(status)
+
+error = logger.error
+warning = logger.warning
+info = logger.info
+debug = logger.debug
+
+
+class CatalystFormatter(logging.Formatter):
+	"""Mark bad messages with colors automatically"""
+
+	_COLORS = {
+		'CRITICAL':	'\033[1;35m',
+		'ERROR':	'\033[1;31m',
+		'WARNING':	'\033[1;33m',
+		'DEBUG':	'\033[1;34m',
+	}
+	_NORMAL = '\033[0m'
+
+	@staticmethod
+	def detect_color():
+		"""Figure out whether the runtime env wants color"""
+		if 'NOCOLOR' is os.environ:
+			return False
+		return os.isatty(sys.stdout.fileno())
+
+	def __init__(self, *args, **kwargs):
+		"""Initialize"""
+		color = kwargs.pop('color', None)
+		if color is None:
+			color = self.detect_color()
+		if not color:
+			self._COLORS = {}
+
+		super(CatalystFormatter, self).__init__(*args, **kwargs)
+
+	def format(self, record, **kwargs):
+		"""Format the |record| with our color settings"""
+		msg = super(CatalystFormatter, self).format(record, **kwargs)
+		color = self._COLORS.get(record.levelname)
+		if color:
+			return color + msg + self._NORMAL
+		else:
+			return msg
+
+
+def setup_logging(level, output=None, debug=False, color=None):
+	"""Initialize the logging module using the |level| level"""
+	# The incoming level will be things like "info", but setLevel wants
+	# the numeric constant.  Convert it here.
+	level = logging.getLevelName(level.upper())
+
+	# The good stuff.
+	fmt = '%(asctime)s: %(levelname)-8s: '
+	if debug:
+		fmt += '%(filename)s:%(funcName)s: '
+	fmt += '%(message)s'
+
+	# Figure out where to send the log output.
+	if output is None:
+		handler = logging.StreamHandler(stream=sys.stdout)
+	else:
+		handler = logging.FileHandler(output)
+
+	# Use a date format that is readable by humans & machines.
+	# Think e-mail/RFC 2822: 05 Oct 2013 18:58:50 EST
+	tzname = time.strftime('%Z', time.localtime())
+	datefmt = '%d %b %Y %H:%M:%S ' + tzname
+	formatter = CatalystFormatter(fmt, datefmt, color=color)
+	handler.setFormatter(formatter)
+
+	logger.addHandler(handler)
+	logger.setLevel(level)

diff --git a/catalyst/main.py b/catalyst/main.py
index e6b6447..adae9a6 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -18,6 +18,7 @@ from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
 	CONTENTS_DEFINITIONS)
 from DeComp.contents import ContentsMap
 
+from catalyst import log
 import catalyst.config
 import catalyst.util
 from catalyst.defaults import confdefaults, option_messages
@@ -176,10 +177,23 @@ $ catalyst -f stage1-specfile.spec"""
 	group = parser.add_argument_group('Program output options')
 	group.add_argument('-d', '--debug',
 		default=False, action='store_true',
-		help='enable debugging')
+		help='enable debugging (and default --log-level debug)')
 	group.add_argument('-v', '--verbose',
 		default=False, action='store_true',
-		help='verbose output')
+		help='verbose output (and default --log-level info)')
+	group.add_argument('--log-level',
+		default=None,
+		choices=('critical', 'error', 'warning', 'notice', 'info', 'debug'),
+		help='set verbosity of output (default: notice)')
+	group.add_argument('--log-file',
+		type=FilePath(exists=False),
+		help='write all output to this file (instead of stdout)')
+	group.add_argument('--color',
+		default=None, action='store_true',
+		help='colorize output all the time (default: detect)')
+	group.add_argument('--nocolor',
+		dest='color', action='store_false',
+		help='never colorize output all the time (default: detect)')
 
 	group = parser.add_argument_group('Temporary file management')
 	group.add_argument('-a', '--clear-autoresume',
@@ -218,6 +232,18 @@ def main():
 	parser = get_parser()
 	opts = parser.parse_args(sys.argv[1:])
 
+	# Initialize the logger before anything else.
+	log_level = opts.log_level
+	if log_level is None:
+		if opts.debug:
+			log_level = 'debug'
+		elif opts.verbose:
+			log_level = 'info'
+		else:
+			log_level = 'notice'
+	log.setup_logging(log_level, output=opts.log_file, debug=opts.debug,
+		color=opts.color)
+
 	# Parse the command line options.
 	myconfig = opts.config
 	myspecfile = opts.file

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     8d350367ff0eeb920d58e8e82e76cf81e9e9cc08
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Fri Oct  9 13:21:21 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Oct  9 13:21:21 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=8d350367

util: drop unused module

The last user of this module was removed in a519c380faabe8278863d4615e24
back in 2013.

 catalyst/main.py |  1 -
 catalyst/util.py | 14 --------------
 2 files changed, 15 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index f33369b..9f563cf 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -21,7 +21,6 @@ from DeComp.contents import ContentsMap
 
 from catalyst import log
 import catalyst.config
-import catalyst.util
 from catalyst.defaults import confdefaults, option_messages
 from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
 from catalyst.support import CatalystError

diff --git a/catalyst/util.py b/catalyst/util.py
deleted file mode 100644
index ff12086..0000000
--- a/catalyst/util.py
+++ /dev/null
@@ -1,14 +0,0 @@
-"""
-Collection of utility functions for catalyst
-"""
-
-import sys, traceback
-
-def capture_traceback():
-	etype, value, tb = sys.exc_info()
-	s = [x.strip() for x in traceback.format_exception(etype, value, tb)]
-	return s
-
-def print_traceback():
-	for x in capture_traceback():
-		print x

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     0a9a9e359e9677e2ebea5a23fe3f0179ae392217
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Fri Oct  9 05:26:51 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Oct  9 05:26:51 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=0a9a9e35

contents: punt unused module

It looks like this code has all been moved to the external DeComp pkg.

 catalyst/contents.py | 87 ----------------------------------------------------
 1 file changed, 87 deletions(-)

diff --git a/catalyst/contents.py b/catalyst/contents.py
deleted file mode 100644
index 73eda61..0000000
--- a/catalyst/contents.py
+++ /dev/null
@@ -1,87 +0,0 @@
-
-from collections import namedtuple
-from subprocess import Popen, PIPE
-
-from catalyst.support import CatalystError, warn
-
-
-# use ContentsMap.fields for the value legend
-# Key:[function, cmd]
-CONTENTS_DEFINITIONS = {
-	# 'find' is disabled because it requires the source path, which is not
-	# always available
-	#"find"		:["calc_contents","find %(path)s"],
-	"tar_tv":["calc_contents","tar --xattrs tvf %(file)s"],
-	"tar_tvz":["calc_contents","tar --xattrs tvzf %(file)s"],
-	"tar_tvj":["calc_contents","tar --xattrs -I lbzip2 -tvf %(file)s"],
-	"isoinfo_l":["calc_contents","isoinfo -l -i %(file)s"],
-	# isoinfo_f should be a last resort only
-	"isoinfo_f":["calc_contents","isoinfo -f -i %(file)s"],
-}
-
-
-class ContentsMap(object):
-	'''Class to encompass all known commands to list
-	the contents of an archive'''
-
-
-	fields = ['func', 'cmd']
-
-
-	def __init__(self, defs=None):
-		'''Class init
-
-		@param defs: dictionary of Key:[function, cmd]
-		'''
-		if defs is None:
-			defs = {}
-		#self.contents = {}
-		self.contents_map = {}
-
-		# create the archive type namedtuple classes
-		for name in list(defs):
-			#obj = self.contents[name] = namedtuple(name, self.fields)
-			obj = namedtuple(name, self.fields)
-			obj.__slots__ = ()
-			self.contents_map[name] = obj._make(defs[name])
-		del obj
-
-
-	def generate_contents(self, file_, getter="auto", verbose=False):
-		try:
-			archive = getter
-			if archive == 'auto' and file_.endswith('.iso'):
-				archive = 'isoinfo_l'
-			if (archive in ['tar_tv','auto']):
-				if file_.endswith('.tgz') or file_.endswith('.tar.gz'):
-					archive = 'tar_tvz'
-				elif file_.endswith('.tbz2') or file_.endswith('.tar.bz2'):
-					archive = 'tar_tvj'
-				elif file_.endswith('.tar'):
-					archive = 'tar_tv'
-
-			if archive == 'auto':
-				warn('File %r has unknown type for automatic detection.'
-					% (file_, ))
-				return None
-			else:
-				getter = archive
-				func = getattr(self, '_%s_' % self.contents_map[getter].func)
-				return func(file_, self.contents_map[getter].cmd, verbose)
-		except:
-			raise CatalystError(
-				"Error generating contents, is appropriate utility " +
-				"(%s) installed on your system?"
-				% (self.contents_map[getter].cmd), print_traceback=True)
-
-
-	@staticmethod
-	def _calc_contents_(file_, cmd, verbose):
-		_cmd = (cmd % {'file': file_ }).split()
-		proc = Popen(_cmd, stdout=PIPE, stderr=PIPE)
-		results = proc.communicate()
-		result = "\n".join(results)
-		if verbose:
-			print result
-		return result
-

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     761dbbd96691eb2c3897741e92acdc2ee7cd70e1
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Fri Oct  9 15:39:21 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Oct  9 15:39:21 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=761dbbd9

fileops: convert to log module

 catalyst/fileops.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 2aa39f6..8fb2a36 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -20,6 +20,8 @@ from stat import ST_UID, ST_GID, ST_MODE
 from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs,
 	pjoin, listdir_files)
 # pylint: enable=unused-import
+
+from catalyst import log
 from catalyst.support import CatalystError
 
 
@@ -61,31 +63,31 @@ def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
 	@remove: boolean, passed through to clear_dir()
 	@return boolean
 	'''
-	#print "fileops.clear_dir()"
+	log.debug('start: %s', target)
 	if not target:
-		#print "fileops.clear_dir(), no target... returning"
+		log.debug('no target... returning')
 		return False
 	if os.path.isdir(target):
-		print "Emptying directory" , target
+		log.info('Emptying directory: %s', target)
 		# stat the dir, delete the dir, recreate the dir and set
 		# the proper perms and ownership
 		try:
-			#print "fileops.clear_dir(), os.stat()"
+			log.debug('os.stat()')
 			mystat=os.stat(target)
 			# There's no easy way to change flags recursively in python
 			if chg_flags and os.uname()[0] == "FreeBSD":
 				os.system("chflags -R noschg " + target)
-			#print "fileops.clear_dir(), shutil.rmtree()"
+			log.debug('shutil.rmtree()')
 			shutil.rmtree(target)
 			if not remove:
-				#print "fileops.clear_dir(), ensure_dirs()"
+				log.debug('ensure_dirs()')
 				ensure_dirs(target, mode=mode)
 				os.chown(target, mystat[ST_UID], mystat[ST_GID])
 				os.chmod(target, mystat[ST_MODE])
-		except Exception as e:
-			print CatalystError("clear_dir(); Exeption: %s" % str(e))
+		except Exception:
+			log.error('clear_dir failed', exc_info=True)
 			return False
 	else:
-		print "fileops.clear_dir(), %s is not a directory" % (target)
-	#print "fileops.clear_dir(), DONE, returning True"
+		log.info('clear_dir failed: %s: is not a directory', target)
+	log.debug('DONE, returning True')
 	return True

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     23d13a16870b5d3f5064e5ccd8b622b540260516
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Fri Oct  9 15:39:45 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Oct  9 15:39:45 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=23d13a16

config: convert to log module

 catalyst/config.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/catalyst/config.py b/catalyst/config.py
index ffad9b3..db81a96 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -1,5 +1,7 @@
 
 import re
+
+from catalyst import log
 from catalyst.support import CatalystError
 
 class ParserBase(object):
@@ -98,7 +100,7 @@ class ParserBase(object):
 			for x in values.keys():
 				# Delete empty key pairs
 				if not values[x]:
-					print "\n\tWARNING: No value set for key " + x + "...deleting"
+					log.warning('No value set for key "%s"; deleting', x)
 					del values[x]
 
 		self.values = values

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     f083637554bf5668ec856c56cfaaa76bb343d941
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 10 01:51:23 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sat Oct 10 23:54:36 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f0836375

lock: gut & replace with snakeoil

The hardlink logic is unused, so start by deleting all of that.
If someone wants that (multiple builds on NFS?), we can look at
restoring it.

 catalyst/lock.py | 467 ++-----------------------------------------------------
 1 file changed, 15 insertions(+), 452 deletions(-)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index 8095a82..39926dd 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -1,467 +1,30 @@
 
-
 import os
-import fcntl
-import errno
-import sys
-import time
-from catalyst.support import CatalystError, normpath
 
-def writemsg(mystr):
-	sys.stderr.write(mystr)
-	sys.stderr.flush()
+from snakeoil import fileutils
+from snakeoil import osutils
 
 
-class LockInUse(Exception):
-	def __init__(self, message):
-		if message:
-			#(type,value)=sys.exc_info()[:2]
-			#if value!=None:
-			    #print
-			    #kprint traceback.print_exc(file=sys.stdout)
-			print
-			print "!!! catalyst lock file in use: "+message
-			print
+LockInUse = osutils.LockException
 
 
 class LockDir(object):
-	locking_method=fcntl.flock
-	lock_dirs_in_use=[]
-	die_on_failed_lock=True
-
-	def __del__(self):
-		#print "Lock.__del__() 1"
-		self.clean_my_hardlocks()
-		#print "Lock.__del__() 2"
-		self.delete_lock_from_path_list()
-		#print "Lock.__del__() 3"
-		if self.islocked():
-			#print "Lock.__del__() 4"
-			self.fcntl_unlock()
-		#print "Lock.__del__() finnished"
-
-	def __init__(self,lockdir):
-		self.locked=False
-		self.myfd=None
-		self.set_gid(250)
-		self.locking_method=LockDir.locking_method
-		self.set_lockdir(lockdir)
-		self.set_lockfilename(".catalyst_lock")
-		self.set_lockfile()
-
-		if LockDir.lock_dirs_in_use.count(lockdir)>0:
-			raise "This directory already associated with a lock object"
-		else:
-			LockDir.lock_dirs_in_use.append(lockdir)
-
-		self.myhardlock = None
-		self.hardlock_paths={}
-
-	def delete_lock_from_path_list(self):
-		try:
-			LockDir.lock_dirs_in_use.remove(self.lockdir)
-		except ValueError:
-			pass
-
-	def islocked(self):
-		if self.locked:
-			return True
-		else:
-			return False
+	"""An object that creates locks inside dirs"""
 
-	def set_gid(self,gid):
-		if not self.islocked():
-#			if self.settings["DEBUG"]:
-#				print "setting gid to", gid
-			self.gid=gid
-
-	def set_lockdir(self,lockdir):
-		if not os.path.exists(lockdir):
-			os.makedirs(lockdir)
-		if os.path.isdir(lockdir):
-			if not self.islocked():
-				if lockdir[-1] == "/":
-					lockdir=lockdir[:-1]
-				self.lockdir=normpath(lockdir)
-#				if self.settings["DEBUG"]:
-#					print "setting lockdir to", self.lockdir
-		else:
-			raise "the lock object needs a path to a dir"
-
-	def set_lockfilename(self,lockfilename):
-		if not self.islocked():
-			self.lockfilename=lockfilename
-#			if self.settings["DEBUG"]:
-#				print "setting lockfilename to", self.lockfilename
-
-	def set_lockfile(self):
-		if not self.islocked():
-			self.lockfile=normpath(self.lockdir+'/'+self.lockfilename)
-#			if self.settings["DEBUG"]:
-#				print "setting lockfile to", self.lockfile
+	def __init__(self, lockdir):
+		self.gid = 250
+		self.lockfile = os.path.join(lockdir, '.catalyst_lock')
+		osutils.ensure_dirs(lockdir)
+		fileutils.touch(self.lockfile, mode=0o664)
+		os.chown(self.lockfile, -1, self.gid)
+		self.lock = osutils.FsLock(self.lockfile)
 
 	def read_lock(self):
-		if not self.locking_method == "HARDLOCK":
-			self.fcntl_lock("read")
-		else:
-			print "HARDLOCKING doesnt support shared-read locks"
-			print "using exclusive write locks"
-			self.hard_lock()
+		self.lock.acquire_read_lock()
 
 	def write_lock(self):
-		if not self.locking_method == "HARDLOCK":
-			self.fcntl_lock("write")
-		else:
-			self.hard_lock()
+		self.lock.acquire_write_lock()
 
 	def unlock(self):
-		if not self.locking_method == "HARDLOCK":
-			self.fcntl_unlock()
-		else:
-			self.hard_unlock()
-
-	def fcntl_lock(self,locktype):
-		if self.myfd==None:
-			if not os.path.exists(os.path.dirname(self.lockdir)):
-				raise CatalystError("DirectoryNotFound: %s"
-					% os.path.dirname(self.lockdir), print_traceback=True)
-			if not os.path.exists(self.lockfile):
-				old_mask=os.umask(000)
-				self.myfd = os.open(self.lockfile, os.O_CREAT|os.O_RDWR,0660)
-				try:
-					if os.stat(self.lockfile).st_gid != self.gid:
-						os.chown(self.lockfile,os.getuid(),self.gid)
-				except OSError, e:
-					if e[0] == 2: #XXX: No such file or directory
-						return self.fcntl_locking(locktype)
-					else:
-						writemsg("Cannot chown a lockfile. This could cause inconvenience later.\n")
-
-				os.umask(old_mask)
-			else:
-				self.myfd = os.open(self.lockfile, os.O_CREAT|os.O_RDWR,0660)
-
-		try:
-			if locktype == "read":
-				self.locking_method(self.myfd,fcntl.LOCK_SH|fcntl.LOCK_NB)
-			else:
-				self.locking_method(self.myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
-		except IOError, e:
-			if "errno" not in dir(e):
-				raise
-			if e.errno == errno.EAGAIN:
-				if not LockDir.die_on_failed_lock:
-					# Resource temp unavailable; eg, someone beat us to the lock.
-					writemsg("waiting for lock on %s\n" % self.lockfile)
-
-					# Try for the exclusive or shared lock again.
-					if locktype == "read":
-						self.locking_method(self.myfd,fcntl.LOCK_SH)
-					else:
-						self.locking_method(self.myfd,fcntl.LOCK_EX)
-				else:
-					raise LockInUse,self.lockfile
-			elif e.errno == errno.ENOLCK:
-				pass
-			else:
-				raise
-		if not os.path.exists(self.lockfile):
-			os.close(self.myfd)
-			self.myfd=None
-			#writemsg("lockfile recurse\n")
-			self.fcntl_lock(locktype)
-		else:
-			self.locked=True
-			#writemsg("Lockfile obtained\n")
-
-	def fcntl_unlock(self):
-		unlinkfile = 1
-		if not os.path.exists(self.lockfile):
-			print "lockfile does not exist '%s'" % self.lockfile
-			#print "fcntl_unlock() , self.myfd:", self.myfd, type(self.myfd)
-			if self.myfd != None:
-				#print "fcntl_unlock() trying to close it "
-				try:
-					os.close(self.myfd)
-					self.myfd=None
-				except Exception:
-					pass
-				return False
-
-			try:
-				if self.myfd == None:
-					self.myfd = os.open(self.lockfile, os.O_WRONLY,0660)
-					unlinkfile = 1
-					self.locking_method(self.myfd,fcntl.LOCK_UN)
-			except Exception, e:
-				#if self.myfd is not None:
-					#print "fcntl_unlock() trying to close", self.myfd
-					#os.close(self.myfd)
-					#self.myfd=None
-				#raise IOError, "Failed to unlock file '%s'\n%s" % (self.lockfile, str(e))
-				try:
-					# This sleep call was added to allow other processes that are
-					# waiting for a lock to be able to grab it before it is deleted.
-					# lockfile() already accounts for this situation, however, and
-					# the sleep here adds more time than is saved overall, so am
-					# commenting until it is proved necessary.
-					#time.sleep(0.0001)
-					if unlinkfile:
-						InUse=False
-						try:
-							self.locking_method(self.myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
-						except Exception:
-							print "Read lock may be in effect. skipping lockfile delete..."
-							InUse=True
-							# We won the lock, so there isn't competition for it.
-							# We can safely delete the file.
-							#writemsg("Got the lockfile...\n")
-							#writemsg("Unlinking...\n")
-							self.locking_method(self.myfd,fcntl.LOCK_UN)
-					if not InUse:
-						os.unlink(self.lockfile)
-						os.close(self.myfd)
-						self.myfd=None
-#						if self.settings["DEBUG"]:
-#							print "Unlinked lockfile..."
-				except Exception, e:
-					# We really don't care... Someone else has the lock.
-					# So it is their problem now.
-					print "Failed to get lock... someone took it."
-					print str(e)
-
-					# Why test lockfilename?  Because we may have been handed an
-					# fd originally, and the caller might not like having their
-					# open fd closed automatically on them.
-					#if type(lockfilename) == types.StringType:
-					#        os.close(myfd)
-		#print "fcntl_unlock() trying a last ditch close", self.myfd
-		if self.myfd != None:
-			os.close(self.myfd)
-			self.myfd=None
-			self.locked=False
-			time.sleep(.0001)
-
-	def hard_lock(self,max_wait=14400):
-		"""Does the NFS, hardlink shuffle to ensure locking on the disk.
-		We create a PRIVATE lockfile, that is just a placeholder on the disk.
-		Then we HARDLINK the real lockfile to that private file.
-		If our file can 2 references, then we have the lock. :)
-		Otherwise we lather, rise, and repeat.
-		We default to a 4 hour timeout.
-		"""
-
-		self.myhardlock = self.hardlock_name(self.lockdir)
-
-		start_time = time.time()
-		reported_waiting = False
-
-		while time.time() < (start_time + max_wait):
-			# We only need it to exist.
-			self.myfd = os.open(self.myhardlock, os.O_CREAT|os.O_RDWR,0660)
-			os.close(self.myfd)
-
-			self.add_hardlock_file_to_cleanup()
-			if not os.path.exists(self.myhardlock):
-				raise CatalystError("FileNotFound: Created lockfile is missing: "
-					"%(filename)s" % {"filename":self.myhardlock},
-					print_traceback=True)
-			try:
-				os.link(self.myhardlock, self.lockfile)
-			except Exception:
-#				if self.settings["DEBUG"]:
-#					print "lockfile(): Hardlink: Link failed."
-#					print "Exception: ",e
-				pass
-
-			if self.hardlink_is_mine(self.myhardlock, self.lockfile):
-				# We have the lock.
-				if reported_waiting:
-					print
-				return True
-
-			if reported_waiting:
-				writemsg(".")
-			else:
-				reported_waiting = True
-				print
-				print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)"
-				print "Lockfile: " + self.lockfile
-			time.sleep(3)
-
-		os.unlink(self.myhardlock)
-		return False
-
-	def hard_unlock(self):
-		try:
-			if os.path.exists(self.myhardlock):
-				os.unlink(self.myhardlock)
-			if os.path.exists(self.lockfile):
-				os.unlink(self.lockfile)
-		except Exception:
-			writemsg("Something strange happened to our hardlink locks.\n")
-
-	def add_hardlock_file_to_cleanup(self):
-		#mypath = self.normpath(path)
-		if os.path.isdir(self.lockdir) and os.path.isfile(self.myhardlock):
-			self.hardlock_paths[self.lockdir]=self.myhardlock
-
-	def remove_hardlock_file_from_cleanup(self):
-		if self.lockdir in self.hardlock_paths:
-			del self.hardlock_paths[self.lockdir]
-			print self.hardlock_paths
-
-	@staticmethod
-	def hardlock_name(path):
-		mypath=path+"/.hardlock-"+os.uname()[1]+"-"+str(os.getpid())
-		newpath = os.path.normpath(mypath)
-		if len(newpath) > 1:
-			if newpath[1] == "/":
-				newpath = "/"+newpath.lstrip("/")
-		return newpath
-
-	@staticmethod
-	def hardlink_is_mine(link, lock):
-		import stat
-		try:
-			myhls = os.stat(link)
-			mylfs = os.stat(lock)
-		except Exception:
-			myhls = None
-			mylfs = None
-
-		if myhls:
-			if myhls[stat.ST_NLINK] == 2:
-				return True
-		if mylfs:
-			if mylfs[stat.ST_INO] == myhls[stat.ST_INO]:
-				return True
-		return False
-
-	@staticmethod
-	def hardlink_active(lock):
-		if not os.path.exists(lock):
-			return False
-
-	def clean_my_hardlocks(self):
-		try:
-			for x in self.hardlock_paths.keys():
-				self.hardlock_cleanup(x)
-		except AttributeError:
-			pass
-
-	def hardlock_cleanup(self,path):
-		#mypid  = str(os.getpid())
-		myhost = os.uname()[1]
-		mydl = os.listdir(path)
-		results = []
-		mycount = 0
-
-		mylist = {}
-		for x in mydl:
-			filepath=path+"/"+x
-			if os.path.isfile(filepath):
-				parts = filepath.split(".hardlock-")
-			if len(parts) == 2:
-				filename = parts[0]
-				hostpid  = parts[1].split("-")
-				host  = "-".join(hostpid[:-1])
-				pid   = hostpid[-1]
-			if filename not in mylist:
-				mylist[filename] = {}
-
-			if host not in mylist[filename]:
-				mylist[filename][host] = []
-				mylist[filename][host].append(pid)
-				mycount += 1
-			else:
-				mylist[filename][host].append(pid)
-				mycount += 1
-
-
-		results.append("Found %(count)s locks" % {"count":mycount})
-		for x in mylist.keys():
-			if myhost in mylist[x]:
-				mylockname = self.hardlock_name(x)
-				if self.hardlink_is_mine(mylockname, self.lockfile) or \
-					not os.path.exists(self.lockfile):
-					for y in mylist[x].keys():
-						for z in mylist[x][y]:
-							filename = x+".hardlock-"+y+"-"+z
-							if filename == mylockname:
-								self.hard_unlock()
-								continue
-							try:
-								# We're sweeping through, unlinking everyone's locks.
-								os.unlink(filename)
-								results.append("Unlinked: " + filename)
-							except Exception:
-								pass
-					try:
-						os.unlink(x)
-						results.append("Unlinked: " + x)
-						os.unlink(mylockname)
-						results.append("Unlinked: " + mylockname)
-					except Exception:
-						pass
-				else:
-					try:
-						os.unlink(mylockname)
-						results.append("Unlinked: " + mylockname)
-					except Exception:
-						pass
-		return results
-
-
-if __name__ == "__main__":
-
-	def lock_work():
-		print
-		for i in range(1,6):
-			print i,time.time()
-			time.sleep(1)
-		print
-
-	print "Lock 5 starting"
-	Lock1=LockDir("/tmp/lock_path")
-	Lock1.write_lock()
-	print "Lock1 write lock"
-
-	lock_work()
-
-	Lock1.unlock()
-	print "Lock1 unlock"
-
-	Lock1.read_lock()
-	print "Lock1 read lock"
-
-	lock_work()
-
-	Lock1.unlock()
-	print "Lock1 unlock"
-
-	Lock1.read_lock()
-	print "Lock1 read lock"
-
-	Lock1.write_lock()
-	print "Lock1 write lock"
-
-	lock_work()
-
-	Lock1.unlock()
-	print "Lock1 unlock"
-
-	Lock1.read_lock()
-	print "Lock1 read lock"
-
-	lock_work()
-
-	Lock1.unlock()
-	print "Lock1 unlock"
-
-#Lock1.write_lock()
-#time.sleep(2)
-#Lock1.unlock()
-    ##Lock1.write_lock()
-    #time.sleep(2)
-    #Lock1.unlock()
+		# Releasing a write lock is the same as a read lock.
+		self.lock.release_write_lock()

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     062b87828d115bbea0dd6b2cefa35902db18f336
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 10 05:20:05 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sat Oct 10 23:54:37 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=062b8782

support: convert to log module

 catalyst/support.py | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 6b1e727..8883acb 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -4,10 +4,10 @@ import sys
 import os
 import types
 import re
-import traceback
 import time
 from subprocess import Popen
 
+from catalyst import log
 from catalyst.defaults import valid_config_file_values
 
 BASH_BINARY             = "/bin/bash"
@@ -29,21 +29,13 @@ def list_bashify(mylist):
 class CatalystError(Exception):
 	def __init__(self, message, print_traceback=False):
 		if message:
-			if print_traceback:
-				(_type, value) = sys.exc_info()[:2]
-				if value!=None:
-					print
-					print "Traceback values found.  listing..."
-					print traceback.print_exc(file=sys.stdout)
-			print
-			print "!!! catalyst: "+message
-			print
+			log.error('CatalystError: %s', message, exc_info=print_traceback)
 
 
 def cmd(mycmd, myexc="", env=None, debug=False, fail_func=None):
 	if env is None:
 		env = {}
-	#print "***** cmd()"
+	log.debug('cmd: %r', mycmd)
 	sys.stdout.flush()
 	args=[BASH_BINARY]
 	if "BASH_ENV" not in env:
@@ -54,12 +46,11 @@ def cmd(mycmd, myexc="", env=None, debug=False, fail_func=None):
 	args.append("-c")
 	args.append(mycmd)
 
-	if debug:
-		print "***** cmd(); args =", args
+	log.debug('args: %r', args)
 	proc = Popen(args, env=env)
 	if proc.wait() != 0:
 		if fail_func:
-			print "CMD(), NON-Zero command return.  Running fail_func()"
+			log.error('CMD(), NON-Zero command return.  Running fail_func().')
 			fail_func()
 		raise CatalystError("cmd() NON-zero return value from: %s" % myexc,
 			print_traceback=False)
@@ -223,15 +214,17 @@ def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 
 def countdown(secs=5, doing="Starting"):
 	if secs:
-		print ">>> Waiting",secs,"seconds before starting..."
-		print ">>> (Control-C to abort)...\n"+doing+" in: ",
+		sys.stdout.write(
+			('>>> Waiting %s seconds before starting...\n'
+			 '>>> (Control-C to abort)...\n'
+			 '%s in: ') % (secs, doing))
 		ticks=range(secs)
 		ticks.reverse()
 		for sec in ticks:
 			sys.stdout.write(str(sec+1)+" ")
 			sys.stdout.flush()
 			time.sleep(1)
-		print
+		sys.stdout.write('\n')
 
 
 def normpath(mypath):

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     f3eea23c5cf7cf4e54073f59796a25db17d18613
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 12 02:17:26 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Mon Oct 12 02:17:26 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f3eea23c

log: make sure NOTICE is an int

The log module wants ints for its levels, so make sure NOTICE is
not a float due to the division.  This doesn't show up so much in
py2, but py3 barfs on floats.

 catalyst/log.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/log.py b/catalyst/log.py
index 871c53c..5938199 100644
--- a/catalyst/log.py
+++ b/catalyst/log.py
@@ -39,7 +39,7 @@ del _klass
 
 
 # Set the notice level between warning and info.
-NOTICE = (logging.WARNING + logging.INFO) / 2
+NOTICE = (logging.WARNING + logging.INFO) // 2
 logging.addLevelName(NOTICE, 'NOTICE')
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     d4efb6bffa322304e6ebba7797f7b9c2e6460465
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 11 19:05:22 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Oct 11 19:05:22 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d4efb6bf

lock: fix ensure_dirs call

We want to use our local wrapper because the snakeoil one defaults
to mode 777 whereas ours defaults to 755.

 catalyst/lock.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index 39926dd..808df4e 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -3,6 +3,7 @@ import os
 
 from snakeoil import fileutils
 from snakeoil import osutils
+from catalyst.fileops import ensure_dirs
 
 
 LockInUse = osutils.LockException
@@ -14,7 +15,7 @@ class LockDir(object):
 	def __init__(self, lockdir):
 		self.gid = 250
 		self.lockfile = os.path.join(lockdir, '.catalyst_lock')
-		osutils.ensure_dirs(lockdir)
+		ensure_dirs(lockdir)
 		fileutils.touch(self.lockfile, mode=0o664)
 		os.chown(self.lockfile, -1, self.gid)
 		self.lock = osutils.FsLock(self.lockfile)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     ac0f1d1f61ed3ec00831867f2a34eea88e7160ec
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 12 03:06:02 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Mon Oct 12 03:06:02 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ac0f1d1f

hash_utils: fix bad keyword w/CatalystError

The exception takes "print_traceback", not "traceback".

 catalyst/hash_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py
index 3db61f1..1161da3 100644
--- a/catalyst/hash_utils.py
+++ b/catalyst/hash_utils.py
@@ -79,7 +79,7 @@ class HashMap(object):
 				hash_)
 		except:
 			raise CatalystError("Error generating hash, is appropriate " + \
-				"utility installed on your system?", traceback=True)
+				"utility installed on your system?", print_traceback=True)
 
 
 	def calc_hash(self, file_, hash_):

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     328650032448425d64ec186fe24d84930f95d037
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 12 03:28:59 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Wed Oct 28 16:49:56 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=32865003

hash_utils: decode output of popen

In py3, the return of popen is binary data.  We need
to decode it to get a string we can work with.

 catalyst/hash_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py
index 1161da3..6e08967 100644
--- a/catalyst/hash_utils.py
+++ b/catalyst/hash_utils.py
@@ -117,7 +117,7 @@ class HashMap(object):
 		log.debug('args = %r', args)
 		source = Popen(args, stdout=PIPE)
 		output = source.communicate()
-		lines = output[0].split('\n')
+		lines = output[0].decode('ascii').split('\n')
 		log.debug('output = %s', output)
 		header = lines[0]
 		h_f = lines[1].split()

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     d307242bf41554640a8966d79f6f36738d3391ee
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 27 05:51:52 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Wed Oct 28 16:49:56 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d307242b

main: extend the -c option to accept multiple files

There's no real reason people can't load multiple config files, so extend
the --config option into --configs and let people specify multiple ones.

 catalyst/defaults.py |  2 ++
 catalyst/main.py     | 55 +++++++++++++++++-----------------------------------
 2 files changed, 20 insertions(+), 37 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 666afca..c5162d6 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -46,6 +46,8 @@ confdefaults={
 	"storedir": "/var/tmp/catalyst",
 	}
 
+DEFAULT_CONFIG_FILE = '/etc/catalyst/catalyst.conf'
+
 PORT_LOGDIR_CLEAN = \
 	'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +30 -delete'
 

diff --git a/catalyst/main.py b/catalyst/main.py
index 9f563cf..176871d 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -21,7 +21,7 @@ from DeComp.contents import ContentsMap
 
 from catalyst import log
 import catalyst.config
-from catalyst.defaults import confdefaults, option_messages
+from catalyst.defaults import confdefaults, option_messages, DEFAULT_CONFIG_FILE
 from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
 from catalyst.support import CatalystError
 from catalyst.version import get_version
@@ -36,40 +36,19 @@ def version():
 	log.info('Copyright 2008-2012 various authors')
 	log.info('Distributed under the GNU General Public License version 2.1')
 
-def parse_config(myconfig):
+def parse_config(config_files):
 	# search a couple of different areas for the main config file
 	myconf={}
-	config_file=""
-	default_config_file = '/etc/catalyst/catalyst.conf'
 
-	# first, try the one passed (presumably from the cmdline)
-	if myconfig:
-		if os.path.exists(myconfig):
-			log.notice('Using command line specified Catalyst configuration file: %s',
-				myconfig)
-			config_file=myconfig
-
-		else:
-			log.critical('Specified configuration file does not exist: %s', myconfig)
-
-	# next, try the default location
-	elif os.path.exists(default_config_file):
-		log.notice('Using default Catalyst configuration file: %s',
-			default_config_file)
-		config_file = default_config_file
-
-	# can't find a config file (we are screwed), so bail out
-	else:
-		log.critical('Could not find a suitable configuration file')
-
-	# now, try and parse the config file "config_file"
-	try:
-#		execfile(config_file, myconf, myconf)
-		myconfig = catalyst.config.ConfigParser(config_file)
-		myconf.update(myconfig.get_values())
-
-	except Exception:
-		log.critical('Could not find parse configuration file: %s', myconfig)
+	# try and parse the config file "config_file"
+	for config_file in config_files:
+		log.notice('Loading configuration file: %s', config_file)
+		try:
+			config = catalyst.config.ConfigParser(config_file)
+			myconf.update(config.get_values())
+		except Exception as e:
+			log.critical('Could not find parse configuration file: %s: %s',
+				config_file, e)
 
 	# now, load up the values into conf_values so that we can use them
 	for x in list(confdefaults):
@@ -209,9 +188,9 @@ $ catalyst -f stage1-specfile.spec"""
 	group.add_argument('-F', '--fetchonly',
 		default=False, action='store_true',
 		help='fetch files only')
-	group.add_argument('-c', '--config',
-		type=FilePath(),
-		help='use specified configuration file')
+	group.add_argument('-c', '--configs',
+		type=FilePath(), action='append',
+		help='use specified configuration files')
 	group.add_argument('-f', '--file',
 		type=FilePath(),
 		help='read specfile')
@@ -241,7 +220,9 @@ def main():
 		color=opts.color)
 
 	# Parse the command line options.
-	myconfig = opts.config
+	myconfigs = opts.configs
+	if not myconfigs:
+		myconfigs = [DEFAULT_CONFIG_FILE]
 	myspecfile = opts.file
 	mycmdline = opts.cli[:]
 
@@ -271,7 +252,7 @@ def main():
 	# made it this far so start by outputting our version info
 	version()
 	# import configuration file and import our main module using those settings
-	parse_config(myconfig)
+	parse_config(myconfigs)
 
 	conf_values["options"].update(options)
 	log.debug('conf_values[options] = %s', conf_values['options'])

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     1cb701ee64e06badb16a0086bb4e46184633c5da
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 29 00:27:46 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Thu Oct 29 00:27:46 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1cb701ee

config: tweak walking of settings for py3

Since py3 returns an iterator with dict.keys(), trying to add/del values
in the loop will throw an exception.  Run it through list() so we get a
static copy to iterate over.

 catalyst/config.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalyst/config.py b/catalyst/config.py
index db81a96..5f72e15 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -97,7 +97,8 @@ class ParserBase(object):
 				values[cur_array[0]] = cur_array[1:]
 
 		if not self.empty_values:
-			for x in values.keys():
+			# Make sure the list of keys is static since we modify inside the loop.
+			for x in list(values.keys()):
 				# Delete empty key pairs
 				if not values[x]:
 					log.warning('No value set for key "%s"; deleting', x)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     426c2f17a3bc6f22a7397854964ec5ba7a33f96d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Nov  9 02:04:04 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Nov  9 02:04:04 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=426c2f17

support.py: Make file_check restrict it's short test to os.path.isfile()

This avoids seedcache being disabled but having a directory of the same name as the seed it is looking for.

 catalyst/support.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 380c1c1..4bf1b22 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -59,7 +59,7 @@ def file_check(filepath):
 	'''Check for the files existence and that only one exists
 	if others are found with various extensions
 	'''
-	if os.path.exists(filepath):
+	if os.path.isfile(filepath):
 		return filepath
 	# it didn't exist
 	# so check if there are files of that name with an extension

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     325c5734ee357d03b6da36c53bbb48de62ca5666
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 23 12:04:30 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Dec 18 01:53:22 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=325c5734

main: add a --profile option

When things get slow, this option helps narrow down where things
are getting hung up.

 catalyst/main.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/catalyst/main.py b/catalyst/main.py
index 3550809..cb90e66 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -174,6 +174,9 @@ $ catalyst -f stage1-specfile.spec"""
 	group.add_argument('--trace',
 		default=False, action='store_true',
 		help='trace program output (akin to `sh -x`)')
+	group.add_argument('--profile',
+		default=False, action='store_true',
+		help='profile program execution')
 
 	group = parser.add_argument_group('Temporary file management')
 	group.add_argument('-a', '--clear-autoresume',
@@ -243,6 +246,25 @@ def trace(func, *args, **kwargs):
 	return tracer.runfunc(func, *args, **kwargs)
 
 
+def profile(func, *args, **kwargs):
+	"""Run |func| through the profile module"""
+	# Should make this an option.
+	sort_keys = ('time',)
+
+	# Collect the profile.
+	import cProfile
+	profiler = cProfile.Profile(subcalls=True, builtins=True)
+	try:
+		ret = profiler.runcall(func, *args, **kwargs)
+	finally:
+		# Then process the results.
+		import pstats
+		stats = pstats.Stats(profiler, stream=sys.stderr)
+		stats.strip_dirs().sort_stats(*sort_keys).print_stats()
+
+	return ret
+
+
 def main(argv):
 	"""The main entry point for frontends to use"""
 	parser = get_parser()
@@ -250,6 +272,8 @@ def main(argv):
 
 	if opts.trace:
 		return trace(_main, parser, opts)
+	elif opts.profile:
+		return profile(_main, parser, opts)
 	else:
 		return _main(parser, opts)
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     f36f7adc0bdd0c60e78d0e4a3d366cac61568e3a
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 23 11:57:19 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Dec 18 01:53:22 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f36f7adc

main: add a --trace option

This helps a lot with debugging as you can quickly get a transcript
of the python code paths that are taken by catalyst.

 catalyst/main.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/catalyst/main.py b/catalyst/main.py
index f48293e..3550809 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -170,6 +170,11 @@ $ catalyst -f stage1-specfile.spec"""
 		dest='color', action='store_false',
 		help='never colorize output all the time (default: detect)')
 
+	group = parser.add_argument_group('Developer options')
+	group.add_argument('--trace',
+		default=False, action='store_true',
+		help='trace program output (akin to `sh -x`)')
+
 	group = parser.add_argument_group('Temporary file management')
 	group.add_argument('-a', '--clear-autoresume',
 		default=False, action='store_true',
@@ -203,10 +208,54 @@ $ catalyst -f stage1-specfile.spec"""
 	return parser
 
 
+def trace(func, *args, **kwargs):
+	"""Run |func| through the trace module (akin to `sh -x`)"""
+	import trace
+
+	# Ignore common system modules we use.
+	ignoremods = set((
+		'argparse',
+		'genericpath', 'gettext',
+		'locale',
+		'os',
+		'posixpath',
+		're',
+		'sre_compile', 'sre_parse', 'sys',
+		'tempfile', 'threading',
+		'UserDict',
+	))
+
+	# Ignore all the system modules.
+	ignoredirs = set(sys.path)
+	# But try to strip out the catalyst paths.
+	try:
+		ignoredirs.remove(os.path.dirname(os.path.dirname(
+			os.path.realpath(__file__))))
+	except KeyError:
+		pass
+
+	tracer = trace.Trace(
+		count=False,
+		trace=True,
+		timing=True,
+		ignoremods=ignoremods,
+		ignoredirs=ignoredirs)
+	return tracer.runfunc(func, *args, **kwargs)
+
+
 def main(argv):
+	"""The main entry point for frontends to use"""
 	parser = get_parser()
 	opts = parser.parse_args(argv)
 
+	if opts.trace:
+		return trace(_main, parser, opts)
+	else:
+		return _main(parser, opts)
+
+
+def _main(parser, opts):
+	"""The "main" main function so we can trace/profile."""
 	# Initialize the logger before anything else.
 	log_level = opts.log_level
 	if log_level is None:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     9dad48cd58b820313ce1cc0fcceb87db749806b2
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Feb  2 06:08:25 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Feb  2 06:08:25 2016 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=9dad48cd

fileops.py: Fix docstring typo

 catalyst/fileops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 1ef6b4c..4fdc044 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -59,7 +59,7 @@ def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
 
 	@target: string, path to be cleared or removed
 	@mode: integer, desired mode to set the directory to
-	@chg_flags: boolean used for FreeBSD hoosts
+	@chg_flags: boolean used for FreeBSD hosts
 	@remove: boolean, passed through to clear_dir()
 	@return boolean
 	'''

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     90fcb4d81e9589a29c3bf1f01384184e713ef6ca
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Feb  2 06:07:52 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Feb  2 06:07:52 2016 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=90fcb4d8

main.py: Add missing value to log statement

 catalyst/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index cb90e66..6608b86 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -388,7 +388,7 @@ def _main(parser, opts):
 			log.critical(
 				'%s is not a valid hash_function entry\n'
 				'Valid hash_function entries:\n'
-				'%s', HASH_DEFINITIONS.keys())
+				'%s', conf_values["hash_function"], HASH_DEFINITIONS.keys())
 		try:
 			process.find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd)
 		except process.CommandNotFound:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     ce31e7ab7b5b8968b7ff0c17bbe00d09921729b7
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 11 12:59:02 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Thu Feb 11 12:59:02 2016 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ce31e7ab

builder: only apply `setarch` on Linux

Bugzilla: https://bugs.gentoo.org/363577
Reported-by: Yuta SATOH <nigoro.dev <AT> gmail.com>

 catalyst/builder.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/catalyst/builder.py b/catalyst/builder.py
index 7380cac..4d58de6 100644
--- a/catalyst/builder.py
+++ b/catalyst/builder.py
@@ -1,3 +1,4 @@
+import os
 
 class generic(object):
 	def __init__(self,myspec):
@@ -9,7 +10,8 @@ class generic(object):
 
 		Useful for building x86-on-amd64 and such.
 		"""
-		self.settings['CHROOT'] = 'setarch %s %s' % (arch, self.settings['CHROOT'])
+		if os.uname()[0] == 'Linux':
+			self.settings['CHROOT'] = 'setarch %s %s' % (arch, self.settings['CHROOT'])
 
 	def mount_safety_check(self):
 		"""

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     cc7958c85d127afc169c34c83b28a77f51516789
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 15 06:49:16 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Feb 17 05:25:21 2016 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=cc7958c8

catalyst/support.py: Fix a py3 compatibility issue

 catalyst/support.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 97fe562..4efe96c 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -225,7 +225,8 @@ def countdown(secs=5, doing="Starting"):
 			('>>> Waiting %s seconds before starting...\n'
 			 '>>> (Control-C to abort)...\n'
 			 '%s in: ') % (secs, doing))
-		ticks=range(secs)
+		# py3 now creates a range object, so wrap it with list()
+		ticks=list(range(secs))
 		ticks.reverse()
 		for sec in ticks:
 			sys.stdout.write(str(sec+1)+" ")

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     f3983e2204c7dd474958e4b8cf2a349288f51657
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun May 22 03:35:02 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun May 22 03:35:02 2016 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f3983e22

countdown: only run when interactive

There's no point in sleeping when running from scripted sessions.

 catalyst/support.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/catalyst/support.py b/catalyst/support.py
index f2ae5bb..60cdaf6 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -219,6 +219,10 @@ def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 
 
 def countdown(secs=5, doing="Starting"):
+	# If this is non-interactive (e.g. a cronjob), then sleeping is pointless.
+	if not os.isatty(sys.stdin.fileno()):
+		return
+
 	if secs:
 		sys.stdout.write(
 			('>>> Waiting %s seconds before starting...\n'

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     d923de2a758e67bcc7970dd2138395d8b1edc4b6
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun May 22 03:48:05 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun May 22 03:48:05 2016 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d923de2a

support: enhance docstrings a bit

 catalyst/support.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 60cdaf6..9cc5d75 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -188,7 +188,7 @@ def pathcompare(path1,path2):
 
 
 def ismount(path):
-	"enhanced to handle bind mounts"
+	"""Like os.path.ismount, but also support bind mounts"""
 	if os.path.ismount(path):
 		return 1
 	a=os.popen("mount")
@@ -239,6 +239,12 @@ def countdown(secs=5, doing="Starting"):
 
 
 def normpath(mypath):
+	"""Clean up a little more than os.path.normpath
+
+	Namely:
+	 - Make sure leading // is turned into /.
+	 - Leave trailing slash intact.
+	"""
 	TrailingSlash=False
 	if mypath[-1] == "/":
 		TrailingSlash=True

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     727d8118db98549b0cc48f3c56db842cf82b0c27
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 25 15:44:40 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jun 25 15:44:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=727d8118

fileops.py: Fix a traceback clearing teh old profile link

The isdir() cconditional was not enough, it also needed to make sure it was not a symlink.

25 Jun 2016 08:02:45 PDT: NOTICE  : --- Running action sequence: config_profile_link
25 Jun 2016 08:02:45 PDT: ERROR   : clear_dir failed
Traceback (most recent call last):
  File "/home/brian/Dev/git/catalyst/catalyst/fileops.py", line 84, in clear_dir
    shutil.rmtree(target)
  File "/usr/lib64/python3.4/shutil.py", line 478, in rmtree
    onerror(os.path.islink, path, sys.exc_info())
  File "/usr/lib64/python3.4/shutil.py", line 476, in rmtree
    raise OSError("Cannot call rmtree on a symbolic link")
OSError: Cannot call rmtree on a symbolic link

 catalyst/fileops.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 6971911..d2bd453 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -70,7 +70,7 @@ def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
 		return False
 
 	mystat = None
-	if os.path.isdir(target):
+	if os.path.isdir(target) and not os.path.islink(target):
 		log.info('Emptying directory: %s', target)
 		# stat the dir, delete the dir, recreate the dir and set
 		# the proper perms and ownership
@@ -87,6 +87,7 @@ def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
 			return False
 	elif os.path.exists(target):
 		if clear_nondir:
+			log.debug("Clearing (unlinking) non-directory: %s", target)
 			os.unlink(target)
 		else:
 			log.info('clear_dir failed: %s: is not a directory', target)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Mike Frysinger]

commit:     e5a53e42022668563965ede6409a3de233af332c
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 20:32:03 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sat Mar 11 22:01:27 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e5a53e42

catalyst: create namespaces for building

This helps isolate the catalyst builds from the rest of the system
and allows us to build as non-root user in more cases.

We don't support pid or user namespaces yet.

 catalyst/main.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/catalyst/main.py b/catalyst/main.py
index 51d2b04..b8ae662 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -12,6 +12,7 @@ import os
 import sys
 
 from snakeoil import process
+from snakeoil.process import namespaces
 
 from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
 	CONTENTS_DEFINITIONS)
@@ -421,6 +422,14 @@ def _main(parser, opts):
 		# catalyst cannot be run as a normal user due to chroots, mounts, etc
 		log.critical('This script requires root privileges to operate')
 
+	# Start off by creating unique namespaces to run in.  Would be nice to
+	# use pid & user namespaces, but snakeoil's namespace module has signal
+	# transfer issues (CTRL+C doesn't propagate), and user namespaces need
+	# more work due to Gentoo build process (uses sudo/root/portage).
+	namespaces.simple_unshare(
+		mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
+		hostname='catalyst')
+
 	# everything is setup, so the build is a go
 	try:
 		success = build_target(addlargs)

[gentoo-commits] proj/catalyst:pending commit in: catalyst/

[Brian Dolbec]

commit:     b9f2806724077740c8fdbf8bcdb5a431cb085e9d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  9 09:24:22 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Nov 22 01:16:21 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b9f28067

defaults.py: Update all repository, DISTDIR, PKGDIR settings to new /var paths

All settings are still configurable in catalyst.conf.
These are new reasonable settings for a relocated tree system.
A user can still set their own locations during install, or at a later time.

Default catalyst.conf settings will be changed at a later time after additional testing and
the automatic stage building scripts have been updated.

 catalyst/defaults.py | 37 +++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 84ed2822..ed5766d6 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -31,6 +31,15 @@ valid_config_file_values.extend([ "distcc", "envscript",
 	"compression_mode", "compressor_options", "decompressor_search_order",
 	])
 
+# set our base defaults here to keep
+# them in one location.
+BASE_GENTOO_DIR = "/var/gentoo"
+REPODIR = BASE_GENTOO_DIR + "/repos"
+DISTDIR = BASE_GENTOO_DIR + "/distfiles"
+PKGDIR = BASE_GENTOO_DIR + "/packages"
+MAINREPO = "gentoo"
+PORTDIR = REPODIR + "/" + MAINREPO
+
 confdefaults={
 	"archdir": "%(PythonDir)s/arch",
 	"comp_prog": COMPRESSOR_PROGRAM_OPTIONS[TAR],
@@ -39,28 +48,28 @@ confdefaults={
 	"compressor_options": XATTRS_OPTIONS[TAR],
 	"decomp_opt": DECOMPRESSOR_PROGRAM_OPTIONS[TAR],
 	"decompressor_search_order": DECOMPRESSOR_SEARCH_ORDER,
-	"distdir": "/usr/portage/distfiles",
+	"distdir": DISTDIR[:],
 	"hash_function": "crc32",
 	"icecream": "/var/cache/icecream",
 	'list_xattrs_opt': LIST_XATTRS_OPTIONS[TAR],
-	"local_overlay": "/usr/local/portage",
+	"local_overlay": REPODIR[:] + "/local",
 	"port_conf": "/etc/portage",
 	"make_conf": "%(port_conf)s/make.conf",
 	"options": set(),
-	"packagedir": "/usr/portage/packages",
-	"portdir": "/usr/portage",
+	"packagedir": PKGDIR[:],
+	"portdir": PORTDIR[:],
 	"port_tmpdir": "/var/tmp/portage",
 	"PythonDir": "./catalyst",
-	"repo_basedir": "/usr",
-	"repo_name": "portage",
+	"repo_basedir": REPODIR[:],
+	"repo_name": MAINREPO[:],
 	"sharedir": "/usr/share/catalyst",
 	"shdir": "/usr/share/catalyst/targets/",
 	"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
-	"snapshot_name": "portage-",
+	"snapshot_name": "%(repo_name)s-",
 	"source_matching": "strict",
 	"storedir": "/var/tmp/catalyst",
-	"target_distdir": "/var/portage/distfiles",
-	"target_pkgdir":"/var/portage/packages",
+	"target_distdir": DISTDIR[:],
+	"target_pkgdir": PKGDIR[:],
 	}
 
 DEFAULT_CONFIG_FILE = '/etc/catalyst/catalyst.conf'
@@ -72,11 +81,11 @@ TARGET_MOUNT_DEFAULTS = {
 	"ccache": "/var/tmp/ccache",
 	"dev": "/dev",
 	"devpts": "/dev/pts",
-	"distdir": "/usr/portage/distfiles",
+	"distdir": DISTDIR[:],
 	"icecream": "/usr/lib/icecc/bin",
 	"kerncache": "/tmp/kerncache",
-	"packagedir": "/usr/portage/packages",
-	"portdir": "/usr/portage",
+	"packagedir": PKGDIR[:],
+	"portdir": PORTDIR[:],
 	"port_tmpdir": "/var/tmp/portage",
 	"port_logdir": "/var/log/portage",
 	"proc": "/proc",
@@ -86,8 +95,8 @@ TARGET_MOUNT_DEFAULTS = {
 SOURCE_MOUNT_DEFAULTS = {
 	"dev": "/dev",
 	"devpts": "/dev/pts",
-	"distdir": "/usr/portage/distfiles",
-	"portdir": "/usr/portage",
+	"distdir": DISTDIR[:],
+	"portdir": PORTDIR[:],
 	"port_tmpdir": "tmpfs",
 	"proc": "/proc",
 	"shm": "shmfs",

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     b9f2806724077740c8fdbf8bcdb5a431cb085e9d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  9 09:24:22 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Nov 22 01:16:21 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b9f28067

defaults.py: Update all repository, DISTDIR, PKGDIR settings to new /var paths

All settings are still configurable in catalyst.conf.
These are new reasonable settings for a relocated tree system.
A user can still set their own locations during install, or at a later time.

Default catalyst.conf settings will be changed at a later time after additional testing and
the automatic stage building scripts have been updated.

 catalyst/defaults.py | 37 +++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 84ed2822..ed5766d6 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -31,6 +31,15 @@ valid_config_file_values.extend([ "distcc", "envscript",
 	"compression_mode", "compressor_options", "decompressor_search_order",
 	])
 
+# set our base defaults here to keep
+# them in one location.
+BASE_GENTOO_DIR = "/var/gentoo"
+REPODIR = BASE_GENTOO_DIR + "/repos"
+DISTDIR = BASE_GENTOO_DIR + "/distfiles"
+PKGDIR = BASE_GENTOO_DIR + "/packages"
+MAINREPO = "gentoo"
+PORTDIR = REPODIR + "/" + MAINREPO
+
 confdefaults={
 	"archdir": "%(PythonDir)s/arch",
 	"comp_prog": COMPRESSOR_PROGRAM_OPTIONS[TAR],
@@ -39,28 +48,28 @@ confdefaults={
 	"compressor_options": XATTRS_OPTIONS[TAR],
 	"decomp_opt": DECOMPRESSOR_PROGRAM_OPTIONS[TAR],
 	"decompressor_search_order": DECOMPRESSOR_SEARCH_ORDER,
-	"distdir": "/usr/portage/distfiles",
+	"distdir": DISTDIR[:],
 	"hash_function": "crc32",
 	"icecream": "/var/cache/icecream",
 	'list_xattrs_opt': LIST_XATTRS_OPTIONS[TAR],
-	"local_overlay": "/usr/local/portage",
+	"local_overlay": REPODIR[:] + "/local",
 	"port_conf": "/etc/portage",
 	"make_conf": "%(port_conf)s/make.conf",
 	"options": set(),
-	"packagedir": "/usr/portage/packages",
-	"portdir": "/usr/portage",
+	"packagedir": PKGDIR[:],
+	"portdir": PORTDIR[:],
 	"port_tmpdir": "/var/tmp/portage",
 	"PythonDir": "./catalyst",
-	"repo_basedir": "/usr",
-	"repo_name": "portage",
+	"repo_basedir": REPODIR[:],
+	"repo_name": MAINREPO[:],
 	"sharedir": "/usr/share/catalyst",
 	"shdir": "/usr/share/catalyst/targets/",
 	"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
-	"snapshot_name": "portage-",
+	"snapshot_name": "%(repo_name)s-",
 	"source_matching": "strict",
 	"storedir": "/var/tmp/catalyst",
-	"target_distdir": "/var/portage/distfiles",
-	"target_pkgdir":"/var/portage/packages",
+	"target_distdir": DISTDIR[:],
+	"target_pkgdir": PKGDIR[:],
 	}
 
 DEFAULT_CONFIG_FILE = '/etc/catalyst/catalyst.conf'
@@ -72,11 +81,11 @@ TARGET_MOUNT_DEFAULTS = {
 	"ccache": "/var/tmp/ccache",
 	"dev": "/dev",
 	"devpts": "/dev/pts",
-	"distdir": "/usr/portage/distfiles",
+	"distdir": DISTDIR[:],
 	"icecream": "/usr/lib/icecc/bin",
 	"kerncache": "/tmp/kerncache",
-	"packagedir": "/usr/portage/packages",
-	"portdir": "/usr/portage",
+	"packagedir": PKGDIR[:],
+	"portdir": PORTDIR[:],
 	"port_tmpdir": "/var/tmp/portage",
 	"port_logdir": "/var/log/portage",
 	"proc": "/proc",
@@ -86,8 +95,8 @@ TARGET_MOUNT_DEFAULTS = {
 SOURCE_MOUNT_DEFAULTS = {
 	"dev": "/dev",
 	"devpts": "/dev/pts",
-	"distdir": "/usr/portage/distfiles",
-	"portdir": "/usr/portage",
+	"distdir": DISTDIR[:],
+	"portdir": PORTDIR[:],
 	"port_tmpdir": "tmpfs",
 	"proc": "/proc",
 	"shm": "shmfs",

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Robin H. Johnson]

commit:     377ed63f69739f761eee3a06c7105bd0c0d02a80
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Wed Dec  6 18:51:39 2017 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Wed Dec  6 18:51:39 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=377ed63f

main: fix leading whitespace

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 catalyst/main.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 3a428d3a..eed40a66 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -190,13 +190,13 @@ $ catalyst -f stage1-specfile.spec"""
 	group.add_argument('-T', '--purgetmponly',
 		default=False, action='store_true',
 		help='clear tmp dirs and autoresume flags and exit')
-        group.add_argument('--versioned-cachedir',
-                dest='versioned_cachedir', action='store_true',
-                help='use stage version on cache directory name')
-        group.add_argument('--unversioned-cachedir',
-                dest='versioned_cachedir', action='store_false',
-                help='do not use stage version on cache directory name')
-        group.set_defaults(versioned_cachedir=False)
+	group.add_argument('--versioned-cachedir',
+		dest='versioned_cachedir', action='store_true',
+		help='use stage version on cache directory name')
+	group.add_argument('--unversioned-cachedir',
+		dest='versioned_cachedir', action='store_false',
+		help='do not use stage version on cache directory name')
+	group.set_defaults(versioned_cachedir=False)
 
 
 	group = parser.add_argument_group('Target/config file management')

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     b29b8d4c19075c9dcfa74dcc10d7dff49df64274
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 30 17:24:17 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Nov 30 17:24:17 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b29b8d4c

catalyst/version.py: Set version for a release

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index 412854d0..8c9c338b 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__= "3.0_rc1"
+__version__= "3.0.0"
 _ver = None
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     46e42a2ee301c03d118046c7394132c2bf9917ed
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Dec  7 07:06:19 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Dec  7 07:22:10 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=46e42a2e

main.py: increase verbosity of the options log

 catalyst/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index eed40a66..4c55a415 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -335,7 +335,7 @@ def _main(parser, opts):
 	parse_config(myconfigs)
 
 	conf_values["options"].update(options)
-	log.debug('conf_values[options] = %s', conf_values['options'])
+	log.notice('conf_values[options] = %s', conf_values['options'])
 
 	# initialize our contents generator
 	contents_map = ContentsMap(CONTENTS_DEFINITIONS,

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     2fdaa2db2cdb2b23a863f14fa4970c614718ae8a
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 29 00:10:53 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Fri Dec 29 00:34:31 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=2fdaa2db

fileops.py: Add extra debug logging...

 catalyst/fileops.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 5e51f1fc..ab6a0c9f 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -92,6 +92,11 @@ def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
 		else:
 			log.info('clear_dir failed: %s: is not a directory', target)
 			return False
+	else:
+		log.debug("Condidtions not met to clear: %s", target)
+		log.debug("                       isdir: %s", os.path.isdir(target))
+		log.debug("                      islink: %s", os.path.islink(target))
+		log.debug("                      exists: %s", os.path.exists(target))
 
 	if not remove:
 		log.debug('ensure_dirs()')

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     16155aaa9b09112a7a49d0c833b19be3399e746a
Author:     Yuta SATOH <nigoro.dev <AT> gmail <DOT> com>
AuthorDate: Fri Dec 29 01:02:02 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Fri Dec 29 02:01:27 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=16155aaa

main.py: Disable snakeoil's namespace module on Gentoo/FreeBSD #363557

Some code improvement by: Patrice Clement <monsieurp <AT> gentoo.org>
Bug url: https://bugs.gentoo.org/363577

 catalyst/main.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 4c55a415..2d2faada 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -430,13 +430,16 @@ def _main(parser, opts):
 		# catalyst cannot be run as a normal user due to chroots, mounts, etc
 		log.critical('This script requires root privileges to operate')
 
-	# Start off by creating unique namespaces to run in.  Would be nice to
-	# use pid & user namespaces, but snakeoil's namespace module has signal
-	# transfer issues (CTRL+C doesn't propagate), and user namespaces need
-	# more work due to Gentoo build process (uses sudo/root/portage).
-	namespaces.simple_unshare(
-		mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
-		hostname='catalyst')
+	# Namespaces aren't supported on *BSDs at the moment. So let's check
+	# whether we're on Linux.
+	if os.uname().sysname in  ["Linux", "linux"]:
+		# Start off by creating unique namespaces to run in.  Would be nice to
+		# use pid & user namespaces, but snakeoil's namespace module has signal
+		# transfer issues (CTRL+C doesn't propagate), and user namespaces need
+		# more work due to Gentoo build process (uses sudo/root/portage).
+		namespaces.simple_unshare(
+			mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
+			hostname='catalyst')
 
 	# everything is setup, so the build is a go
 	try:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     edc049c85af1a73bb029e3b850a863ce1c871c8e
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 30 18:22:56 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Dec 30 18:23:31 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=edc049c8

Update version for a release

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index 8c9c338b..f0a0db20 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__= "3.0.0"
+__version__= "3.0.1"
 _ver = None
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     e70f11a248bc836da2d5db25171bb59907e14a41
Author:     Brian Dolbec dolsen <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 20 16:24:37 2018 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Fri Jul 20 16:36:15 2018 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e70f11a2

Add glob support to clean_path()

Fixes licecd/rm removals.

Signed-off-by: Brian Dolbec dolsen <dolsen <AT> gentoo.org>

 catalyst/fileops.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index ab6a0c9f..69faf4cb 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -10,6 +10,7 @@ ensuring directories exist,... imports snakeoils osutils
 functions for use throughout catalyst.
 '''
 
+import glob
 import os
 import shutil
 from stat import ST_UID, ST_GID, ST_MODE
@@ -109,9 +110,11 @@ def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
 	return True
 
 
-def clear_path(target):
-	"""Nuke |target| regardless of it being a dir or file."""
-	clear_dir(target, remove=True)
+def clear_path(target_path):
+	"""Nuke |target_path| regardless of it being a dir, file or glob."""
+	targets = glob.glob(target_path)
+	for path in targets:
+		clear_dir(path, remove=True)
 
 
 def move_path(src, dest):

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     3f644f813b0ec97d990bec9f86b4e9e3f3f78518
Author:     Brian Dolbec dolsen <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 21 19:15:56 2018 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jul 21 19:15:56 2018 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3f644f81

Set version for a release

Signed-off-by: Brian Dolbec dolsen <dolsen <AT> gentoo.org>

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index f0a0db20..4c3134f9 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__= "3.0.1"
+__version__= "3.0.2"
 _ver = None
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     b4fc26fa8e68cbac9a536e29c52f71d7f5a614ca
Author:     M. J. Everitt <m.j.everitt <AT> iee <DOT> org>
AuthorDate: Fri Jul 13 16:52:39 2018 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Oct  1 16:17:14 2018 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b4fc26fa

Fix typo in fileops.py:clear_dir

 catalyst/fileops.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 69faf4cb..878e6303 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -94,10 +94,10 @@ def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
 			log.info('clear_dir failed: %s: is not a directory', target)
 			return False
 	else:
-		log.debug("Condidtions not met to clear: %s", target)
-		log.debug("                       isdir: %s", os.path.isdir(target))
-		log.debug("                      islink: %s", os.path.islink(target))
-		log.debug("                      exists: %s", os.path.exists(target))
+		log.debug("Conditions not met to clear: %s", target)
+		log.debug("                      isdir: %s", os.path.isdir(target))
+		log.debug("                     islink: %s", os.path.islink(target))
+		log.debug("                     exists: %s", os.path.exists(target))
 
 	if not remove:
 		log.debug('ensure_dirs()')

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Brian Dolbec]

commit:     a955852a5cf850b01de696988777ec26f5d24da1
Author:     Brian Dolbec dolsen <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  2 14:24:09 2018 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Oct  2 14:24:09 2018 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=a955852a

Set version for a release

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index 4c3134f9..a22bd6f4 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__= "3.0.2"
+__version__= "3.0.3"
 _ver = None
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     249ed2908251727e28fcfe44538c207f3fa47ae9
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Tue Feb  5 02:58:51 2019 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Tue Feb  5 02:58:51 2019 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=249ed290

catalyst 3.0.4

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index a22bd6f4..fbbef174 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__= "3.0.3"
+__version__= "3.0.4"
 _ver = None
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     42ce8b5fde54ad3634c4cdd61bb0c61513f107ed
Author:     Matt Turner <mattst88 <AT> gmail <DOT> com>
AuthorDate: Sun Jun 16 18:34:09 2019 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sun Jun 16 18:34:09 2019 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=42ce8b5f

catalyst 3.0.5

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index fbbef174..eac78fe8 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__= "3.0.4"
+__version__= "3.0.5"
 _ver = None
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     1f4ae13a3bd077889b34756a176fecf169a996c5
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 26 16:49:41 2019 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Mon Aug 26 16:49:41 2019 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1f4ae13a

catalyst 3.0.6

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index eac78fe8..2d883ff4 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__= "3.0.5"
+__version__= "3.0.6"
 _ver = None
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     469eca32620b16b3e2136d7ca3232c90679c5cab
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 19 23:26:05 2019 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sun Oct 20 00:00:08 2019 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=469eca32

catalyst 3.0.7

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index 2d883ff4..91b0a743 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -12,7 +12,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__= "3.0.6"
+__version__= "3.0.7"
 _ver = None
 
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     137a83f50b42ab9ddd4c3c0908aadc723ce940d2
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 10 17:59:33 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Apr 10 21:03:53 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=137a83f5

catalyst: Simplify countdown()

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/support.py | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index eb0b7d14..c7a8fc73 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -218,23 +218,19 @@ def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 
 
 def countdown(secs=5, doing="Starting"):
-	# If this is non-interactive (e.g. a cronjob), then sleeping is pointless.
-	if not os.isatty(sys.stdin.fileno()):
+	# Don't sleep if this is non-interactive
+	if not os.isatty(sys.stdin.fileno()) or secs == 0:
 		return
 
-	if secs:
-		sys.stdout.write(
-			('>>> Waiting %s seconds before starting...\n'
-			 '>>> (Control-C to abort)...\n'
-			 '%s in: ') % (secs, doing))
-		# py3 now creates a range object, so wrap it with list()
-		ticks=list(range(secs))
-		ticks.reverse()
-		for sec in ticks:
-			sys.stdout.write(str(sec+1)+" ")
-			sys.stdout.flush()
-			time.sleep(1)
-		sys.stdout.write('\n')
+	sys.stdout.write(
+		('>>> Waiting %s seconds before starting...\n'
+		 '>>> (Control-C to abort)...\n'
+		 '%s in: ') % (secs, doing))
+	for sec in reversed(range(1, secs + 1)):
+		sys.stdout.write(str(sec) + " ")
+		sys.stdout.flush()
+		time.sleep(1)
+	sys.stdout.write('\n')
 
 
 def normpath(mypath):

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     0239c60e9a811f46b080fa96074ce9cf82cc55e0
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 17 03:41:24 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Apr 17 03:42:51 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=0239c60e

catalyst: Reformat some array definitions

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 47 ++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 8e276500..570fc4df 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -5,22 +5,47 @@ from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS, LIST_XATTRS_OPTIONS
 
 
 # these should never be touched
-required_build_targets = ["targetbase", "generic_stage_target"]
+required_build_targets = [
+    "generic_stage_target",
+    "targetbase",
+]
 
 # new build types should be added here
-valid_build_targets = ["stage1_target", "stage2_target", "stage3_target",
-                       "stage4_target", "livecd_stage1_target", "livecd_stage2_target",
-                       "embedded_target", "snapshot_target", "netboot_target"
-                       ]
+valid_build_targets = [
+    "embedded_target",
+    "livecd_stage1_target",
+    "livecd_stage2_target",
+    "netboot_target",
+    "snapshot_target",
+    "stage1_target",
+    "stage2_target",
+    "stage3_target",
+    "stage4_target",
+]
 
-required_config_file_values = ["storedir", "sharedir", "distdir", "portdir"]
+required_config_file_values = [
+    "distdir",
+    "portdir",
+    "sharedir",
+    "storedir",
+]
 
 valid_config_file_values = required_config_file_values[:]
-valid_config_file_values.extend(["distcc", "envscript",
-                                 "options", "DEBUG", "VERBOSE",
-                                 "snapshot_cache", "hash_function", "digests", "contents", "compressor_arch",
-                                 "compression_mode", "compressor_options", "decompressor_search_order",
-                                 ])
+valid_config_file_values.extend([
+    "compression_mode",
+    "compressor_arch",
+    "compressor_options",
+    "contents",
+    "DEBUG",
+    "decompressor_search_order",
+    "digests",
+    "distcc",
+    "envscript",
+    "hash_function",
+    "options",
+    "snapshot_cache",
+    "VERBOSE",
+])
 
 # set our base defaults here to keep
 # them in one location.

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     04aba402175bdcfb63c9ec7127e13b46998904ac
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 17 17:32:26 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Apr 17 19:51:30 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=04aba402

catalyst: Raise an exception on invalid digest

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/main.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index c91fc8b6..7e2a6c06 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -347,11 +347,10 @@ def _main(parser, opts):
 
         # First validate all the requested digests are valid keys.
         if digests - valid_digests:
-            log.critical(
-                'These are not valid digest entries:\n%s\n'
-                'Valid digest entries:\n%s',
-                ', '.join(sorted(digests - valid_digests)),
-                ', '.join(sorted(valid_digests)))
+            raise CatalystError('These are not valid digest entries:\n%s\n'
+                                'Valid digest entries:\n%s' %
+                                ', '.join(sorted(digests - valid_digests)),
+                                ', '.join(sorted(valid_digests)))
 
     addlargs = {}
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     d29853f01fbdd87cf9cef0f04ae110b4185b1ecd
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 22 00:17:07 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Apr 22 00:22:24 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d29853f0

catalyst: Remove unused import

Fixes: 6565ad2eb90f (catalyst: Use hashlib instead of external tools)
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/main.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 7e2a6c06..8ded4bd1 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -4,7 +4,6 @@ import hashlib
 import os
 import sys
 
-from snakeoil import process
 from snakeoil.process import namespaces
 
 from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     bc9e8f841689c4b5b778adcc979105dbb4dd4147
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 22 01:33:49 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Apr 22 05:42:31 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=bc9e8f84

catalyst: Make shdir default match config

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 0c18f9f5..77133cb1 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -70,9 +70,9 @@ confdefaults = {
     "repo_basedir": "/var/db/repos",
     "repo_name": "gentoo",
     "sharedir": "/usr/share/catalyst",
-    "shdir": "/usr/share/catalyst/targets/",
     "snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
     "snapshot_name": "%(repo_name)s-",
+    "shdir": "%(sharedir)s/targets",
     "source_matching": "strict",
     "storedir": "/var/tmp/catalyst",
     "target_distdir": "/var/cache/distfiles",

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     80254c3695752a6732d2d65dfe0f9014eb746ded
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 22 00:48:18 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Apr 22 01:52:15 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=80254c36

catalyst: Remove and inline some default paths

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index e09d08e8..a0b87f96 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -47,15 +47,6 @@ valid_config_file_values.extend([
     "VERBOSE",
 ])
 
-# set our base defaults here to keep
-# them in one location.
-BASE_GENTOO_DIR = "/var/gentoo"
-REPODIR = BASE_GENTOO_DIR + "/repos"
-DISTDIR = BASE_GENTOO_DIR + "/distfiles"
-PKGDIR = BASE_GENTOO_DIR + "/packages"
-MAINREPO = "gentoo"
-PORTDIR = REPODIR + "/" + MAINREPO
-
 confdefaults = {
     "comp_prog": COMPRESSOR_PROGRAM_OPTIONS['linux'],
     "compression_mode": 'lbzip2',
@@ -63,27 +54,27 @@ confdefaults = {
     "compressor_options": XATTRS_OPTIONS['linux'],
     "decomp_opt": DECOMPRESSOR_PROGRAM_OPTIONS['linux'],
     "decompressor_search_order": DECOMPRESSOR_SEARCH_ORDER,
-    "distdir": DISTDIR[:],
+    "distdir": "/var/cache/distfiles",
     "icecream": "/var/cache/icecream",
     'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'],
-    "local_overlay": REPODIR[:] + "/local",
+    "local_overlay": "/var/db/repos/local",
     "port_conf": "/etc/portage",
     "make_conf": "%(port_conf)s/make.conf",
     "options": set(),
-    "pkgdir": PKGDIR[:],
-    "portdir": PORTDIR[:],
+    "pkgdir": "/var/cache/binpkgs",
+    "portdir": "/var/db/repos/gentoo",
     "port_tmpdir": "/var/tmp/portage",
     "PythonDir": "./catalyst",
-    "repo_basedir": REPODIR[:],
-    "repo_name": MAINREPO[:],
+    "repo_basedir": "/var/db/repos",
+    "repo_name": "gentoo",
     "sharedir": "/usr/share/catalyst",
     "shdir": "/usr/share/catalyst/targets/",
     "snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
     "snapshot_name": "%(repo_name)s-",
     "source_matching": "strict",
     "storedir": "/var/tmp/catalyst",
-    "target_distdir": DISTDIR[:],
-    "target_pkgdir": PKGDIR[:],
+    "target_distdir": "/var/cache/distfiles",
+    "target_pkgdir": "/var/cache/binpkgs",
 }
 
 DEFAULT_CONFIG_FILE = '/etc/catalyst/catalyst.conf'

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     be43719867e694a08699aa07c2f5f519df26b59b
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 18 04:33:52 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Apr 23 18:20:35 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=be437198

catalyst: Add read/write_lock contextmanagers

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/lock.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index ab005163..e31745b2 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -1,6 +1,8 @@
 
 import os
 
+from contextlib import contextmanager
+
 from snakeoil import fileutils
 from snakeoil import osutils
 from catalyst.fileops import ensure_dirs
@@ -36,3 +38,21 @@ class LockDir(Lock):
         lockfile = os.path.join(lockdir, '.catalyst_lock')
 
         Lock.__init__(self, lockfile)
+
+@contextmanager
+def read_lock(filename):
+    lock = Lock(filename)
+    lock.read_lock()
+    try:
+        yield
+    finally:
+        lock.unlock()
+
+@contextmanager
+def write_lock(filename):
+    lock = Lock(filename)
+    lock.write_lock()
+    try:
+        yield
+    finally:
+        lock.unlock()

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     562a9dddee7038804e6c17d8da79262ad45deaf5
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 18 01:27:31 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Apr 23 18:20:35 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=562a9ddd

catalyst: Add a Lock class and refactor LockDir

This allows us to make lock files directly.

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/lock.py | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/catalyst/lock.py b/catalyst/lock.py
index 5e039fae..ab005163 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -8,17 +8,14 @@ from catalyst.fileops import ensure_dirs
 
 LockInUse = osutils.LockException
 
-
-class LockDir():
-    """An object that creates locks inside dirs"""
-
-    def __init__(self, lockdir):
-        self.gid = 250
-        self.lockfile = os.path.join(lockdir, '.catalyst_lock')
-        ensure_dirs(lockdir)
-        fileutils.touch(self.lockfile, mode=0o664)
-        os.chown(self.lockfile, -1, self.gid)
-        self.lock = osutils.FsLock(self.lockfile)
+class Lock:
+    """
+    A fnctl-based filesystem lock
+    """
+    def __init__(self, lockfile):
+        fileutils.touch(lockfile, mode=0o664)
+        os.chown(lockfile, uid=-1, gid=250)
+        self.lock = osutils.FsLock(lockfile)
 
     def read_lock(self):
         self.lock.acquire_read_lock()
@@ -29,3 +26,13 @@ class LockDir():
     def unlock(self):
         # Releasing a write lock is the same as a read lock.
         self.lock.release_write_lock()
+
+class LockDir(Lock):
+    """
+    A fnctl-based filesystem lock in a directory
+    """
+    def __init__(self, lockdir):
+        ensure_dirs(lockdir)
+        lockfile = os.path.join(lockdir, '.catalyst_lock')
+
+        Lock.__init__(self, lockfile)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     ed0de74506f5ce9741fe61c6c1e46701ee7ca3f0
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 19 18:03:44 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Apr 23 18:20:35 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ed0de745

catalyst: Add a function to get full path of executable

Just a simple wrapper around shutil.which().

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/support.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/catalyst/support.py b/catalyst/support.py
index 988a81f5..0925af47 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -3,6 +3,7 @@ import glob
 import sys
 import os
 import re
+import shutil
 import time
 from subprocess import Popen
 
@@ -17,6 +18,11 @@ class CatalystError(Exception):
         if message:
             log.error('CatalystError: %s', message, exc_info=print_traceback)
 
+def command(name):
+    c = shutil.which(name)
+    if not c:
+        raise CatalystError(f'"{name}" not found or is not executable')
+    return c
 
 def cmd(mycmd, env=None, debug=False, fail_func=None):
     """Run the external |mycmd|.

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     5b1793bc3d71b81c70a83b8fd92c0069c490f64d
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Mon May 18 23:36:41 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 01:49:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5b1793bc

catalyst: Delete unused {required,valid}_build_targets

Fixes: 11423a21603e ([2 of 3] Update module loading for the new python structure)
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index f6bc1e14..0da717f1 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -8,25 +8,6 @@ from DeComp.definitions import COMPRESSOR_PROGRAM_OPTIONS, XATTRS_OPTIONS
 from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS, LIST_XATTRS_OPTIONS
 
 
-# these should never be touched
-required_build_targets = [
-    "generic_stage_target",
-    "targetbase",
-]
-
-# new build types should be added here
-valid_build_targets = [
-    "embedded_target",
-    "livecd_stage1_target",
-    "livecd_stage2_target",
-    "netboot_target",
-    "snapshot_target",
-    "stage1_target",
-    "stage2_target",
-    "stage3_target",
-    "stage4_target",
-]
-
 required_config_file_values = [
     "distdir",
     "portdir",

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     3b721ac38a6d86c952e8d65e0bba942aa49a3628
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Mon May 18 22:41:54 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 01:49:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3b721ac3

catalyst: Switch internal snapshot option parsing to SpecParser

The --snapshot/-s option internally creates a .spec file but uses the
ConfigParser (nominally used for parsing catalyst.conf) rather than
SpecParser (used for parsing .spec files) and as a result has to use
'=' rather than ':' as the key/value delimiter.

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/main.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index b01d7a6a..be06ccd7 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -291,8 +291,8 @@ def _main(parser, opts):
 
     mycmdline = list()
     if opts.snapshot:
-        mycmdline.append('target=snapshot')
-        mycmdline.append('snapshot_treeish=' + opts.snapshot)
+        mycmdline.append('target: snapshot')
+        mycmdline.append('snapshot_treeish: ' + opts.snapshot)
 
     conf_values['DEBUG'] = opts.debug
     conf_values['VERBOSE'] = opts.debug or opts.verbose
@@ -354,7 +354,7 @@ def _main(parser, opts):
 
     if mycmdline:
         try:
-            cmdline = catalyst.config.ConfigParser()
+            cmdline = catalyst.config.SpecParser()
             cmdline.parse_lines(mycmdline)
             addlargs.update(cmdline.get_values())
         except CatalystError:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     f40238e0c5644db5dcaaab9f0305cc9d62b144a1
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Mon May 18 22:52:36 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 01:49:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f40238e0

catalyst: Remove PythonDir setting

Was used to find the arch directory containing, e.g., sparc.py, but all
that code is gone now after the modules were converted to TOML.

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 1 -
 catalyst/main.py     | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 404f4892..f6bc1e14 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -64,7 +64,6 @@ confdefaults = {
     "options": set(),
     "pkgdir": "/var/cache/binpkgs",
     "port_tmpdir": "/var/tmp/portage",
-    "PythonDir": "./catalyst",
     "repo_basedir": "/var/db/repos",
     "repo_name": "gentoo",
     "repos": "%(storedir)s/repos",

diff --git a/catalyst/main.py b/catalyst/main.py
index bad712fa..b01d7a6a 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -55,9 +55,6 @@ def parse_config(config_files):
         else:
             conf_values[x] = confdefaults[x]
 
-    # add our python base directory to use for loading target arch's
-    conf_values["PythonDir"] = os.path.dirname(os.path.realpath(__file__))
-
     # print out any options messages
     for opt in conf_values['options']:
         if opt in option_messages:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     6ed67c24fe3bb8e3baa8c936249e4507c45f86a2
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Mon May 18 23:48:22 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 01:49:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=6ed67c24

catalyst: Remove spec file options from valid_config_file_values

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index ccabd88d..2f2c907f 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -17,12 +17,7 @@ required_config_file_values = [
 
 valid_config_file_values = required_config_file_values[:]
 valid_config_file_values.extend([
-    "compression_mode",
-    "compressor_arch",
-    "compressor_options",
-    "decompressor_search_order",
     "digests",
-    "distcc",
     "envscript",
     "options",
 ])

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     59a3b592d0ede9f0bb6e323f58443acb6ef3b297
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Mon May 18 23:49:54 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 01:49:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=59a3b592

catalyst: Add a missing options to valid_config_file_values

And remove required_config_file_values since we want to support running
catalyst without a config file.

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 2f2c907f..412cb956 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -8,18 +8,20 @@ from DeComp.definitions import COMPRESSOR_PROGRAM_OPTIONS, XATTRS_OPTIONS
 from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS, LIST_XATTRS_OPTIONS
 
 
-required_config_file_values = [
-    "distdir",
-    "portdir",
-    "sharedir",
-    "storedir",
-]
-
-valid_config_file_values = required_config_file_values[:]
-valid_config_file_values.extend([
+valid_config_file_values = frozenset([
     "digests",
+    "distdir",
     "envscript",
     "options",
+    "port_logdir",
+    "repo_basedir",
+    "repo_name",
+    "repos",
+    "sharedir",
+    "storedir",
+    "target_distdir",
+    "target_pkgdir",
+    "var_tmpfs_portage",
 ])
 
 confdefaults = {

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     fd731821ae4c7b15c7fedfad794bd3b0d05ebb7a
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Mon May 18 23:34:13 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 01:49:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=fd731821

catalyst: Disallow config file options in spec files

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/support.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 0925af47..c4a5c797 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -8,7 +8,6 @@ import time
 from subprocess import Popen
 
 from catalyst import log
-from catalyst.defaults import valid_config_file_values
 
 BASH_BINARY = "/bin/bash"
 
@@ -211,7 +210,7 @@ def addl_arg_parse(myspec, addlargs, requiredspec, validspec):
     "helper function to help targets parse additional arguments"
     messages = []
     for x in addlargs.keys():
-        if x not in validspec and x not in valid_config_file_values and x not in requiredspec:
+        if x not in validspec and x not in requiredspec:
             messages.append("Argument \""+x+"\" not recognized.")
         else:
             myspec[x] = addlargs[x]

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     4c97ff75f71b5fc86b0a3b89bc960966fd0959a6
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Mon May 18 23:43:52 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 01:49:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4c97ff75

catalyst: Disallow DEBUG and VERBOSE in spec files

Catalyst has --debug/-d and --verbose/-v options for this.

Since conf_values is assigned in a very confusing manner in main.py, I
suspect these values were added to the list due to a misunderstanding.

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 0da717f1..ccabd88d 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -20,13 +20,11 @@ valid_config_file_values.extend([
     "compression_mode",
     "compressor_arch",
     "compressor_options",
-    "DEBUG",
     "decompressor_search_order",
     "digests",
     "distcc",
     "envscript",
     "options",
-    "VERBOSE",
 ])
 
 confdefaults = {

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     8443f89f3c5dfbc011a333e90bfef772a95e47cd
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Tue May 19 00:13:14 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 01:49:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=8443f89f

catalyst: Verify config options against valid_config_file_values

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/main.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 159fe454..543895c6 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -15,7 +15,8 @@ from DeComp.contents import ContentsMap
 
 from catalyst import log
 import catalyst.config
-from catalyst.defaults import confdefaults, option_messages, DEFAULT_CONFIG_FILE
+from catalyst.defaults import (confdefaults, option_messages,
+                               DEFAULT_CONFIG_FILE, valid_config_file_values)
 from catalyst.support import CatalystError
 from catalyst.version import get_version
 
@@ -34,7 +35,12 @@ def parse_config(config_files):
     for config_file in config_files:
         log.notice('Loading configuration file: %s', config_file)
         try:
-            conf_values.update(toml.load(config_file))
+            config = toml.load(config_file)
+            for key in config:
+                if key not in valid_config_file_values:
+                    log.critical("Unknown option '%s' in config file %s",
+                                 key, config_file)
+            conf_values.update(config)
         except Exception as e:
             log.critical('Could not find parse configuration file: %s: %s',
                          config_file, e)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     016c8460c88c89e55f85e34664fe7928b0f1c6a6
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Tue May 19 19:14:29 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu May 21 20:26:15 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=016c8460

catalyst: Drop ConfigParser

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/config.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/catalyst/config.py b/catalyst/config.py
index b527ada0..e1963f71 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -118,14 +118,3 @@ class SpecParser(ParserBase):
     def __init__(self, filename=""):
         if filename:
             self.parse_file(filename)
-
-
-class ConfigParser(ParserBase):
-
-    key_value_separator = '='
-    multiple_values = False
-    empty_values = True
-
-    def __init__(self, filename=""):
-        if filename:
-            self.parse_file(filename)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     2c82d8a8833edd200db1bf6ae36807376bbb61b8
Author:     Daniel Cordero <catalyst <AT> 0xdc <DOT> io>
AuthorDate: Fri May 29 19:04:31 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Jun  5 21:09:29 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=2c82d8a8

catalyst: Allow compression_mode in config file

Allow overriding compression mode globally from the catalyst config.

Otherwise, if one wanted a different compression method than the
default, one would need to add compression_mode to every spec file.

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/defaults.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 14f671fe..c153fcc4 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -9,6 +9,7 @@ from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS, LIST_XATTRS_OPTIONS
 
 
 valid_config_file_values = frozenset([
+    "compression_mode",
     "digests",
     "distdir",
     "envscript",

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     b3f0a3535e3550ed7d0dbd3100e43b82e9b34c3b
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Oct  8 21:15:03 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Oct  8 21:15:03 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b3f0a353

catalyst: Fix typo

Fixes: 3c5ef2e7 ("log: new logging module to standardize catalyst output")
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/log.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/log.py b/catalyst/log.py
index 9d534355..ee124392 100644
--- a/catalyst/log.py
+++ b/catalyst/log.py
@@ -74,7 +74,7 @@ class CatalystFormatter(logging.Formatter):
     @staticmethod
     def detect_color():
         """Figure out whether the runtime env wants color"""
-        if 'NOCOLOR' is os.environ:
+        if 'NOCOLOR' in os.environ:
             return False
         return os.isatty(sys.stdout.fileno())
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     cf50ce228e766252fe0ea901f86671fed6a99cfa
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 28 20:50:00 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Oct 30 22:40:52 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=cf50ce22

catalyst: Replace pathcompare()

Modern Python allows us to do this in a much cleaner way.

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/support.py | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index f49315a5..4458ed20 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -5,6 +5,7 @@ import os
 import re
 import shutil
 import time
+from pathlib import Path
 from subprocess import Popen
 
 from catalyst import log
@@ -179,31 +180,20 @@ def read_makeconf(mymakeconffile):
         return makeconf
 
 
-def pathcompare(path1, path2):
-    # Change double slashes to slash
-    path1 = re.sub(r"//", r"/", path1)
-    path2 = re.sub(r"//", r"/", path2)
-    # Removing ending slash
-    path1 = re.sub("/$", "", path1)
-    path2 = re.sub("/$", "", path2)
-
-    if path1 == path2:
-        return 1
-    return 0
-
-
 def ismount(path):
     """Like os.path.ismount, but also support bind mounts"""
     if os.path.ismount(path):
-        return 1
+        return True
+
     a = os.popen("mount")
     mylines = a.readlines()
     a.close()
     for line in mylines:
         mysplit = line.split()
-        if pathcompare(path, mysplit[2]):
-            return 1
-    return 0
+        if Path(path) == Path(mysplit[2]):
+            return True
+
+    return False
 
 
 def addl_arg_parse(myspec, addlargs, requiredspec, validspec):

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     4bcc64dcf2b27ab3d0effed0fa2516e3513a9171
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 28 21:59:17 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Oct 30 22:40:52 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4bcc64dc

catalyst: Add and use namespace context manager

Wraps snakeoil's simple_unshare; returns to the previous namespaces on
context exit. Will be used by the next commit.

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/context.py | 32 ++++++++++++++++++++++++++++++++
 catalyst/main.py    | 17 +++++++----------
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/catalyst/context.py b/catalyst/context.py
new file mode 100644
index 00000000..8a58f33d
--- /dev/null
+++ b/catalyst/context.py
@@ -0,0 +1,32 @@
+
+import contextlib
+import os
+
+from snakeoil.process.namespaces import setns, simple_unshare
+
+@contextlib.contextmanager
+def namespace(mount=False, uts=False, ipc=False, net=False, pid=False,
+              user=False, hostname=None):
+    namespaces = {
+        (mount, "mnt"):  None,
+        (uts,   "uts"):  None,
+        (ipc,   "ipc"):  None,
+        (net,   "net"):  None,
+        (pid,   "pid"):  None,
+        (user,  "user"): None,
+    }
+
+    # Save fds of current namespaces
+    for ns in [ns for ns in namespaces if ns[0]]:
+        fp = open(f"/proc/self/ns/{ns[1]}")
+        namespaces[ns] = fp
+
+    simple_unshare(mount=mount, uts=uts, ipc=ipc, net=net, pid=pid, user=user,
+                   hostname=hostname)
+    try:
+        yield
+    finally:
+        for ns in [ns for ns in namespaces if ns[0]]:
+            fp = namespaces[ns]
+            setns(fp.fileno(), 0)
+            fp.close()

diff --git a/catalyst/main.py b/catalyst/main.py
index 543895c6..93a4a0d3 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -7,14 +7,13 @@ import textwrap
 
 import toml
 
-from snakeoil.process import namespaces
-
 from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
                                 CONTENTS_DEFINITIONS)
 from DeComp.contents import ContentsMap
 
 from catalyst import log
 import catalyst.config
+from catalyst.context import namespace
 from catalyst.defaults import (confdefaults, option_messages,
                                DEFAULT_CONFIG_FILE, valid_config_file_values)
 from catalyst.support import CatalystError
@@ -356,15 +355,13 @@ def _main(parser, opts):
     # use pid & user namespaces, but snakeoil's namespace module has signal
     # transfer issues (CTRL+C doesn't propagate), and user namespaces need
     # more work due to Gentoo build process (uses sudo/root/portage).
-    namespaces.simple_unshare(
-        mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
-        hostname='catalyst')
+    with namespace(mount=True, uts=True, ipc=True, hostname='catalyst'):
+        # everything is setup, so the build is a go
+        try:
+            success = build_target(addlargs)
+        except KeyboardInterrupt:
+            log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
 
-    # everything is setup, so the build is a go
-    try:
-        success = build_target(addlargs)
-    except KeyboardInterrupt:
-        log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
     if not success:
         sys.exit(2)
     sys.exit(0)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     dccfafea4e8b4622a7527a764d581ac0123b107e
Author:     Felix Bier <Felix.Bier <AT> rohde-schwarz <DOT> com>
AuthorDate: Mon Nov  9 19:36:33 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sat Nov 14 16:34:57 2020 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=dccfafea

catalyst: Restore root and cwd after exiting mount namespace

This commit saves the file descriptor of /proc/self/{root,cwd}
before entering into the new mount namespace. When restoring the
previous mount namespace, it restores /proc/self/{root,cwd}
based on the saved file descriptors.

Without this change, catalyst cannot be run in a chroot when
using the recent changes regarding mount namespaces: After the
mount namespace has been exited, /proc/self/root points to the "/"
of the host system, not the "/" of the chroot. Therefore, the
cleanup phase of catalyst runs outside of the chroot.

The code is similar to how nsenter(1) sets root and cwd:
https://git.kernel.org/pub/scm/utils/util-linux/util-linux.git/tree/sys-utils/nsenter.c#n452

Tested in a Gentoo chroot and in Gentoo VM (non-chroot).

Signed-off-by: Felix Bier <felix.bier <AT> rohde-schwarz.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/context.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/catalyst/context.py b/catalyst/context.py
index 8a58f33d..01a6d930 100644
--- a/catalyst/context.py
+++ b/catalyst/context.py
@@ -16,11 +16,21 @@ def namespace(mount=False, uts=False, ipc=False, net=False, pid=False,
         (user,  "user"): None,
     }
 
+    dirs = {
+        "root": None,
+        "cwd":  None,
+    }
+
     # Save fds of current namespaces
     for ns in [ns for ns in namespaces if ns[0]]:
         fp = open(f"/proc/self/ns/{ns[1]}")
         namespaces[ns] = fp
 
+    # Save fds of current directories
+    if mount:
+        for d in dirs:
+            dirs[d] = os.open(f"/proc/self/{d}", os.O_RDONLY)
+
     simple_unshare(mount=mount, uts=uts, ipc=ipc, net=net, pid=pid, user=user,
                    hostname=hostname)
     try:
@@ -30,3 +40,15 @@ def namespace(mount=False, uts=False, ipc=False, net=False, pid=False,
             fp = namespaces[ns]
             setns(fp.fileno(), 0)
             fp.close()
+
+        if mount:
+            # Restore original root and cwd. Since we cannot directly chroot to
+            # a fd, first change the current directory to the fd of the
+            # original root, then chroot to "."
+
+            os.fchdir(dirs["root"])
+            os.chroot(".")
+            os.fchdir(dirs["cwd"])
+
+            for fd in dirs.values():
+                os.close(fd)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     b8b6de7bc84677d9ae13cf8bfff2c0e6a281ff75
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 24 02:34:06 2021 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Jan 28 02:06:46 2021 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b8b6de7b

catalyst: Remove fallback make.conf parsing code

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/support.py | 42 +++---------------------------------------
 1 file changed, 3 insertions(+), 39 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index f3a865a7..fa652987 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -2,7 +2,6 @@
 import glob
 import sys
 import os
-import re
 import shutil
 import time
 from pathlib import Path
@@ -12,6 +11,8 @@ import libmount
 
 from portage.repository.config import RepoConfig
 
+from snakeoil.bash import read_bash_dict
+
 from catalyst import log
 
 BASH_BINARY = "/bin/bash"
@@ -135,47 +136,10 @@ defined are not preserved. In other words, "foo", "bar", "oni" ordering is prese
                                     print_traceback=True)
 
 
-def parse_makeconf(mylines):
-    mymakeconf = {}
-    pos = 0
-    pat = re.compile("([0-9a-zA-Z_]*)=(.*)")
-    while pos < len(mylines):
-        if len(mylines[pos]) <= 1:
-            # skip blanks
-            pos += 1
-            continue
-        if mylines[pos][0] in ["#", " ", "\t"]:
-            # skip indented lines, comments
-            pos += 1
-            continue
-        else:
-            myline = mylines[pos]
-            mobj = pat.match(myline)
-            pos += 1
-            if mobj.group(2):
-                clean_string = re.sub(r"\"", r"", mobj.group(2))
-                mymakeconf[mobj.group(1)] = clean_string
-    return mymakeconf
-
-
 def read_makeconf(mymakeconffile):
     if os.path.exists(mymakeconffile):
         try:
-            try:
-                import snakeoil.bash  # import snakeoil.fileutils
-                return snakeoil.bash.read_bash_dict(mymakeconffile, sourcing_command="source")
-            except ImportError:
-                try:
-                    import portage.util
-                    return portage.util.getconfig(mymakeconffile, tolerant=1, allow_sourcing=True)
-                except Exception:
-                    try:
-                        import portage_util
-                        return portage_util.getconfig(mymakeconffile, tolerant=1, allow_sourcing=True)
-                    except ImportError:
-                        with open(mymakeconffile, "r") as myf:
-                            mylines = myf.readlines()
-                        return parse_makeconf(mylines)
+            return read_bash_dict(mymakeconffile, sourcing_command="source")
         except Exception:
             raise CatalystError("Could not parse make.conf file " +
                                 mymakeconffile, print_traceback=True)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     caf55a942580e02e66ed846f5c3ab4ad5ab8846f
Author:     Felix Bier <Felix.Bier <AT> rohde-schwarz <DOT> com>
AuthorDate: Thu Feb  4 00:37:06 2021 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sat Feb 20 21:27:29 2021 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=caf55a94

Extend get_repo_name to handle squashed repos

This commit extends the method get_repo_name to also handle
squashed repos. This is done by mounting the squash file to
a temporary directory and then extracting the repository from
that directory with the already existing code.

This is motivated by wanting to mount each repo
to e.g. /var/db/repos/<repo-name> in a later commit.
For squashed repos, we don't know <repo-name> without
mounting the repo first. For this reason, it is mounted to
a temporary directory first to extract <repo-name>.

Signed-off-by: Felix Bier <felix.bier <AT> rohde-schwarz.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/support.py | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index fc50fa34..37d53bc4 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -10,10 +10,12 @@ from subprocess import Popen
 import libmount
 
 from portage.repository.config import RepoConfig
+from tempfile import TemporaryDirectory
 
 from snakeoil.bash import read_bash_dict
 
 from catalyst import log
+from catalyst.context import namespace
 
 BASH_BINARY = "/bin/bash"
 
@@ -148,7 +150,7 @@ def read_makeconf(mymakeconffile):
         return makeconf
 
 
-def get_repo_name(repo_path):
+def get_repo_name_from_dir(repo_path):
     """ Get the name of the repo at the given repo_path.
 
          References:
@@ -164,6 +166,38 @@ def get_repo_name(repo_path):
     return repo_config.name
 
 
+def get_repo_name_from_squash(repo_squash_path):
+    """ Get the name of the repo at the given repo_squash_path.
+        To obtain the name, the squash file is mounted to a temporary directory.
+    """
+
+    repo_name = None
+
+    # Mount squash file to temp directory in separate mount namespace
+    with TemporaryDirectory() as temp, namespace(mount=True):
+        try:
+            source = str(repo_squash_path)
+            target = str(temp)
+            fstype = 'squashfs'
+            options = 'ro,loop'
+            cxt = libmount.Context(source=source, target=target,
+                                   fstype=fstype, options=options)
+            cxt.mount()
+            repo_name = get_repo_name_from_dir(target)
+
+        except Exception as e:
+            raise CatalystError(f"Couldn't mount: {source}, {e}") from e
+
+    return repo_name
+
+
+def get_repo_name(repo_path):
+    if not Path(repo_path).is_dir():
+        return get_repo_name_from_squash(repo_path)
+
+    return get_repo_name_from_dir(repo_path)
+
+
 def ismount(path):
     """Like os.path.ismount, but also support bind mounts"""
     path = Path(path)

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     4fd2ac23250ab2ac1f6a506ee433f466a4f9e026
Author:     Felix Bier <Felix.Bier <AT> rohde-schwarz <DOT> com>
AuthorDate: Sat Feb 13 23:18:17 2021 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sat Feb 20 21:27:29 2021 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4fd2ac23

Enable recursive globbing for clear_path

This commit enables recursive globbing in clear_path, allowing the
usage of '**' to match an arbitrary number of sub-directories.

Before this commit, clear_path used only non-recursive globbing. This
allowed to use '*' to expand names within one directory, e.g. '/a/*/c'
can expand to '/a/b/c', but not '/a/b/b/c'. With this commit, '/a/**/c'
can be used to expand to '/a/b/c', '/a/b/b/c', '/a/b/b/b/c' etc.

This is motivated by wanting to recursively delete all occurences of a
filename with the 'stage4/rm' entry of a spec file. The '/rm' entries
are processed with 'clear_path' in the existing code.

Additionally, 'glob.glob' is replaced with 'glob.iglob',
which returns the same files as 'glob.glob', but as an iterator
instead of as a list (so it no longer necessary to hold
all matches in memory at once).

Recursive globbing has been added in Python 3.5.

References:
https://docs.python.org/3/library/glob.html#glob.glob
https://docs.python.org/3/library/glob.html#glob.iglob

Signed-off-by: Felix Bier <felix.bier <AT> rohde-schwarz.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/fileops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 5c6f5cd8..59525420 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -99,7 +99,7 @@ def clear_dir(target, mode=0o755, remove=False,
 
 def clear_path(target_path):
     """Nuke |target_path| regardless of it being a dir, file or glob."""
-    targets = glob.glob(target_path)
+    targets = glob.iglob(target_path, recursive=True)
     for path in targets:
         clear_dir(path, remove=True)
 

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     fbe56d75989fabf29e7b642515df69737cf62cc1
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Jun  9 04:32:08 2021 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Jun  9 04:32:08 2021 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=fbe56d75

catalyst: Remove clear_dir's never used clear_nondir parameter

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/fileops.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 59525420..4252285e 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -46,8 +46,7 @@ def ensure_dirs(path, gid=-1, uid=-1, mode=0o755, minimal=True,
     return succeeded
 
 
-def clear_dir(target, mode=0o755, remove=False,
-              clear_nondir=True):
+def clear_dir(target, mode=0o755, remove=False):
     '''Universal directory clearing function
 
     @target: string, path to be cleared or removed
@@ -74,12 +73,8 @@ def clear_dir(target, mode=0o755, remove=False,
             log.error('clear_dir failed', exc_info=True)
             return False
     elif os.path.exists(target):
-        if clear_nondir:
-            log.debug("Clearing (unlinking) non-directory: %s", target)
-            os.unlink(target)
-        else:
-            log.info('clear_dir failed: %s: is not a directory', target)
-            return False
+        log.debug("Clearing (unlinking) non-directory: %s", target)
+        os.unlink(target)
     else:
         log.debug("Conditions not met to clear: %s", target)
         log.debug("                      isdir: %s", os.path.isdir(target))

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     1a8d6cd3d31f960755c3237679539d9cab175bc6
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 16 22:33:17 2022 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Feb 16 22:33:52 2022 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1a8d6cd3

catalyst: Add .sha256 to list of ignored extensions

Otherwise the presence of a .sha256 file would cause this error:

ERROR:catalyst:CatalystError: Ambiguous Filename: /home/catalyst/builds/default/stage3-ia64-openrc-20220216T032203Z
ERROR:catalyst:Please specify the correct extension as well

Fixes: 7457cd3b ("catalyst: generate .sha256 file if any digest is enabled")
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/support.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index 37d53bc4..a69566d2 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -77,8 +77,11 @@ def file_check(filepath, extensions=None):
     # so check if there are files of that name with an extension
     files = glob.glob("%s.*" % filepath)
     # remove any false positive files
-    files = [x for x in files if not x.endswith(
-        ".CONTENTS") and not x.endswith(".CONTENTS.gz") and not x.endswith(".DIGESTS")]
+    files = [x for x in files if
+             not x.endswith( ".CONTENTS") and
+             not x.endswith(".CONTENTS.gz") and
+             not x.endswith(".DIGESTS") and
+             not x.endswith(".sha256")]
     if len(files) == 1:
         return files[0]
     if len(files) > 1:

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Matt Turner]

commit:     f365eb43a81989ad24f75995df97552e08d3e7ad
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 17 23:41:33 2022 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Nov 17 23:42:24 2022 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f365eb43

catalyst: Remove stray space

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 catalyst/support.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index a69566d2..6945d053 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -78,7 +78,7 @@ def file_check(filepath, extensions=None):
     files = glob.glob("%s.*" % filepath)
     # remove any false positive files
     files = [x for x in files if
-             not x.endswith( ".CONTENTS") and
+             not x.endswith(".CONTENTS") and
              not x.endswith(".CONTENTS.gz") and
              not x.endswith(".DIGESTS") and
              not x.endswith(".sha256")]

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Andreas K. Hüttel]

commit:     27035bc1a25e57fc3127db865b999acf15bd1697
Author:     Doug Freed <dwfreed <AT> mtu <DOT> edu>
AuthorDate: Mon Jul 29 22:24:44 2024 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Tue Jul 30 11:08:12 2024 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=27035bc1

log: hide our functions and include line numbers

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 catalyst/log.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/catalyst/log.py b/catalyst/log.py
index ee124392..fb852f2f 100644
--- a/catalyst/log.py
+++ b/catalyst/log.py
@@ -20,6 +20,11 @@ class CatalystLogger(logging.Logger):
 
     def _log(self, level, msg, args, **kwargs):
         """If given a multiline message, split it"""
+
+        # Increment stacklevel to hide this function call
+        stacklevel = kwargs.get("stacklevel", 1)
+        kwargs["stacklevel"] = stacklevel + 1
+
         # We have to interpolate it first in case they spread things out
         # over multiple lines like: Bad Thing:\n%s\nGoodbye!
         msg %= args
@@ -44,11 +49,21 @@ logging.addLevelName(NOTICE, 'NOTICE')
 # The API we expose to consumers.
 def notice(msg, *args, **kwargs):
     """Log a notice message"""
+
+    # Increment stacklevel to hide this function call
+    stacklevel = kwargs.get("stacklevel", 1)
+    kwargs["stacklevel"] = stacklevel + 1
+
     logger.log(NOTICE, msg, *args, **kwargs)
 
 
 def critical(msg, *args, **kwargs):
     """Log a critical message and then exit"""
+
+    # Increment stacklevel to hide this function call
+    stacklevel = kwargs.get("stacklevel", 1)
+    kwargs["stacklevel"] = stacklevel + 1
+
     status = kwargs.pop('status', 1)
     logger.critical(msg, *args, **kwargs)
     sys.exit(status)
@@ -110,7 +125,7 @@ def setup_logging(level, output=None, debug=False, color=None):
     # The good stuff.
     fmt = '%(asctime)s: %(levelname)-8s: '
     if debug:
-        fmt += '%(filename)s:%(funcName)s: '
+        fmt += '%(filename)s:%(funcName)s:%(lineno)d: '
     fmt += '%(message)s'
 
     # Figure out where to send the log output.

[gentoo-commits] proj/catalyst:master commit in: catalyst/

[Andreas K. Hüttel]

commit:     e34c79404c1dbe23343214132f92b83ec82cec0c
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 12 12:43:42 2024 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Sat Oct 12 12:43:42 2024 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e34c7940

Set version for a release

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 catalyst/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalyst/version.py b/catalyst/version.py
index dbada51a..52fa2f27 100644
--- a/catalyst/version.py
+++ b/catalyst/version.py
@@ -5,7 +5,7 @@ import os
 
 from snakeoil.version import get_git_version as get_ver
 
-__version__ = "3.0.7"
+__version__ = "4.0.0"
 _ver = None