* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
2015-01-01 5:59 [gentoo-commits] proj/catalyst:pending " Brian Dolbec
@ 2015-02-26 4:12 ` Brian Dolbec
0 siblings, 0 replies; 16+ messages in thread
From: Brian Dolbec @ 2015-02-26 4:12 UTC (permalink / raw
To: gentoo-commits
commit: e78b8bef22297f61a237aa4a7bfc4562b76064a7
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 12 04:06:01 2013 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 1 05:58:06 2015 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=e78b8bef
Break out more repeated (path1 + path2)'s...
Just do it once and use the temp variable.
Comment out some debug print's.
Fix options conversion for export to bash.
---
catalyst/base/stagebase.py | 43 ++++++++++++++++++++++---------------------
catalyst/defaults.py | 2 ++
2 files changed, 24 insertions(+), 21 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index bebb5dc..b44a957 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -631,11 +631,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
"""
self.setup_environment()
- if os.path.exists(self.settings["sharedir"]+\
- "/targets/support/kill-chroot-pids.sh"):
- cmd("/bin/bash "+self.settings["sharedir"]+\
- "/targets/support/kill-chroot-pids.sh",\
- "kill-chroot-pids script failed.",env=self.env)
+ killcmd = normpath(self.settings["sharedir"] +
+ self.settings["shdir"] + "/support/kill-chroot-pids.sh")
+ if os.path.exists(killcmd):
+ cmd(killcmd, "kill-chroot-pids script failed.",env=self.env)
def mount_safety_check(self):
"""
@@ -647,10 +646,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not os.path.exists(self.settings["chroot_path"]):
return
- print "self.mounts =", self.mounts
+ #print "self.mounts =", self.mounts
for x in self.mounts:
target = normpath(self.settings["chroot_path"] + self.target_mounts[x])
- print "mount_safety_check() x =", x, target
+ #print "mount_safety_check() x =", x, target
if not os.path.exists(target):
continue
@@ -1020,8 +1019,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
"Couldn't umount one or more bind-mounts; aborting for safety."
def chroot_setup(self):
- self.makeconf=read_makeconf(self.settings["chroot_path"]+\
- "/etc/portage/make.conf")
+ self.makeconf=read_makeconf(normpath(self.settings["chroot_path"]+
+ self.settings["make.conf"]))
self.override_cbuild()
self.override_chost()
self.override_cflags()
@@ -1035,8 +1034,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
else:
print "Setting up chroot..."
- #self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/portage/make.conf")
-
cmd("cp /etc/resolv.conf " + self.settings["chroot_path"] + "/etc/",
"Could not copy resolv.conf into place.",env=self.env)
@@ -1070,10 +1067,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
"Could not copy /etc/hosts",env=self.env)
""" Modify and write out make.conf (for the chroot) """
- cmd("rm -f "+self.settings["chroot_path"]+"/etc/portage/make.conf",\
- "Could not remove "+self.settings["chroot_path"]+\
- "/etc/portage/make.conf",env=self.env)
- myf=open(self.settings["chroot_path"]+"/etc/portage/make.conf","w")
+ makepath = normpath(self.settings["chroot_path"] +
+ self.settings["make.conf"])
+ cmd("rm -f " + makepath,\
+ "Could not remove " + makepath, env=self.env)
+ myf=open(makepath, "w")
myf.write("# These settings were set by the catalyst build script that automatically\n# built this stage.\n")
myf.write("# Please consult /usr/share/portage/config/make.conf.example for a more\n# detailed example.\n")
if "CFLAGS" in self.settings:
@@ -1123,10 +1121,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
myf.write('PORTDIR_OVERLAY="/usr/local/portage"\n')
myf.close()
- cmd("cp "+self.settings["chroot_path"]+"/etc/portage/make.conf "+\
- self.settings["chroot_path"]+"/etc/portage/make.conf.catalyst",\
- "Could not backup /etc/portage/make.conf",env=self.env)
- touch(chroot_setup_resume)
+ makepath = normpath(self.settings["chroot_path"] +
+ self.settings["make.conf"])
+ cmd("cp " + makepath + " " + makepath + ".catalyst",\
+ "Could not backup " + self.settings["make.conf"],env=self.env)
+ touch(self.settings["autoresume_path"]+"chroot_setup")
def fsscript(self):
fsscript_resume = pjoin(self.settings["autoresume_path"], "fsscript")
@@ -1289,6 +1288,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
else:
try:
if os.path.exists(self.settings["controller_file"]):
+ print "run_local() starting controller script..."
cmd(self.settings["controller_file"]+" run",\
"run script failed.",env=self.env)
touch(run_local_resume)
@@ -1324,11 +1324,12 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.env[varname]=string.join(self.settings[x])
elif type(self.settings[x])==types.BooleanType:
if self.settings[x]:
- self.env[varname]="true"
+ self.env[varname] = "true"
else:
- self.env[varname]="false"
+ self.env[varname] = "false"
if "makeopts" in self.settings:
self.env["MAKEOPTS"]=self.settings["makeopts"]
+ #print "setup_environment(); env =", self.env
def run(self):
self.chroot_lock.write_lock()
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index f6b2163..ce7e919 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -25,12 +25,14 @@ confdefaults={
"hash_function": "crc32",
"icecream": "/var/cache/icecream",
"local_overlay": "/usr/local/portage",
+ "make.conf": "/etc/portage/make.conf",
"options": "",
"packagedir": "/usr/portage/packages",
"portdir": "/usr/portage",
"port_tmpdir": "/var/tmp/portage",
"repo_name": "portage",
"sharedir": "/usr/lib/catalyst",
+ "shdir": "/usr/lib/catalyst/targets/",
"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
"snapshot_name": "portage-",
"storedir": "/var/tmp/catalyst",
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
2015-01-01 5:59 [gentoo-commits] proj/catalyst:pending commit in: catalyst/base/, catalyst/ Brian Dolbec
@ 2015-02-26 20:12 ` Brian Dolbec
0 siblings, 0 replies; 16+ messages in thread
From: Brian Dolbec @ 2015-02-26 20:12 UTC (permalink / raw
To: gentoo-commits
commit: f186361602b69e1c80bea5b43098d7cde7bddc15
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 12 04:13:13 2013 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 1 05:58:06 2015 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=f1863616
Add archdir to settings
---
catalyst/base/stagebase.py | 8 +++-----
catalyst/defaults.py | 2 ++
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 96b1c19..daf8e54 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -66,17 +66,15 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.archmap = {}
self.subarchmap = {}
machinemap = {}
- arch_dir = self.settings["PythonDir"] + "/arch/"
- for x in [x[:-3] for x in os.listdir(arch_dir) if x.endswith(".py")]:
- if x == "__init__":
- continue
+ arch_dir = self.settings["archdir"] + "/"
+ for x in [x[:-3] for x in os.listdir(arch_dir) if x.endswith(".py") and x != "__init__.py"]:
try:
fh=open(arch_dir + x + ".py")
"""
This next line loads the plugin as a module and assigns it to
archmap[x]
"""
- self.archmap[x]=imp.load_module(x,fh,"../arch/" + x + ".py",
+ self.archmap[x]=imp.load_module(x,fh, arch_dir + x + ".py",
(".py", "r", imp.PY_SOURCE))
"""
This next line registers all the subarches supported in the
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 2839a3d..2f4f429 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -21,6 +21,7 @@ valid_config_file_values.extend([ "distcc", "envscript",
verbosity = 1
confdefaults={
+ "archdir": "%(PythonDir)s/arch",
"distdir": "/usr/portage/distfiles",
"hash_function": "crc32",
"icecream": "/var/cache/icecream",
@@ -30,6 +31,7 @@ confdefaults={
"packagedir": "/usr/portage/packages",
"portdir": "/usr/portage",
"port_tmpdir": "/var/tmp/portage",
+ "PythonDir": "./catalyst",
"repo_name": "portage",
"sharedir": "/usr/lib/catalyst",
"shdir": "/usr/lib/catalyst/targets/",
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2015-02-26 22:18 Brian Dolbec
0 siblings, 0 replies; 16+ messages in thread
From: Brian Dolbec @ 2015-02-26 22:18 UTC (permalink / raw
To: gentoo-commits
commit: 2f8312854d9fac7476f8f972660824941dc4c852
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 30 21:19:28 2013 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Feb 26 20:08:06 2015 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=2f831285
Move some duplicate code to fileops, extend it's capability to not re-make the directory.
---
catalyst/base/clearbase.py | 71 +++++++++++++++-------------------------------
catalyst/base/resume.py | 26 ++++-------------
catalyst/fileops.py | 43 +++++++++++++++++++++++++++-
3 files changed, 70 insertions(+), 70 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index 0ebe299..e38b1a8 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -5,7 +5,7 @@ from stat import ST_UID, ST_GID, ST_MODE
from catalyst.support import cmd, countdown
-from catalyst.fileops import ensure_dirs
+from catalyst.fileops import ensure_dirs, clear_dir
class ClearBase(object):
"""
@@ -16,68 +16,43 @@ class ClearBase(object):
self.resume = None
- def clear_autoresume(self):
+ def clear_autoresume(self, remove=False):
""" Clean resume points since they are no longer needed """
if "autoresume" in self.settings["options"]:
print "Removing AutoResume Points: ..."
self.resume.clear_all()
- def clear_chroot(self):
+ def clear_chroot(self, remove=False):
print 'Clearing the chroot path ...'
- self.clear_dir(self.settings["chroot_path"], 0755, True)
+ clear_dir(self.settings["chroot_path"], 0755, True, remove)
- def clear_packages(self):
+ def clear_packages(self, remove=False):
if "pkgcache" in self.settings["options"]:
print "purging the pkgcache ..."
- self.clear_dir(self.settings["pkgcache_path"])
+ clear_dir(self.settings["pkgcache_path"], remove=remove)
- def clear_kerncache(self):
+ def clear_kerncache(self, remove=False):
if "kerncache" in self.settings["options"]:
print "purging the kerncache ..."
- self.clear_dir(self.settings["kerncache_path"])
+ clear_dir(self.settings["kerncache_path"], remove=remove)
- def purge(self):
+ def purge(self, remove=False):
countdown(10,"Purging Caches ...")
- if any(k in self.settings["options"] for k in ("purge","purgeonly","purgetmponly")):
- print "clearing autoresume ..."
- self.clear_autoresume()
-
- print "clearing chroot ..."
- self.clear_chroot()
-
- if "PURGETMPONLY" not in self.settings:
- print "clearing package cache ..."
- self.clear_packages()
-
- print "clearing kerncache ..."
- self.clear_kerncache()
-
-
- def clear_dir(self, myemp, mode=0755, chg_flags=False):
- '''Universal directory clearing function
- '''
- if not myemp:
- return False
- if os.path.isdir(myemp):
- print "Emptying directory" , myemp
- """
- stat the dir, delete the dir, recreate the dir and set
- the proper perms and ownership
- """
- try:
- mystat=os.stat(myemp)
- """ There's no easy way to change flags recursively in python """
- if chg_flags and os.uname()[0] == "FreeBSD":
- os.system("chflags -R noschg " + myemp)
- shutil.rmtree(myemp)
- ensure_dirs(myemp, mode=mode)
- os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
- os.chmod(myemp,mystat[ST_MODE])
- except Exception as e:
- print CatalystError("clear_dir(); Exeption: %s" % str(e))
- return False
- return True
+ if any(k in self.settings["options"] for k in ("purge",
+ "purgeonly", "purgetmponly")):
+ print "purge(); clearing autoresume ..."
+ self.clear_autoresume(remove)
+
+ print "purge(); clearing chroot ..."
+ self.clear_chroot(remove)
+
+ if "purgetmponly" not in self.settings["options"]:
+ print "purge(); clearing package cache ..."
+ self.clear_packages(remove)
+
+ print "purge(); clearing kerncache ..."
+ self.clear_kerncache(remove)
diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index e42c7dc..cf495fc 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -16,7 +16,7 @@ import shutil
from stat import ST_UID, ST_GID, ST_MODE
import traceback
-from catalyst.fileops import ensure_dirs, pjoin, listdir_files
+from catalyst.fileops import ensure_dirs, pjoin, listdir_files, clear_dir
from catalyst.support import touch
@@ -139,28 +139,12 @@ class AutoResume(object):
return list(self._points)
- def clear_all(self):
+ def clear_all(self, remove=False):
'''Clear all active resume points
@return boolean
'''
- try:
- print "Emptying directory---", self.basedir
- """
- stat the dir, delete the dir, recreate the dir and set
- the proper perms and ownership
- """
- mystat=os.stat(self.basedir)
- if os.uname()[0] == "FreeBSD":
- cmd("chflags -R noschg " + self.basedir,\
- "Could not remove immutable flag for file "\
- + self.basedir)
- shutil.rmtree(self.basedir)
- ensure_dirs(self.basedir, 0755)
- os.chown(self.basedir,mystat[ST_UID],mystat[ST_GID])
- os.chmod(self.basedir,mystat[ST_MODE])
+ if clear_dir(self.basedir, mode=0755, chg_flags=True, remove=remove):
self._points = {}
- except Exception as e:
- print AutoResumeError(str(e))
- return False
- return True
+ return True
+ return False
diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index e3a4ead..245c83e 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -10,7 +10,13 @@ ensuring directories exist,... imports snakeoils osutils
functions for use throughout catalyst.
'''
-from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs, normpath,
+import os
+import shutil
+from stat import ST_UID, ST_GID, ST_MODE
+
+# NOTE: pjoin and listdir_files are imported here for export
+# to other catalyst modules
+from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs,
pjoin, listdir_files)
from catalyst.support import CatalystError
@@ -42,3 +48,38 @@ def ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True,
raise CatalystError(
"Failed to create directory: %s" % path, print_traceback=True)
return succeeded
+
+
+def clear_dir(target, mode=0755, chg_flags=False, remove=False):
+ '''Universal directory clearing function
+ '''
+ #print "fileops.clear_dir()"
+ if not target:
+ #print "fileops.clear_dir(), no target... returning"
+ return False
+ if os.path.isdir(target):
+ print "Emptying directory" , target
+ """
+ stat the dir, delete the dir, recreate the dir and set
+ the proper perms and ownership
+ """
+ try:
+ #print "fileops.clear_dir(), os.stat()"
+ mystat=os.stat(target)
+ """ There's no easy way to change flags recursively in python """
+ if chg_flags and os.uname()[0] == "FreeBSD":
+ os.system("chflags -R noschg " + target)
+ #print "fileops.clear_dir(), shutil.rmtree()"
+ shutil.rmtree(target)
+ if not remove:
+ #print "fileops.clear_dir(), ensure_dirs()"
+ ensure_dirs(target, mode=mode)
+ os.chown(target, mystat[ST_UID], mystat[ST_GID])
+ os.chmod(target, mystat[ST_MODE])
+ except Exception as e:
+ print CatalystError("clear_dir(); Exeption: %s" % str(e))
+ return False
+ else:
+ print "fileops.clear_dir(), %s is not a directory" % (target)
+ #print "fileops.clear_dir(), DONE, returning True"
+ return True
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
2015-05-21 23:53 [gentoo-commits] proj/catalyst:pending " Brian Dolbec
@ 2015-05-24 0:08 ` Brian Dolbec
0 siblings, 0 replies; 16+ messages in thread
From: Brian Dolbec @ 2015-05-24 0:08 UTC (permalink / raw
To: gentoo-commits
commit: 64cae407aa8e334a2b925fd5aab1f6a4d164a47a
Author: Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 6 16:48:27 2014 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Mar 28 02:28:16 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=64cae407
Add --xattrs and --acls to the tar calls so we don't lose file properties while building the stages - should fix bug 531788.
Signed-off-by: Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo.org>
catalyst/base/stagebase.py | 18 +++++++++---------
catalyst/contents.py | 6 +++---
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index c94f6b6..a33cae4 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -696,10 +696,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["chroot_path"]+\
" (This may take some time) ...\n"
if "bz2" == self.settings["chroot_path"][-3:]:
- unpack_cmd="tar -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
+ unpack_cmd="tar --xattrs --acls -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
self.settings["chroot_path"]
else:
- unpack_cmd="tar -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
+ unpack_cmd="tar --xattrs --acls -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
self.settings["chroot_path"]
error_msg="Tarball extraction of "+\
self.settings["source_path"]+" to "+\
@@ -711,10 +711,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["chroot_path"]+\
" (This may take some time) ...\n"
if "bz2" == self.settings["chroot_path"][-3:]:
- unpack_cmd="tar -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
+ unpack_cmd="tar --xattrs --acls -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
self.settings["chroot_path"]
else:
- unpack_cmd="tar -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
+ unpack_cmd="tar --xattrs --acls -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
self.settings["chroot_path"]
error_msg="Tarball extraction of "+self.settings["source_path"]+\
" to "+self.settings["chroot_path"]+" failed."
@@ -806,9 +806,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
"catalyst-hash")
destdir=self.settings["snapshot_cache_path"]
if "bz2" == self.settings["chroot_path"][-3:]:
- unpack_cmd="tar -I lbzip2 -xpf "+self.settings["snapshot_path"]+" -C "+destdir
+ unpack_cmd="tar --xattrs --acls -I lbzip2 -xpf "+self.settings["snapshot_path"]+" -C "+destdir
else:
- unpack_cmd="tar xpf "+self.settings["snapshot_path"]+" -C "+destdir
+ unpack_cmd="tar --xattrs --acls xpf "+self.settings["snapshot_path"]+" -C "+destdir
unpack_errmsg="Error unpacking snapshot"
cleanup_msg="Cleaning up invalid snapshot cache at \n\t"+\
self.settings["snapshot_cache_path"]+\
@@ -825,10 +825,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
cleanup_msg=\
"Cleaning up existing portage tree (This can take a long time)..."
if "bz2" == self.settings["chroot_path"][-3:]:
- unpack_cmd="tar -I lbzip2 -xpf "+self.settings["snapshot_path"]+" -C "+\
+ unpack_cmd="tar --xattrs --acls -I lbzip2 -xpf "+self.settings["snapshot_path"]+" -C "+\
self.settings["chroot_path"]+"/usr"
else:
- unpack_cmd="tar xpf "+self.settings["snapshot_path"]+" -C "+\
+ unpack_cmd="tar --xattrs --acls xpf "+self.settings["snapshot_path"]+" -C "+\
self.settings["chroot_path"]+"/usr"
unpack_errmsg="Error unpacking snapshot"
@@ -1248,7 +1248,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
print "Creating stage tarball..."
- cmd("tar -I lbzip2 -cpf "+self.settings["target_path"]+" -C "+\
+ cmd("tar --xattrs --acls -I lbzip2 -cpf "+self.settings["target_path"]+" -C "+\
self.settings["stage_path"]+" .",\
"Couldn't create stage tarball",env=self.env)
diff --git a/catalyst/contents.py b/catalyst/contents.py
index 9611890..a06b2db 100644
--- a/catalyst/contents.py
+++ b/catalyst/contents.py
@@ -11,9 +11,9 @@ CONTENTS_DEFINITIONS = {
# 'find' is disabled because it requires the source path, which is not
# always available
#"find" :["calc_contents","find %(path)s"],
- "tar_tv":["calc_contents","tar tvf %(file)s"],
- "tar_tvz":["calc_contents","tar tvzf %(file)s"],
- "tar_tvj":["calc_contents","tar -I lbzip2 -tvf %(file)s"],
+ "tar_tv":["calc_contents","tar --xattrs tvf %(file)s"],
+ "tar_tvz":["calc_contents","tar --xattrs tvzf %(file)s"],
+ "tar_tvj":["calc_contents","tar --xattrs -I lbzip2 -tvf %(file)s"],
"isoinfo_l":["calc_contents","isoinfo -l -i %(file)s"],
# isoinfo_f should be a last resort only
"isoinfo_f":["calc_contents","isoinfo -f -i %(file)s"],
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2015-09-09 15:22 Brian Dolbec
0 siblings, 0 replies; 16+ messages in thread
From: Brian Dolbec @ 2015-09-09 15:22 UTC (permalink / raw
To: gentoo-commits
commit: a7dd0129e0f01026e29583805e0b0c1232a8e747
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Sep 9 15:16:15 2015 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Sep 9 15:16:36 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=a7dd0129
Fix LockInUse return code failure
Move trapping of the lock failure from the top most build call to the run() where
it can do cleanup work before exiting.
catalyst/base/stagebase.py | 18 +++++++++++++++---
catalyst/main.py | 11 +++++------
2 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 5eb8fc9..60acd55 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -20,7 +20,7 @@ from catalyst.support import (CatalystError, msg, file_locate, normpath,
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
-from catalyst.lock import LockDir
+from catalyst.lock import LockDir, LockInUse
from catalyst.fileops import ensure_dirs, pjoin
from catalyst.base.resume import AutoResume
@@ -1419,18 +1419,30 @@ class StageBase(TargetBase, ClearBase, GenBase):
print "StageBase: run() purge"
self.purge()
+ failure = False
for x in self.settings["action_sequence"]:
print "--- Running action sequence: "+x
sys.stdout.flush()
try:
apply(getattr(self,x))
+ except LockInUse:
+ print "Error, unable to aquire the lock..."
+ print " Catalyst aborting...."
+ failure = True
+ break
except Exception as error:
print "Exception running action sequence %s" % x
print "Error:", str(error)
- print "Running unbind()"
- self.unbind()
+ print " Catalyst aborting...."
+ failure = True
break
+ if failure:
+ print "Cleaning up... Running unbind()"
+ self.unbind()
+ return False
+ return True
+
def unmerge(self):
if "autoresume" in self.settings["options"] \
diff --git a/catalyst/main.py b/catalyst/main.py
index 2a5bdd3..dfa0609 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -21,7 +21,6 @@ import catalyst.config
import catalyst.util
from catalyst.defaults import confdefaults, option_messages
from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
-from catalyst.lock import LockInUse
from catalyst.support import CatalystError, find_binary
from catalyst.version import get_version
@@ -164,8 +163,8 @@ def build_target(addlargs):
"Target \"%s\" not available." % target,
print_traceback=True)
except CatalystError:
- return
- target.run()
+ return False
+ return target.run()
def main():
@@ -345,12 +344,12 @@ def main():
# everything is setup, so the build is a go
try:
- build_target(addlargs)
+ success = build_target(addlargs)
except KeyboardInterrupt:
print "\nCatalyst build aborted due to user interrupt ( Ctrl-C )"
print
print "Catalyst aborting...."
sys.exit(2)
- except LockInUse:
- print "Catalyst aborting...."
+ if not success:
sys.exit(2)
+ sys.exit(0)
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2015-10-06 15:31 Mike Frysinger
0 siblings, 0 replies; 16+ messages in thread
From: Mike Frysinger @ 2015-10-06 15:31 UTC (permalink / raw
To: gentoo-commits
commit: e0c76c5cca51ad07c40ab9800ffe6fcfdec24aee
Author: Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 6 14:33:06 2015 +0000
Commit: Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct 6 14:33:06 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e0c76c5c
lint: init all members in __init__
A few class members were being set outside of __init__. Usually this is
an oversight, so having the linter complain is helpful. Set up the few
places in the code that don't do this.
catalyst/base/stagebase.py | 2 ++
catalyst/lock.py | 1 +
2 files changed, 3 insertions(+)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index e393c5b..7bc7522 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -73,6 +73,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
previously. -agaffney
"""
+ self.makeconf = {}
self.archmap = {}
self.subarchmap = {}
machinemap = {}
@@ -156,6 +157,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.set_source_subpath()
""" Set paths """
+ self.snapshot_lock_object = None
self.set_snapshot_path()
self.set_root_path()
self.set_source_path()
diff --git a/catalyst/lock.py b/catalyst/lock.py
index d6653f7..d079b2d 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -54,6 +54,7 @@ class LockDir(object):
else:
LockDir.lock_dirs_in_use.append(lockdir)
+ self.myhardlock = None
self.hardlock_paths={}
def delete_lock_from_path_list(self):
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2015-10-09 2:40 Mike Frysinger
0 siblings, 0 replies; 16+ messages in thread
From: Mike Frysinger @ 2015-10-09 2:40 UTC (permalink / raw
To: gentoo-commits
commit: b2e05841e9b4eed6c287631bde8be58ff9430bb6
Author: Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 6 22:28:36 2015 +0000
Commit: Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Thu Oct 8 22:07:57 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b2e05841
support: use snakeoil.fileutils.touch
No point in duplicating our own when snakeoil provides a better one.
catalyst/base/resume.py | 5 +++--
catalyst/base/stagebase.py | 6 ++++--
catalyst/support.py | 8 --------
3 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index e3f03e7..443b46d 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -15,8 +15,9 @@ import os
import sys
import traceback
+from snakeoil import fileutils
+
from catalyst.fileops import ensure_dirs, pjoin, listdir_files, clear_dir
-from catalyst.support import touch
class AutoResumeError(Exception):
@@ -69,7 +70,7 @@ class AutoResume(object):
myf.write(data)
else:
try:
- touch(fname)
+ fileutils.touch(fname)
self._points[point] = fname
except Exception as e:
print AutoResumeError(str(e))
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 813e5f1..f81c51b 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -9,12 +9,14 @@ from stat import ST_UID, ST_GID, ST_MODE
# for convienience
pjoin = os.path.join
+from snakeoil import fileutils
+
from DeComp.compress import CompressMap
from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, msg, file_locate, normpath,
- touch, cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount,
+ cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount,
file_check)
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
@@ -1513,7 +1515,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
cmd(self.settings["controller_file"]+\
" build_packages "+mypack,\
"Error in attempt to build packages",env=self.env)
- touch(build_packages_resume)
+ fileutils.touch(build_packages_resume)
self.resume.enable("build_packages")
except CatalystError:
self.unbind()
diff --git a/catalyst/support.py b/catalyst/support.py
index 1207d36..960a91d 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -250,14 +250,6 @@ def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
raise CatalystError('\n\tAlso: '.join(messages))
-def touch(myfile):
- try:
- myf=open(myfile,"w")
- myf.close()
- except IOError:
- raise CatalystError("Could not touch "+myfile+".", print_traceback=True)
-
-
def countdown(secs=5, doing="Starting"):
if secs:
print ">>> Waiting",secs,"seconds before starting..."
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2015-10-09 2:40 Mike Frysinger
0 siblings, 0 replies; 16+ messages in thread
From: Mike Frysinger @ 2015-10-09 2:40 UTC (permalink / raw
To: gentoo-commits
commit: 08c84b0ad645c0166bbedf26d3972d8dee23db74
Author: Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 7 04:26:38 2015 +0000
Commit: Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Fri Oct 9 00:23:49 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=08c84b0a
stagebase: replace read_from_clst with snakeoil
The read_from_clst function is used in one place, and this function is
an overwrought "read all the data from a file" function. Replace it
with a snakeoil function.
catalyst/base/stagebase.py | 5 ++---
catalyst/support.py | 15 ---------------
2 files changed, 2 insertions(+), 18 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 7d069fa..bffafb7 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -16,8 +16,7 @@ from DeComp.compress import CompressMap
from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, msg, file_locate, normpath,
- cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount,
- file_check)
+ cmd, warn, list_bashify, read_makeconf, ismount, file_check)
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
@@ -817,7 +816,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
if "snapcache" in self.settings["options"]:
snapshot_cache_hash_path = pjoin(
self.settings['snapshot_cache_path'], 'catalyst-hash')
- snapshot_cache_hash = read_from_clst(snapshot_cache_hash_path)
+ snapshot_cache_hash = fileutils.readfile(snapshot_cache_hash_path, True)
unpack_info['mode'] = self.decompressor.determine_mode(
unpack_info['source'])
diff --git a/catalyst/support.py b/catalyst/support.py
index 960a91d..c629025 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -13,21 +13,6 @@ from catalyst.defaults import verbosity, valid_config_file_values
BASH_BINARY = "/bin/bash"
-def read_from_clst(path):
- line = ''
- myline = ''
- try:
- myf = open(path, "r")
- except Exception:
- return -1
- #raise CatalystError("Could not open file " + path)
- for line in myf.readlines():
- #line = line.replace("\n", "") # drop newline
- myline = myline + line
- myf.close()
- return myline
-
-
def list_bashify(mylist):
if type(mylist)==types.StringType:
mypack=[mylist]
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2015-10-11 17:26 Mike Frysinger
0 siblings, 0 replies; 16+ messages in thread
From: Mike Frysinger @ 2015-10-11 17:26 UTC (permalink / raw
To: gentoo-commits
commit: 20dd6faa8a902a1ae05af6bda13e2b4273a3ef38
Author: Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 10 05:13:25 2015 +0000
Commit: Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sat Oct 10 23:54:36 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=20dd6faa
stagebase: convert to log module
This also allows cleaning up a few funcs from the support module
as they were only lightly used in stagebase.
catalyst/base/stagebase.py | 262 ++++++++++++++++++++++-----------------------
catalyst/defaults.py | 2 -
catalyst/support.py | 16 +--
3 files changed, 128 insertions(+), 152 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 3ce7dba..e7514e2 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -13,10 +13,11 @@ from snakeoil import fileutils
from DeComp.compress import CompressMap
+from catalyst import log
from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
PORT_LOGDIR_CLEAN)
-from catalyst.support import (CatalystError, msg, file_locate, normpath,
- cmd, warn, list_bashify, read_makeconf, ismount, file_check)
+from catalyst.support import (CatalystError, file_locate, normpath,
+ cmd, list_bashify, read_makeconf, ismount, file_check)
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
@@ -97,7 +98,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
# This message should probably change a bit, since everything in
# the dir should load just fine. If it doesn't, it's probably a
# syntax error in the module
- msg("Can't find/load " + x + ".py plugin in " + arch_dir)
+ log.warning("Can't find/load %s.py plugin in %s", x, arch_dir)
if "chost" in self.settings:
hostmachine = self.settings["chost"].split("-")[0]
@@ -123,23 +124,22 @@ class StageBase(TargetBase, ClearBase, GenBase):
try:
self.arch=self.subarchmap[self.settings["subarch"]](self.settings)
except KeyError:
- print "Invalid subarch: "+self.settings["subarch"]
- print "Choose one of the following:",
- for x in self.subarchmap:
- print x,
- print
- sys.exit(2)
-
- print "Using target:",self.settings["target"]
+ log.critical(
+ 'Invalid subarch: %s\n'
+ 'Choose one of the following:\n'
+ ' %s',
+ self.settings['subarch'], ' '.join(self.subarchmap))
+
+ log.notice('Using target: %s', self.settings['target'])
# Print a nice informational message
if self.settings["buildarch"]==self.settings["hostarch"]:
- print "Building natively for",self.settings["hostarch"]
+ log.info('Building natively for %s', self.settings['hostarch'])
elif self.settings["crosscompile"]:
- print "Cross-compiling on",self.settings["buildarch"],\
- "for different machine type",self.settings["hostarch"]
+ log.info('Cross-compiling on %s for different machine type %s',
+ self.settings['buildarch'], self.settings['hostarch'])
else:
- print "Building on",self.settings["buildarch"],\
- "for alternate personality type",self.settings["hostarch"]
+ log.info('Building on %s for alternate personality type %s',
+ self.settings['buildarch'], self.settings['hostarch'])
# This must be set first as other set_ options depend on this
self.set_spec_prefix()
@@ -218,15 +218,13 @@ class StageBase(TargetBase, ClearBase, GenBase):
# the command line).
if "pkgcache" in self.settings["options"]:
self.set_pkgcache_path()
- print "Location of the package cache is "+\
- self.settings["pkgcache_path"]
+ log.info('Location of the package cache is %s', self.settings['pkgcache_path'])
self.mounts.append("packagedir")
self.mountmap["packagedir"] = self.settings["pkgcache_path"]
if "kerncache" in self.settings["options"]:
self.set_kerncache_path()
- print "Location of the kerncache is "+\
- self.settings["kerncache_path"]
+ log.info('Location of the kerncache is %s', self.settings['kerncache_path'])
self.mounts.append("kerncache")
self.mountmap["kerncache"] = self.settings["kerncache_path"]
@@ -344,8 +342,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"/builds/"+self.settings["target_subpath"])
if "autoresume" in self.settings["options"]\
and self.resume.is_enabled("setup_target_path"):
- print \
- "Resume point detected, skipping target path setup operation..."
+ log.notice('Resume point detected, skipping target path setup operation...')
else:
# First clean up any existing target stuff
# XXX WTF are we removing the old tarball before we start building the
@@ -406,8 +403,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["fstype"]="normal"
for x in self.valid_values:
if x == self.settings["spec_prefix"]+"/fstype":
- print "\n"+self.settings["spec_prefix"]+\
- "/fstype is being set to the default of \"normal\"\n"
+ log.info('%s/fstype is being set to the default of "normal"',
+ self.settings['spec_prefix'])
def set_fsops(self):
if "fstype" in self.settings:
@@ -435,13 +432,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["hash_map"].generate_hash(
self.settings["source_path"],
hash_=self.settings["hash_function"])
- print "Source path set to "+self.settings["source_path"]
+ log.info('Source path set to %s', self.settings['source_path'])
if os.path.isdir(self.settings["source_path"]):
- print "\tIf this is not desired, remove this directory or turn off"
- print "\tseedcache in the options of catalyst.conf the source path"
- print "\twill then be "+\
- normpath(self.settings["storedir"]+"/builds/"+\
- self.settings["source_subpath"]+"\n")
+ log.warning(
+ 'If this is not desired, remove this directory or turn off\n'
+ 'seedcache in the options of catalyst.conf the source path\n'
+ 'will then be %s',
+ normpath(self.settings['storedir'] + '/builds/' +
+ self.settings['source_subpath']))
def set_dest_path(self):
if "root_path" in self.settings:
@@ -458,7 +456,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["snapshot_path"]= file_check(normpath(self.settings["storedir"]+\
"/snapshots/" + self.settings["snapshot_name"] +
self.settings["snapshot"]))
- print "*** SNAPSHOT_PATH set to:", self.settings["snapshot_path"]
+ log.info('SNAPSHOT_PATH set to: %s', self.settings['snapshot_path'])
self.settings["snapshot_path_hash"] = \
self.settings["hash_map"].generate_hash(
self.settings["snapshot_path"],
@@ -474,7 +472,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["snapshot"]))
self.snapcache_lock=\
LockDir(self.settings["snapshot_cache_path"])
- print "Setting snapshot cache to "+self.settings["snapshot_cache_path"]
+ log.info('Setting snapshot cache to %s', self.settings['snapshot_cache_path'])
def set_chroot_path(self):
"""
@@ -493,7 +491,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["version_stamp"])
))
if "autoresume" in self.settings["options"]:
- print "The autoresume path is " + self.settings["autoresume_path"]
+ log.info('The autoresume path is %s', self.settings['autoresume_path'])
self.resume = AutoResume(self.settings["autoresume_path"], mode=0755)
def set_controller_file(self):
@@ -582,8 +580,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
if type(self.settings["portage_overlay"])==types.StringType:
self.settings["portage_overlay"]=\
self.settings["portage_overlay"].split()
- print "portage_overlay directories are set to: \""+\
- ' '.join(self.settings["portage_overlay"])+"\""
+ log.info('portage_overlay directories are set to: %s',
+ ' '.join(self.settings['portage_overlay']))
def set_overlay(self):
if self.settings["spec_prefix"]+"/overlay" in self.settings:
@@ -639,7 +637,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
del self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
def kill_chroot_pids(self):
- print "Checking for processes running in chroot and killing them."
+ log.info('Checking for processes running in chroot and killing them.')
# Force environment variables to be exported so script can see them
self.setup_environment()
@@ -659,23 +657,23 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not os.path.exists(self.settings["chroot_path"]):
return
- #print "self.mounts =", self.mounts
+ log.debug('self.mounts = %s', self.mounts)
for x in self.mounts:
target = normpath(self.settings["chroot_path"] + self.target_mounts[x])
- #print "mount_safety_check() x =", x, target
+ log.debug('mount_safety_check() x = %s %s', x, target)
if not os.path.exists(target):
continue
if ismount(target):
# Something is still mounted
try:
- print target + " is still mounted; performing auto-bind-umount...",
+ log.warning('%s is still mounted; performing auto-bind-umount...', target)
# Try to umount stuff ourselves
self.unbind()
if ismount(target):
raise CatalystError("Auto-unbind failed for " + target)
else:
- print "Auto-unbind successful..."
+ log.notice('Auto-unbind successful...')
except CatalystError:
raise CatalystError("Unable to auto-unbind " + target)
@@ -691,8 +689,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
'auto-ext': False,
}
- display_msg="\nStarting %(mode)s from %(source)s\nto "+\
- "%(destination)s (This may take some time) ...\n"
+ display_msg = (
+ 'Starting %(mode)s from %(source)s\nto '
+ '%(destination)s (this may take some time) ..')
error_msg="'%(mode)s' extraction of %(source)s to %(destination)s failed."
@@ -702,7 +701,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
unpack_info['mode'] = "rsync"
else:
# SEEDCACHE is a not a directory, try untar'ing
- print "Referenced SEEDCACHE does not appear to be a directory, trying to untar..."
+ log.notice('Referenced SEEDCACHE does not appear to be a directory, trying to untar...')
unpack_info['source'] = file_check(unpack_info['source'])
else:
# No SEEDCACHE, use tar
@@ -764,7 +763,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
if invalid_snapshot:
if "autoresume" in self.settings["options"]:
- print "No Valid Resume point detected, cleaning up..."
+ log.notice('No Valid Resume point detected, cleaning up...')
self.clear_autoresume()
self.clear_chroot()
@@ -779,11 +778,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
if "kerncache" in self.settings["options"]:
ensure_dirs(self.settings["kerncache_path"],mode=0755)
- print display_msg %(unpack_info)
+ log.notice('%s', display_msg % unpack_info)
# now run the decompressor
if not self.decompressor.extract(unpack_info):
- print error_msg %(unpack_info)
+ log.error('%s', error_msg % unpack_info)
if "source_path_hash" in self.settings:
self.resume.enable("unpack",
@@ -791,7 +790,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
else:
self.resume.enable("unpack")
else:
- print "Resume point detected, skipping unpack operation..."
+ log.notice('Resume point detected, skipping unpack operation...')
def unpack_snapshot(self):
unpack=True
@@ -808,8 +807,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
target_portdir = normpath(self.settings["chroot_path"] +
self.settings["repo_basedir"] + "/" + self.settings["repo_name"])
- print self.settings["chroot_path"]
- print "unpack(), target_portdir = " + target_portdir
+ log.info('%s', self.settings['chroot_path'])
+ log.info('unpack(), target_portdir = %s', target_portdir)
if "snapcache" in self.settings["options"]:
snapshot_cache_hash_path = pjoin(
self.settings['snapshot_cache_path'], 'catalyst-hash')
@@ -819,16 +818,16 @@ class StageBase(TargetBase, ClearBase, GenBase):
cleanup_msg="Cleaning up invalid snapshot cache at \n\t"+\
self.settings["snapshot_cache_path"]+\
- " (This can take a long time)..."
+ " (this can take a long time)..."
cleanup_errmsg="Error removing existing snapshot cache directory."
if self.settings["snapshot_path_hash"]==snapshot_cache_hash:
- print "Valid snapshot cache, skipping unpack of portage tree..."
+ log.info('Valid snapshot cache, skipping unpack of portage tree...')
unpack=False
else:
cleanup_errmsg="Error removing existing snapshot directory."
cleanup_msg=\
- "Cleaning up existing portage tree (This can take a long time)..."
+ 'Cleaning up existing portage tree (this can take a long time)...'
unpack_info['destination'] = normpath(
self.settings["chroot_path"] + self.settings["repo_basedir"])
unpack_info['mode'] = self.decompressor.determine_mode(
@@ -838,30 +837,29 @@ class StageBase(TargetBase, ClearBase, GenBase):
and os.path.exists(target_portdir) \
and self.resume.is_enabled("unpack_portage") \
and self.settings["snapshot_path_hash"] == snapshot_hash:
- print \
- "Valid Resume point detected, skipping unpack of portage tree..."
+ log.notice('Valid Resume point detected, skipping unpack of portage tree...')
unpack = False
if unpack:
if "snapcache" in self.settings["options"]:
self.snapcache_lock.write_lock()
if os.path.exists(target_portdir):
- print cleanup_msg
+ log.info('%s', cleanup_msg)
cleanup_cmd = "rm -rf " + target_portdir
- print "unpack() cleanup_cmd = " + cleanup_cmd
+ log.info('unpack() cleanup_cmd = %s', cleanup_cmd)
cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
ensure_dirs(target_portdir, mode=0755)
- print "Unpacking portage tree (This can take a long time) ..."
+ log.notice('Unpacking portage tree (this can take a long time) ...')
if not self.decompressor.extract(unpack_info):
- print unpack_errmsg %(unpack_info)
+ log.error('%s', unpack_errmsg % unpack_info)
if "snapcache" in self.settings["options"]:
myf = open(snapshot_cache_hash_path, 'w')
myf.write(self.settings["snapshot_path_hash"])
myf.close()
else:
- print "Setting snapshot autoresume point"
+ log.info('Setting snapshot autoresume point')
self.resume.enable("unpack_portage",
data=self.settings["snapshot_path_hash"])
@@ -871,12 +869,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
def config_profile_link(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("config_profile_link"):
- print \
- "Resume point detected, skipping config_profile_link operation..."
+ log.notice('Resume point detected, skipping config_profile_link operation...')
else:
# TODO: zmedico and I discussed making this a directory and pushing
# in a parent file, as well as other user-specified configuration.
- print "Configuring profile link..."
+ log.info('Configuring profile link...')
cmd("rm -f " + self.settings["chroot_path"] +
self.settings["port_conf"] + "/make.profile",
"Error zapping profile link",env=self.env)
@@ -892,10 +889,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
def setup_confdir(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("setup_confdir"):
- print "Resume point detected, skipping setup_confdir operation..."
+ log.notice('Resume point detected, skipping setup_confdir operation...')
else:
if "portage_confdir" in self.settings:
- print "Configuring %s..." % self.settings["port_conf"]
+ log.info('Configuring %s...', self.settings['port_conf'])
cmd("rsync -a " + self.settings["portage_confdir"] + "/ " +
self.settings["chroot_path"] + self.settings["port_conf"],
"Error copying %s" % self.settings["port_conf"],
@@ -907,7 +904,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
if "portage_overlay" in self.settings:
for x in self.settings["portage_overlay"]:
if os.path.exists(x):
- print "Copying overlay dir " +x
+ log.info('Copying overlay dir %s', x)
cmd("mkdir -p "+self.settings["chroot_path"]+\
self.settings["local_overlay"],\
"Could not make portage_overlay dir",env=self.env)
@@ -921,7 +918,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
for x in self.settings[self.settings["spec_prefix"]+\
"/root_overlay"]:
if os.path.exists(x):
- print "Copying root_overlay: "+x
+ log.info('Copying root_overlay: %s', x)
cmd("rsync -a "+x+"/ "+\
self.settings["chroot_path"],\
self.settings["spec_prefix"]+"/root_overlay: "+x+\
@@ -933,7 +930,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def bind(self):
for x in self.mounts:
_cmd = ''
- #print "bind(); x =", x
+ log.debug('bind(); x = %s', x)
target = normpath(self.settings["chroot_path"] + self.target_mounts[x])
ensure_dirs(target, mode=0755)
@@ -942,7 +939,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
ensure_dirs(self.mountmap[x], mode=0755)
src=self.mountmap[x]
- #print "bind(); src =", src
+ log.debug('bind(); src = %s', src)
if "snapcache" in self.settings["options"] and x == "portdir":
self.snapcache_lock.read_lock()
if os.uname()[0] == "FreeBSD":
@@ -960,9 +957,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
_cmd = "mount -t tmpfs -o noexec,nosuid,nodev shm " + target
else:
_cmd = "mount --bind " + src + " " + target
- #print "bind(); _cmd =", _cmd
+ log.debug('bind(); _cmd = %s', _cmd)
cmd(_cmd, "Bind mounting Failed", env=self.env, fail_func=self.unbind)
- #print "bind(); finished :D"
+ log.debug('bind(); finished :D')
def unbind(self):
ouch=0
@@ -981,15 +978,15 @@ class StageBase(TargetBase, ClearBase, GenBase):
retval=os.system("umount " + target)
if retval!=0:
- warn("First attempt to unmount: " + target + " failed.")
- warn("Killing any pids still running in the chroot")
+ log.warning('First attempt to unmount failed: %s', target)
+ log.warning('Killing any pids still running in the chroot')
self.kill_chroot_pids()
retval2 = os.system("umount " + target)
if retval2!=0:
ouch=1
- warn("Couldn't umount bind mount: " + target)
+ log.warning("Couldn't umount bind mount: %s", target)
if "snapcache" in self.settings["options"] and x == "/usr/portage":
try:
@@ -1019,9 +1016,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.override_asflags()
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("chroot_setup"):
- print "Resume point detected, skipping chroot_setup operation..."
+ log.notice('Resume point detected, skipping chroot_setup operation...')
else:
- print "Setting up chroot..."
+ log.notice('Setting up chroot...')
cmd("cp /etc/resolv.conf " + self.settings["chroot_path"] + "/etc/",
"Could not copy resolv.conf into place.",env=self.env)
@@ -1033,13 +1030,12 @@ class StageBase(TargetBase, ClearBase, GenBase):
"Can't find envscript " + self.settings["envscript"],
print_traceback=True)
- print "\nWarning!!!!"
- print "\tOverriding certain env variables may cause catastrophic failure."
- print "\tIf your build fails look here first as the possible problem."
- print "\tCatalyst assumes you know what you are doing when setting"
- print "\t\tthese variables."
- print "\tCatalyst Maintainers use VERY minimal envscripts if used at all"
- print "\tYou have been warned\n"
+ log.warning(
+ 'Overriding certain env variables may cause catastrophic failure.\n'
+ 'If your build fails look here first as the possible problem.\n'
+ 'Catalyst assumes you know what you are doing when setting these variables.\n'
+ 'Catalyst Maintainers use VERY minimal envscripts, if used at all.\n'
+ 'You have been warned.')
cmd("cp "+self.settings["envscript"]+" "+\
self.settings["chroot_path"]+"/tmp/envscript",\
@@ -1113,10 +1109,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
myusevars = sorted(set(myusevars))
myf.write('USE="' + ' '.join(myusevars) + '"\n')
if '-*' in myusevars:
- print "\nWarning!!! "
- print "\tThe use of -* in "+self.settings["spec_prefix"]+\
- "/use will cause portage to ignore"
- print "\tpackage.use in the profile and portage_confdir. You've been warned!"
+ log.warning(
+ 'The use of -* in %s/use will cause portage to ignore\n'
+ 'package.use in the profile and portage_confdir.\n'
+ "You've been warned!", self.settings['spec_prefix'])
myuseexpandvars={}
if "HOSTUSEEXPAND" in self.settings:
@@ -1141,7 +1137,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def fsscript(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("fsscript"):
- print "Resume point detected, skipping fsscript operation..."
+ log.notice('Resume point detected, skipping fsscript operation...')
else:
if "fsscript" in self.settings:
if os.path.exists(self.settings["controller_file"]):
@@ -1152,7 +1148,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def rcupdate(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("rcupdate"):
- print "Resume point detected, skipping rcupdate operation..."
+ log.notice('Resume point detected, skipping rcupdate operation...')
else:
if os.path.exists(self.settings["controller_file"]):
cmd(self.settings["controller_file"]+" rc-update",\
@@ -1162,10 +1158,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
def clean(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("clean"):
- print "Resume point detected, skipping clean operation..."
+ log.notice('Resume point detected, skipping clean operation...')
else:
for x in self.settings["cleanables"]:
- print "Cleaning chroot: "+x+"... "
+ log.notice('Cleaning chroot: %s', x)
cmd("rm -rf "+self.settings["destpath"]+x,"Couldn't clean "+\
x,env=self.env)
@@ -1197,7 +1193,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def empty(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("empty"):
- print "Resume point detected, skipping empty operation..."
+ log.notice('Resume point detected, skipping empty operation...')
else:
if self.settings["spec_prefix"]+"/empty" in self.settings:
if type(self.settings[self.settings["spec_prefix"]+\
@@ -1208,9 +1204,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
for x in self.settings[self.settings["spec_prefix"]+"/empty"]:
myemp=self.settings["destpath"]+x
if not os.path.isdir(myemp) or os.path.islink(myemp):
- print x,"not a directory or does not exist, skipping 'empty' operation."
+ log.warning('not a directory or does not exist, skipping "empty" operation: %s', x)
continue
- print "Emptying directory",x
+ log.info('Emptying directory %s', x)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
mystat=os.stat(myemp)
@@ -1223,13 +1219,13 @@ class StageBase(TargetBase, ClearBase, GenBase):
def remove(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("remove"):
- print "Resume point detected, skipping remove operation..."
+ log.notice('Resume point detected, skipping remove operation...')
else:
if self.settings["spec_prefix"]+"/rm" in self.settings:
for x in self.settings[self.settings["spec_prefix"]+"/rm"]:
# We're going to shell out for all these cleaning
# operations, so we get easy glob handling.
- print "livecd: removing "+x
+ log.notice('livecd: removing %s', x)
os.system("rm -rf "+self.settings["chroot_path"]+x)
try:
if os.path.exists(self.settings["controller_file"]):
@@ -1243,7 +1239,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def preclean(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("preclean"):
- print "Resume point detected, skipping preclean operation..."
+ log.notice('Resume point detected, skipping preclean operation...')
else:
try:
if os.path.exists(self.settings["controller_file"]):
@@ -1264,9 +1260,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("capture"):
- print "Resume point detected, skipping capture operation..."
+ log.notice('Resume point detected, skipping capture operation...')
else:
- print "Capture target in a tarball"
+ log.notice('Capture target in a tarball')
# Remove filename from path
mypath = os.path.dirname(self.settings["target_path"])
@@ -1283,29 +1279,30 @@ class StageBase(TargetBase, ClearBase, GenBase):
target_filename = ".".join([self.settings["target_path"].rstrip('/'),
self.compressor.extension(pack_info['mode'])])
- print "Creating stage tarball... mode:", \
- self.settings["compression_mode"]
+ log.notice('Creating stage tarball... mode: %s',
+ self.settings['compression_mode'])
if self.compressor.compress(pack_info):
self.gen_contents_file(target_filename)
self.gen_digest_file(target_filename)
self.resume.enable("capture")
else:
- print "Couldn't create stage tarball:", target_filename
+ log.warning("Couldn't create stage tarball: %s", target_filename)
def run_local(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("run_local"):
- print "Resume point detected, skipping run_local operation..."
+ log.notice('Resume point detected, skipping run_local operation...')
else:
try:
if os.path.exists(self.settings["controller_file"]):
- print "run_local() starting controller script..."
+ log.info('run_local() starting controller script...')
cmd(self.settings["controller_file"]+" run",\
"run script failed.",env=self.env)
self.resume.enable("run_local")
else:
- print "run_local() no controller_file found...", self.settings["controller_file"]
+ log.info('run_local() no controller_file found... %s',
+ self.settings['controller_file'])
except CatalystError:
self.unbind()
@@ -1318,9 +1315,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
fixed. We need this to use the os.system() call since we can't
specify our own environ
"""
- #print "setup_environment(); settings =", list(self.settings)
+ log.debug('setup_environment(); settings = %r', self.settings)
for x in list(self.settings):
- #print "setup_environment(); processing:", x
+ log.debug('setup_environment(); processing: %s', x)
if x == "options":
#self.env['clst_' + x] = ' '.join(self.settings[x])
for opt in self.settings[x]:
@@ -1367,7 +1364,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
if "makeopts" in self.settings:
self.env["MAKEOPTS"]=self.settings["makeopts"]
- #print "setup_environment(); env =", self.env
+ log.debug('setup_environment(); env = %r', self.env)
def run(self):
self.chroot_lock.write_lock()
@@ -1386,33 +1383,30 @@ class StageBase(TargetBase, ClearBase, GenBase):
return
if "purgeonly" in self.settings["options"]:
- print "StageBase: run() purgeonly"
+ log.info('StageBase: run() purgeonly')
self.purge()
if "purge" in self.settings["options"]:
- print "StageBase: run() purge"
+ log.info('StageBase: run() purge')
self.purge()
failure = False
for x in self.settings["action_sequence"]:
- print "--- Running action sequence: "+x
+ log.notice('--- Running action sequence: %s', x)
sys.stdout.flush()
try:
getattr(self, x)()
except LockInUse:
- print "Error, unable to aquire the lock..."
- print " Catalyst aborting...."
+ log.error('Unable to aquire the lock...')
failure = True
break
- except Exception as error:
- print "Exception running action sequence %s" % x
- print "Error:", str(error)
- print " Catalyst aborting...."
+ except Exception:
+ log.error('Exception running action sequence %s', x, exc_info=True)
failure = True
break
if failure:
- print "Cleaning up... Running unbind()"
+ log.notice('Cleaning up... Running unbind()')
self.unbind()
return False
return True
@@ -1421,7 +1415,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def unmerge(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("unmerge"):
- print "Resume point detected, skipping unmerge operation..."
+ log.notice('Resume point detected, skipping unmerge operation...')
else:
if self.settings["spec_prefix"]+"/unmerge" in self.settings:
if type(self.settings[self.settings["spec_prefix"]+\
@@ -1442,7 +1436,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
cmd(self.settings["controller_file"]+\
" unmerge "+ myunmerge,"Unmerge script failed.",\
env=self.env)
- print "unmerge shell script"
+ log.info('unmerge shell script')
except CatalystError:
self.unbind()
raise
@@ -1451,9 +1445,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
def target_setup(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("target_setup"):
- print "Resume point detected, skipping target_setup operation..."
+ log.notice('Resume point detected, skipping target_setup operation...')
else:
- print "Setting up filesystems per filesystem type"
+ log.notice('Setting up filesystems per filesystem type')
cmd(self.settings["controller_file"]+\
" target_image_setup "+ self.settings["target_path"],\
"target_image_setup script failed.",env=self.env)
@@ -1462,7 +1456,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def setup_overlay(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("setup_overlay"):
- print "Resume point detected, skipping setup_overlay operation..."
+ log.notice('Resume point detected, skipping setup_overlay operation...')
else:
if self.settings["spec_prefix"]+"/overlay" in self.settings:
for x in self.settings[self.settings["spec_prefix"]+"/overlay"]:
@@ -1476,7 +1470,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def create_iso(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("create_iso"):
- print "Resume point detected, skipping create_iso operation..."
+ log.notice('Resume point detected, skipping create_iso operation...')
else:
# Create the ISO
if "iso" in self.settings:
@@ -1487,20 +1481,19 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.gen_digest_file(self.settings["iso"])
self.resume.enable("create_iso")
else:
- print "WARNING: livecd/iso was not defined."
- print "An ISO Image will not be created."
+ log.warning('livecd/iso was not defined. An ISO Image will not be created.')
def build_packages(self):
build_packages_resume = pjoin(self.settings["autoresume_path"],
"build_packages")
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("build_packages"):
- print "Resume point detected, skipping build_packages operation..."
+ log.notice('Resume point detected, skipping build_packages operation...')
else:
if self.settings["spec_prefix"]+"/packages" in self.settings:
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("build_packages"):
- print "Resume point detected, skipping build_packages operation..."
+ log.notice('Resume point detected, skipping build_packages operation...')
else:
mypack=\
list_bashify(self.settings[self.settings["spec_prefix"]\
@@ -1520,7 +1513,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
'''Build all configured kernels'''
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("build_kernel"):
- print "Resume point detected, skipping build_kernel operation..."
+ log.notice('Resume point detected, skipping build_kernel operation...')
else:
if "boot/kernel" in self.settings:
try:
@@ -1544,7 +1537,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"Build a single configured kernel by name"
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("build_kernel_"+kname):
- print "Resume point detected, skipping build_kernel for "+kname+" operation..."
+ log.notice('Resume point detected, skipping build_kernel for %s operation...', kname)
return
self._copy_kernel_config(kname=kname)
@@ -1579,7 +1572,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
if "boot/kernel/"+kname+"/initramfs_overlay" in self.settings:
if os.path.exists(self.settings["chroot_path"]+\
"/tmp/initramfs_overlay/"):
- print "Cleaning up temporary overlay dir"
+ log.notice('Cleaning up temporary overlay dir')
cmd("rm -R "+self.settings["chroot_path"]+\
"/tmp/initramfs_overlay/",env=self.env)
@@ -1615,9 +1608,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
if "boot/kernel/"+kname+"/initramfs_overlay" in self.settings:
if os.path.exists(self.settings["boot/kernel/"+\
kname+"/initramfs_overlay"]):
- print "Copying initramfs_overlay dir "+\
- self.settings["boot/kernel/"+kname+\
- "/initramfs_overlay"]
+ log.notice('Copying initramfs_overlay dir %s',
+ self.settings['boot/kernel/' + kname + '/initramfs_overlay'])
cmd("mkdir -p "+\
self.settings["chroot_path"]+\
@@ -1635,7 +1627,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def bootloader(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("bootloader"):
- print "Resume point detected, skipping bootloader operation..."
+ log.notice('Resume point detected, skipping bootloader operation...')
else:
try:
cmd(self.settings["controller_file"]+\
@@ -1649,7 +1641,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def livecd_update(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("livecd_update"):
- print "Resume point detected, skipping build_packages operation..."
+ log.notice('Resume point detected, skipping build_packages operation...')
else:
try:
cmd(self.settings["controller_file"]+\
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 5db7ab6..666afca 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -22,8 +22,6 @@ valid_config_file_values.extend([ "distcc", "envscript",
"compression_mode", "decompressor_search_order",
])
-verbosity = 1
-
confdefaults={
"archdir": "%(PythonDir)s/arch",
"compression_mode": 'lbzip2_x',
diff --git a/catalyst/support.py b/catalyst/support.py
index c629025..6b1e727 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -8,7 +8,7 @@ import traceback
import time
from subprocess import Popen
-from catalyst.defaults import verbosity, valid_config_file_values
+from catalyst.defaults import valid_config_file_values
BASH_BINARY = "/bin/bash"
@@ -40,15 +40,6 @@ class CatalystError(Exception):
print
-def die(msg=None):
- warn(msg)
- sys.exit(1)
-
-
-def warn(msg):
- print "!!! catalyst: "+msg
-
-
def cmd(mycmd, myexc="", env=None, debug=False, fail_func=None):
if env is None:
env = {}
@@ -186,11 +177,6 @@ def read_makeconf(mymakeconffile):
return makeconf
-def msg(mymsg,verblevel=1):
- if verbosity>=verblevel:
- print mymsg
-
-
def pathcompare(path1,path2):
# Change double slashes to slash
path1 = re.sub(r"//",r"/",path1)
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2019-10-20 0:00 Matt Turner
0 siblings, 0 replies; 16+ messages in thread
From: Matt Turner @ 2019-10-20 0:00 UTC (permalink / raw
To: gentoo-commits
commit: 2e91db10c272e3943b768573b27477bca02e53d3
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 19 23:55:46 2019 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sun Oct 20 00:00:08 2019 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=2e91db10
Mount tmpfs at /run
Closes: https://bugs.gentoo.org/659238
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 3 +++
catalyst/defaults.py | 2 ++
2 files changed, 5 insertions(+)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 2ad53381..85e30450 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -232,6 +232,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
if os.uname()[0] == "Linux":
self.mounts.append("devpts")
self.mounts.append("shm")
+ self.mounts.append("run")
self.set_mounts()
@@ -977,6 +978,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
_cmd = ['mount', '-t', 'tmpfs',
'-o', 'size=' + self.settings['var_tmpfs_portage'] + 'G',
src, target]
+ else:
+ _cmd = ['mount', '-t', 'tmpfs', src, target]
else:
if os.uname()[0] == "FreeBSD":
if src == "/dev":
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index f2fe29df..cb5f84e3 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -91,6 +91,7 @@ TARGET_MOUNT_DEFAULTS = {
"port_logdir": "/var/log/portage",
"proc": "/proc",
"shm": "/dev/shm",
+ "run": "/run",
}
SOURCE_MOUNT_DEFAULTS = {
@@ -101,6 +102,7 @@ SOURCE_MOUNT_DEFAULTS = {
"port_tmpdir": "tmpfs",
"proc": "/proc",
"shm": "shmfs",
+ "run": "tmpfs",
}
# legend: key: message
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2020-04-10 21:04 Matt Turner
0 siblings, 0 replies; 16+ messages in thread
From: Matt Turner @ 2020-04-10 21:04 UTC (permalink / raw
To: gentoo-commits
commit: e8379bb2c9b695399dc6d8e0b512ae16ed60c177
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 10 07:33:13 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Apr 10 07:33:13 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e8379bb2
catalyst: Remove object inheritance
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/clearbase.py | 2 +-
catalyst/base/genbase.py | 2 +-
catalyst/base/resume.py | 2 +-
catalyst/base/targetbase.py | 2 +-
catalyst/builder.py | 2 +-
catalyst/config.py | 2 +-
catalyst/hash_utils.py | 2 +-
catalyst/lock.py | 2 +-
catalyst/main.py | 2 +-
9 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index 644a385f..f1d4d1ba 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -4,7 +4,7 @@ from catalyst import log
from catalyst.support import countdown
from catalyst.fileops import clear_dir
-class ClearBase(object):
+class ClearBase():
"""
This class does all of clearing after task completion
"""
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index 8af3b97a..d6fea954 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -3,7 +3,7 @@ import io
import os
-class GenBase(object):
+class GenBase():
"""
This class does generation of the contents and digests files.
"""
diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index 6a4ebdfa..7732dd5d 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -14,7 +14,7 @@ from catalyst import log
from catalyst.fileops import ensure_dirs, pjoin, listdir_files, clear_dir
-class AutoResume(object):
+class AutoResume():
'''Class for tracking and handling all aspects of
the autoresume option and related files.
'''
diff --git a/catalyst/base/targetbase.py b/catalyst/base/targetbase.py
index 4dcd88b7..0977bad3 100644
--- a/catalyst/base/targetbase.py
+++ b/catalyst/base/targetbase.py
@@ -2,7 +2,7 @@ import os
from catalyst.support import addl_arg_parse
-class TargetBase(object):
+class TargetBase():
"""
The toplevel class for all targets. This is about as generic as we get.
"""
diff --git a/catalyst/builder.py b/catalyst/builder.py
index 4d58de65..5d7f76c1 100644
--- a/catalyst/builder.py
+++ b/catalyst/builder.py
@@ -1,6 +1,6 @@
import os
-class generic(object):
+class generic():
def __init__(self,myspec):
self.settings=myspec
self.settings.setdefault('CHROOT', 'chroot')
diff --git a/catalyst/config.py b/catalyst/config.py
index 9f184ed5..5e70de41 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -4,7 +4,7 @@ import re
from catalyst import log
from catalyst.support import CatalystError
-class ParserBase(object):
+class ParserBase():
filename = ""
lines = None
diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py
index 05279b2c..8b0ff06c 100644
--- a/catalyst/hash_utils.py
+++ b/catalyst/hash_utils.py
@@ -41,7 +41,7 @@ HASH_DEFINITIONS = {
}
-class HashMap(object):
+class HashMap():
'''Class for handling
Catalyst's hash generation'''
diff --git a/catalyst/lock.py b/catalyst/lock.py
index 808df4ec..4c99e085 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -9,7 +9,7 @@ from catalyst.fileops import ensure_dirs
LockInUse = osutils.LockException
-class LockDir(object):
+class LockDir():
"""An object that creates locks inside dirs"""
def __init__(self, lockdir):
diff --git a/catalyst/main.py b/catalyst/main.py
index 23093e97..1d52dffe 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -106,7 +106,7 @@ def build_target(addlargs):
return target.run()
-class FilePath(object):
+class FilePath():
"""Argparse type for getting a path to a file."""
def __init__(self, exists=True):
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2020-04-13 20:43 Matt Turner
0 siblings, 0 replies; 16+ messages in thread
From: Matt Turner @ 2020-04-13 20:43 UTC (permalink / raw
To: gentoo-commits
commit: 2a0b7a7693751a5702bd077aa4f799d143eacd68
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 11 22:30:03 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Mon Apr 13 20:43:10 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=2a0b7a76
catalyst: Just import things where they're needed
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/resume.py | 3 ++-
catalyst/base/stagebase.py | 3 ++-
catalyst/fileops.py | 7 +------
3 files changed, 5 insertions(+), 8 deletions(-)
diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index a0770b9d..82f97cd3 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -9,9 +9,10 @@ set, unset, is_set, is_unset, enabled, clear_all
import os
from snakeoil import fileutils
+from snakeoil.osutils import pjoin, listdir_files
from catalyst import log
-from catalyst.fileops import ensure_dirs, pjoin, listdir_files, clear_dir
+from catalyst.fileops import ensure_dirs, clear_dir
class AutoResume():
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 27516af6..d1dd14b4 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -5,6 +5,7 @@ import shutil
import sys
from snakeoil import fileutils
+from snakeoil.osutils import pjoin
from DeComp.compress import CompressMap
@@ -17,7 +18,7 @@ from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
from catalyst.lock import LockDir, LockInUse
-from catalyst.fileops import ensure_dirs, pjoin, clear_dir, clear_path
+from catalyst.fileops import ensure_dirs, clear_dir, clear_path
from catalyst.base.resume import AutoResume
diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index fffcfc6a..671d8d9f 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -10,12 +10,7 @@ import os
import shutil
from stat import ST_UID, ST_GID, ST_MODE
-# NOTE: pjoin and listdir_files are imported here for export
-# to other catalyst modules
-# pylint: disable=unused-import
-from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs,
- pjoin, listdir_files)
-# pylint: enable=unused-import
+from snakeoil.osutils import ensure_dirs as snakeoil_ensure_dirs
from catalyst import log
from catalyst.support import CatalystError
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2020-04-22 5:52 Matt Turner
0 siblings, 0 replies; 16+ messages in thread
From: Matt Turner @ 2020-04-22 5:52 UTC (permalink / raw
To: gentoo-commits
commit: 773270f3647a0d08d6468eccb141496f26e0a601
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 18 20:15:18 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Apr 22 00:22:24 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=773270f3
catalyst: Consolidate mount code into an OrderedDict
Keeping source and target together in a single data structure makes
things tidier than having
self.mountmap (map of mount name to source mount)
self.targets_map (map of mount name to target mount)
self.mounts (list of mounts in the order they're mounted)
This also has another nice benefit: the the order that the mounts take
place is stored in one location and isn't affected by the order in which
the code chooses to enable or disable them (dependent on configuration
options).
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 94 ++++++++++++++++++++------------------------
catalyst/defaults.py | 97 +++++++++++++++++++++++++++++++++-------------
2 files changed, 112 insertions(+), 79 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index f8c1611e..cc997688 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -11,8 +11,7 @@ from snakeoil.osutils import pjoin
from DeComp.compress import CompressMap
from catalyst import log
-from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
- PORT_LOGDIR_CLEAN)
+from catalyst.defaults import (MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
cmd, read_makeconf, ismount, file_check)
from catalyst.base.targetbase import TargetBase
@@ -188,46 +187,39 @@ class StageBase(TargetBase, ClearBase, GenBase):
file_locate(self.settings, ["portage_confdir"], expand=0)
# Setup our mount points.
- # initialize our target mounts.
- self.target_mounts = TARGET_MOUNT_DEFAULTS.copy()
-
- self.mounts = ["proc", "dev", "portdir", "distdir", "port_tmpdir"]
- # initialize our source mounts
- self.mountmap = SOURCE_MOUNT_DEFAULTS.copy()
- # update these from settings
- self.mountmap["portdir"] = self.settings["portdir"]
- self.mountmap["distdir"] = self.settings["distdir"]
- self.target_mounts["portdir"] = normpath(self.settings["repo_basedir"] +
- "/" + self.settings["repo_name"])
- self.target_mounts["distdir"] = self.settings["target_distdir"]
- self.target_mounts["packagedir"] = self.settings["target_pkgdir"]
+ self.mount = MOUNT_DEFAULTS.copy()
+
+ self.mount['portdir']['source'] = self.settings['portdir']
+ self.mount['portdir']['target'] = normpath(self.settings["repo_basedir"]
+ + "/" +
+ self.settings["repo_name"])
+ self.mount['distdir']['source'] = self.settings['distdir']
+ self.mount["distdir"]['target'] = self.settings["target_distdir"]
+
if "snapcache" not in self.settings["options"]:
- self.mounts.remove("portdir")
- self.mountmap["portdir"] = None
+ self.mount['portdir']['enable'] = False
else:
- self.mountmap["portdir"] = normpath("/".join([
- self.settings["snapshot_cache_path"],
- self.settings["repo_name"],
+ self.mount['portdir']['source'] = normpath("/".join([
+ self.settings['snapshot_cache_path'],
+ self.settings['repo_name']
]))
- self.mounts.append("devpts")
- self.mounts.append("shm")
- self.mounts.append("run")
# Configure any user specified options (either in catalyst.conf or on
# the command line).
if "pkgcache" in self.settings["options"]:
self.set_pkgcache_path()
+ self.mount['pkgdir']['enable'] = True
+ self.mount['pkgdir']['source'] = self.settings['pkgcache_path']
+ self.mount['pkgdir']['target'] = self.settings["target_pkgdir"]
log.info('Location of the package cache is %s',
self.settings['pkgcache_path'])
- self.mounts.append("packagedir")
- self.mountmap["packagedir"] = self.settings["pkgcache_path"]
if "kerncache" in self.settings["options"]:
self.set_kerncache_path()
+ self.mount['kerncache']['enable'] = True
+ self.mount['kerncache']['source'] = self.settings["kerncache_path"]
log.info('Location of the kerncache is %s',
self.settings['kerncache_path'])
- self.mounts.append("kerncache")
- self.mountmap["kerncache"] = self.settings["kerncache_path"]
if "ccache" in self.settings["options"]:
if "CCACHE_DIR" in os.environ:
@@ -238,20 +230,19 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not os.path.isdir(ccdir):
raise CatalystError(
"Compiler cache support can't be enabled (can't find " + ccdir+")")
- self.mounts.append("ccache")
- self.mountmap["ccache"] = ccdir
- # for the chroot:
- self.env["CCACHE_DIR"] = self.target_mounts["ccache"]
+ self.mount['ccache']['enable'] = True
+ self.mount['ccache']['source'] = ccdir
+ self.env["CCACHE_DIR"] = self.mount['ccache']['target']
if "icecream" in self.settings["options"]:
- self.mounts.append("icecream")
- self.mountmap["icecream"] = self.settings["icecream"]
- self.env["PATH"] = self.target_mounts["icecream"] + \
+ self.mount['icecream']['enable'] = True
+ self.mount['icecream']['source'] = self.settings['icecream']
+ self.env["PATH"] = self.mount['icecream']['target'] + \
":" + self.env["PATH"]
if "port_logdir" in self.settings:
- self.mounts.append("port_logdir")
- self.mountmap["port_logdir"] = self.settings["port_logdir"]
+ self.mount['port_logdir']['enable'] = True
+ self.mount['port_logdir']['source'] = self.settings['port_logdir']
self.env["PORT_LOGDIR"] = self.settings["port_logdir"]
self.env["PORT_LOGDIR_CLEAN"] = PORT_LOGDIR_CLEAN
@@ -697,10 +688,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not os.path.exists(self.settings["chroot_path"]):
return
- log.debug('self.mounts = %s', self.mounts)
- for x in self.mounts:
- target = normpath(
- self.settings["chroot_path"] + self.target_mounts[x])
+ log.debug('self.mount = %s', self.mount)
+ for x in [x for x in self.mount if self.mount[x]['enable']]:
+ target = normpath(self.settings['chroot_path'] +
+ self.mount[x]['target'])
log.debug('mount_safety_check() x = %s %s', x, target)
if not os.path.exists(target):
continue
@@ -971,17 +962,17 @@ class StageBase(TargetBase, ClearBase, GenBase):
env=self.env)
def bind(self):
- for x in self.mounts:
+ for x in [x for x in self.mount if self.mount[x]['enable']]:
log.debug('bind(); x = %s', x)
- target = normpath(
- self.settings["chroot_path"] + self.target_mounts[x])
+ target = normpath(self.settings['chroot_path'] +
+ self.mount[x]['target'])
ensure_dirs(target, mode=0o755)
- if not os.path.exists(self.mountmap[x]):
- if self.mountmap[x] not in ("maybe_tmpfs", "tmpfs", "shmfs"):
- ensure_dirs(self.mountmap[x], mode=0o755)
+ if not os.path.exists(self.mount[x]['source']):
+ if self.mount[x]['source'] not in ("maybe_tmpfs", "tmpfs", "shmfs"):
+ ensure_dirs(self.mount[x]['source'], mode=0o755)
- src = self.mountmap[x]
+ src = self.mount[x]['source']
log.debug('bind(); src = %s', src)
if "snapcache" in self.settings["options"] and x == "portdir":
self.snapcache_lock.read_lock()
@@ -1008,11 +999,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
def unbind(self):
ouch = 0
mypath = self.settings["chroot_path"]
- myrevmounts = self.mounts[:]
- myrevmounts.reverse()
- # Unmount in reverse order for nested bind-mounts
- for x in myrevmounts:
- target = normpath(mypath + self.target_mounts[x])
+
+ # Unmount in reverse order
+ for x in [x for x in reversed(self.mount) if self.mount[x]['enable']]:
+ target = normpath(mypath + self.mount[x]['target'])
if not os.path.exists(target):
log.notice('%s does not exist. Skipping', target)
continue
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index f292c211..e09d08e8 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -1,4 +1,6 @@
+from collections import OrderedDict
+
from DeComp.definitions import DECOMPRESSOR_SEARCH_ORDER
from DeComp.definitions import COMPRESSOR_PROGRAM_OPTIONS, XATTRS_OPTIONS
from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS, LIST_XATTRS_OPTIONS
@@ -68,7 +70,7 @@ confdefaults = {
"port_conf": "/etc/portage",
"make_conf": "%(port_conf)s/make.conf",
"options": set(),
- "packagedir": PKGDIR[:],
+ "pkgdir": PKGDIR[:],
"portdir": PORTDIR[:],
"port_tmpdir": "/var/tmp/portage",
"PythonDir": "./catalyst",
@@ -89,32 +91,73 @@ DEFAULT_CONFIG_FILE = '/etc/catalyst/catalyst.conf'
PORT_LOGDIR_CLEAN = \
'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +30 -delete'
-TARGET_MOUNT_DEFAULTS = {
- "ccache": "/var/tmp/ccache",
- "dev": "/dev",
- "devpts": "/dev/pts",
- "distdir": DISTDIR[:],
- "icecream": "/usr/lib/icecc/bin",
- "kerncache": "/tmp/kerncache",
- "packagedir": PKGDIR[:],
- "portdir": PORTDIR[:],
- "port_tmpdir": "/var/tmp/portage",
- "port_logdir": "/var/log/portage",
- "proc": "/proc",
- "shm": "/dev/shm",
- "run": "/run",
-}
-
-SOURCE_MOUNT_DEFAULTS = {
- "dev": "/dev",
- "devpts": "/dev/pts",
- "distdir": DISTDIR[:],
- "portdir": PORTDIR[:],
- "port_tmpdir": "maybe_tmpfs",
- "proc": "/proc",
- "shm": "shmfs",
- "run": "tmpfs",
-}
+MOUNT_DEFAULTS = OrderedDict([
+ ('proc', {
+ 'enable': True,
+ 'source': '/proc',
+ 'target': '/proc',
+ }),
+ ('dev', {
+ 'enable': True,
+ 'source': '/dev',
+ 'target': '/dev',
+ }),
+ ('devpts', {
+ 'enable': True,
+ 'source': '/dev/pts',
+ 'target': '/dev/pts',
+ }),
+ ('shm', {
+ 'enable': True,
+ 'source': 'shmfs',
+ 'target': '/dev/shm',
+ }),
+ ('run', {
+ 'enable': True,
+ 'source': 'tmpfs',
+ 'target': '/run',
+ }),
+ ('portdir', {
+ 'enable': True,
+ 'source': 'config',
+ 'target': 'config',
+ }),
+ ('distdir', {
+ 'enable': True,
+ 'source': 'config',
+ 'target': 'config',
+ }),
+ ('pkgdir', {
+ 'enable': False,
+ 'source': 'config',
+ 'target': 'config',
+ }),
+ ('port_tmpdir', {
+ 'enable': True,
+ 'source': 'maybe_tmpfs',
+ 'target': '/var/tmp/portage',
+ }),
+ ('kerncache', {
+ 'enable': False,
+ 'source': 'config',
+ 'target': '/tmp/kerncache',
+ }),
+ ('port_logdir', {
+ 'enable': False,
+ 'source': 'config',
+ 'target': '/var/log/portage',
+ }),
+ ('ccache', {
+ 'enable': False,
+ 'source': 'config',
+ 'target': '/var/tmp/ccache',
+ }),
+ ('icecream', {
+ 'enable': False,
+ 'source': ...,
+ 'target': '/usr/lib/icecc/bin',
+ }),
+])
option_messages = {
"autoresume": "Autoresuming support enabled.",
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
@ 2020-05-16 6:43 Matt Turner
0 siblings, 0 replies; 16+ messages in thread
From: Matt Turner @ 2020-05-16 6:43 UTC (permalink / raw
To: gentoo-commits
commit: 111f8243f3261d8cb0871b6f4b2087d7ce5e9b69
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Fri May 15 06:16:28 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri May 15 06:31:34 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=111f8243
catalyst: Rename shmfs -> shm
This will allow us to factor out the ['source', 'target'] from each
case.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 4 ++--
catalyst/defaults.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 68945da8..52f9cd5b 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -864,9 +864,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
target]
elif source == 'tmpfs':
_cmd = ['mount', '-t', 'tmpfs', source, target]
- elif source == 'shmfs':
+ elif source == 'shm':
_cmd = ['mount', '-t', 'tmpfs', '-o', 'noexec,nosuid,nodev',
- 'shm', target]
+ source, target]
else:
_cmd = ['mount', source, target]
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 9d5771d5..404f4892 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -99,7 +99,7 @@ MOUNT_DEFAULTS = OrderedDict([
}),
('shm', {
'enable': True,
- 'source': 'shmfs',
+ 'source': 'shm',
'target': '/dev/shm',
}),
('run', {
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
2020-12-19 19:56 [gentoo-commits] proj/catalyst:wip/mattst88 " Matt Turner
@ 2020-11-14 16:37 ` Matt Turner
0 siblings, 0 replies; 16+ messages in thread
From: Matt Turner @ 2020-11-14 16:37 UTC (permalink / raw
To: gentoo-commits
commit: b556ff31b8d4caaef6c0e7612a70f5f0c397b02c
Author: Felix Bier <Felix.Bier <AT> rohde-schwarz <DOT> com>
AuthorDate: Tue Nov 10 00:59:01 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sat Nov 14 16:34:57 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b556ff31
Ensure deep copying of config defaults
This commit adds deep copying operations when initializing config
objects from a default config. This prevents the config from being
a shallow copy of the default, ensuring that modifications to the
config do not modify the default.
In particular, this fixes a check in write_make_conf, where the PORTDIR
variable is supposed to be only written to the generated make.conf when
a non-default repo_basedir is set in /etc/catalyst/catalyst.conf.
This check is never satisfied, because confvalues is a shallow copy of
confdefaults, therefore both will always hold the same value for
repo_basedir.
For self.mounts / MOUNT_DEFAULTS this problem can also be observed, the
modifications done to self.mounts are also visible in MOUNT_DEFAULTS.
I am not aware of any bugs due to this shallow copy, but I would prefer
adding a deep copy to prevent future bugs, in case a comparision
against the default mounts is ever needed.
Signed-off-by: Felix Bier <felix.bier <AT> rohde-schwarz.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 3 ++-
catalyst/main.py | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index a75dbdf9..21cf96a0 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1,4 +1,5 @@
+import copy
import os
import platform
import shutil
@@ -187,7 +188,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
file_locate(self.settings, ["portage_confdir"], expand=0)
# Setup our mount points.
- self.mount = MOUNT_DEFAULTS.copy()
+ self.mount = copy.deepcopy(MOUNT_DEFAULTS)
self.mount['portdir']['source'] = self.snapshot
self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/' + self.settings['repo_name']
diff --git a/catalyst/main.py b/catalyst/main.py
index 5536471a..48daf004 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -1,4 +1,5 @@
import argparse
+import copy
import datetime
import hashlib
import os
@@ -19,7 +20,7 @@ from catalyst.defaults import (confdefaults, option_messages,
from catalyst.support import CatalystError
from catalyst.version import get_version
-conf_values = confdefaults
+conf_values = copy.deepcopy(confdefaults)
def version():
^ permalink raw reply related [flat|nested] 16+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
2021-01-18 19:53 [gentoo-commits] proj/catalyst:pending/mattst88 " Matt Turner
@ 2020-11-14 16:37 ` Matt Turner
0 siblings, 0 replies; 16+ messages in thread
From: Matt Turner @ 2020-11-14 16:37 UTC (permalink / raw
To: gentoo-commits
commit: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6
Author: Felix Bier <Felix.Bier <AT> rohde-schwarz <DOT> com>
AuthorDate: Tue Nov 10 01:03:03 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sat Nov 14 16:34:57 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=37a38605
Move from PORTDIR_OVERLAY to repos.conf
This commit fixes the following issues:
* The PORTDIR_OVERLAY variable has been deprecated by Gentoo.
With this commit, the variable is no longer written to the
generated make.conf. Instead, a config file
/etc/portage/repos.conf/<repo-name>.conf
is generated for each overlay. The repo name is read from the
overlay using the portage API. Internally, portage parses
metadata/layout.conf and profiles/repo_name to obtain the name.
References:
https://wiki.gentoo.org/wiki//etc/portage/make.conf
https://wiki.gentoo.org/wiki//etc/portage/repos.conf
* All overlays were copied into the same target directory. If the
same file name occurred in multiple overlays, the last overlay
would overwrite all previous files with this name. In particular,
only the metadata/layout.conf of the last overlay was retained,
so it was not possible to reference the other overlays e.g. via
the masters entry in the layout.conf or the portage-2 syntax
for specifying a parent profile from another overlay. Also,
this created problems when the overlays contained ebuilds
for the same package, but with differing versions, because
after copying, the target directory contained both versions of the
ebuild but only the manifest file of the last overlay.
With this commit, each overlay is copied into a separate
sub-directory, e.g. /var/db/repos/<repo-name>.
This directory is referenced via the location entry in the
generated /etc/portage/repos.conf/<repo-name>.conf.
Signed-off-by: Felix Bier <felix.bier <AT> rohde-schwarz.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 84 ++++++++++++++++++++++++++++++++++------------
catalyst/defaults.py | 2 +-
catalyst/support.py | 18 ++++++++++
3 files changed, 81 insertions(+), 23 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 21cf96a0..fe79b55a 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1,4 +1,5 @@
+import configparser
import copy
import os
import platform
@@ -19,8 +20,8 @@ from catalyst import log
from catalyst.context import namespace
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
- cmd, read_makeconf, ismount, file_check,
- sanitize_name)
+ cmd, read_makeconf, get_repo_name, ismount,
+ file_check, sanitize_name)
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
@@ -786,17 +787,55 @@ class StageBase(TargetBase, ClearBase, GenBase):
env=self.env)
self.resume.enable("setup_confdir")
+ def to_chroot(self, path):
+ """ Prepend chroot path to the given path. """
+
+ chroot = Path(self.settings['chroot_path'])
+ return chroot / path.relative_to(path.anchor)
+
+ def get_repo_conf_path(self, repo_name):
+ """ Construct repo conf path: {repos_conf}/{name}.conf """
+ return Path(self.settings['repos_conf'], repo_name + ".conf")
+
+ def get_repo_location(self, repo_name):
+ """ Construct overlay repo path: {repo_basedir}/{name} """
+ return Path(self.settings['repo_basedir'], repo_name)
+
+ def write_repo_conf(self, repo_name, config):
+ """ Write ConfigParser to {chroot}/{repos_conf}/{name}.conf """
+
+ repo_conf = self.get_repo_conf_path(repo_name)
+
+ repo_conf_chroot = self.to_chroot(repo_conf)
+ repo_conf_chroot.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+ log.info(f'Creating repo config {repo_conf_chroot}.')
+
+ try:
+ with open(repo_conf_chroot, 'w') as f:
+ config.write(f)
+ except OSError as e:
+ raise CatalystError(f'Could not write {repo_conf_chroot}: {e}') from e
+
def portage_overlay(self):
- """ We copy the contents of our overlays to /usr/local/portage """
+ """ We copy the contents of our repos to get_repo_location(repo_name) """
if "portage_overlay" in self.settings:
for x in self.settings["portage_overlay"]:
if os.path.exists(x):
- log.info('Copying overlay dir %s', x)
- ensure_dirs(
- self.settings['chroot_path'] + self.settings['local_overlay'])
- cmd("cp -a " + x + "/* " + self.settings["chroot_path"] +
- self.settings["local_overlay"],
- env=self.env)
+ name = get_repo_name(x)
+
+ location = self.get_repo_location(name)
+ config = configparser.ConfigParser()
+ config[name] = {'location': location}
+ self.write_repo_conf(name, config)
+
+ location_chroot = self.to_chroot(location)
+ location_chroot.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+ log.info(f'Copying overlay dir {x} to {location_chroot}')
+ cmd(f'cp -a {x}/* {location_chroot}', env=self.env)
+ else:
+ log.warning(f'Skipping missing overlay {x}.')
def root_overlay(self):
""" Copy over the root_overlay """
@@ -852,8 +891,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
cxt = libmount.Context(source=source, target=target,
fstype=fstype, options=options)
cxt.mount()
- except OSError as e:
- raise CatalystError(f"Couldn't mount: {source}, {e.strerror}")
+ except Exception as e:
+ raise CatalystError(f"Couldn't mount: {source}, {e}")
def chroot_setup(self):
self.makeconf = read_makeconf(normpath(self.settings["chroot_path"] +
@@ -1018,12 +1057,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
varname = x.split('_')[1].upper()
myf.write(f'{varname}="{self.settings[x]}"\n')
- if setup:
- # Setup the portage overlay
- if "portage_overlay" in self.settings:
- myf.write('PORTDIR_OVERLAY="%s"\n' %
- self.settings["local_overlay"])
-
# Set default locale for system responses. #478382
myf.write(
'\n'
@@ -1097,11 +1130,18 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.warning("You've been hacking. Clearing target patches: %s", target)
clear_path(target)
- # Remove our overlay
- overlay = normpath(
- self.settings["chroot_path"] + self.settings["local_overlay"])
- if os.path.exists(overlay):
- clear_path(overlay)
+ # Remove our overlays
+ if "portage_overlay" in self.settings:
+ for repo_path in self.settings["portage_overlay"]:
+ repo_name = get_repo_name(repo_path)
+
+ repo_conf = self.get_repo_conf_path(repo_name)
+ chroot_repo_conf = self.to_chroot(repo_conf)
+ chroot_repo_conf.unlink()
+
+ location = self.get_repo_location(repo_name)
+ chroot_location = self.to_chroot(location)
+ clear_path(str(chroot_location))
if "sticky-config" not in self.settings["options"]:
# re-write the make.conf to be sure it is clean
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 0f399b56..3f12b8d5 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -38,9 +38,9 @@ confdefaults = {
"distdir": portage.settings['DISTDIR'],
"icecream": "/var/cache/icecream",
'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'],
- "local_overlay": "/var/db/repos/local",
"port_conf": "/etc/portage",
"make_conf": "%(port_conf)s/make.conf",
+ "repos_conf": "%(port_conf)s/repos.conf",
"options": set(),
"pkgdir": "/var/cache/binpkgs",
"port_tmpdir": "/var/tmp/portage",
diff --git a/catalyst/support.py b/catalyst/support.py
index ddbd9ab9..f3a865a7 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -10,6 +10,8 @@ from subprocess import Popen
import libmount
+from portage.repository.config import RepoConfig
+
from catalyst import log
BASH_BINARY = "/bin/bash"
@@ -182,6 +184,22 @@ def read_makeconf(mymakeconffile):
return makeconf
+def get_repo_name(repo_path):
+ """ Get the name of the repo at the given repo_path.
+
+ References:
+ https://wiki.gentoo.org/wiki/Repository_format/profiles/repo_name
+ https://wiki.gentoo.org/wiki/Repository_format/metadata/layout.conf#repo-name
+ """
+
+ repo_config = RepoConfig(None, {"location": repo_path})
+
+ if repo_config.missing_repo_name:
+ raise CatalystError(f"Missing name in repository {repo_path}")
+
+ return repo_config.name
+
+
def ismount(path):
"""Like os.path.ismount, but also support bind mounts"""
path = Path(path)
^ permalink raw reply related [flat|nested] 16+ messages in thread
end of thread, other threads:[~2020-11-14 16:37 UTC | newest]
Thread overview: 16+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-04-10 21:04 [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/ Matt Turner
-- strict thread matches above, loose matches on Subject: below --
2021-01-18 19:53 [gentoo-commits] proj/catalyst:pending/mattst88 " Matt Turner
2020-11-14 16:37 ` [gentoo-commits] proj/catalyst:master " Matt Turner
2020-12-19 19:56 [gentoo-commits] proj/catalyst:wip/mattst88 " Matt Turner
2020-11-14 16:37 ` [gentoo-commits] proj/catalyst:master " Matt Turner
2020-05-16 6:43 Matt Turner
2020-04-22 5:52 Matt Turner
2020-04-13 20:43 Matt Turner
2019-10-20 0:00 Matt Turner
2015-10-11 17:26 Mike Frysinger
2015-10-09 2:40 Mike Frysinger
2015-10-09 2:40 Mike Frysinger
2015-10-06 15:31 Mike Frysinger
2015-09-09 15:22 Brian Dolbec
2015-05-21 23:53 [gentoo-commits] proj/catalyst:pending " Brian Dolbec
2015-05-24 0:08 ` [gentoo-commits] proj/catalyst:master " Brian Dolbec
2015-02-26 22:18 Brian Dolbec
2015-01-01 5:59 [gentoo-commits] proj/catalyst:pending commit in: catalyst/base/, catalyst/ Brian Dolbec
2015-02-26 20:12 ` [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/ Brian Dolbec
2015-01-01 5:59 [gentoo-commits] proj/catalyst:pending " Brian Dolbec
2015-02-26 4:12 ` [gentoo-commits] proj/catalyst:master " Brian Dolbec
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox