From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 7975A1389E2 for ; Thu, 1 Jan 2015 05:59:20 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 5277AE08B0; Thu, 1 Jan 2015 05:59:12 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id D8A03E08A2 for ; Thu, 1 Jan 2015 05:59:09 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id B2C253406DE for ; Thu, 1 Jan 2015 05:59:08 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id EF368E9FB for ; Thu, 1 Jan 2015 05:59:04 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1420091887.a1c2785438593104d7c3d2ec20dd4fb25cb19884.dolsen@gentoo> Subject: [gentoo-commits] proj/catalyst:pending commit in: catalyst/targets/, catalyst/base/ X-VCS-Repository: proj/catalyst X-VCS-Files: catalyst/base/clearbase.py catalyst/base/resume.py catalyst/base/stagebase.py catalyst/targets/grp.py catalyst/targets/livecd_stage1.py catalyst/targets/livecd_stage2.py catalyst/targets/netboot2.py X-VCS-Directories: catalyst/targets/ catalyst/base/ X-VCS-Committer: dolsen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: a1c2785438593104d7c3d2ec20dd4fb25cb19884 X-VCS-Branch: pending Date: Thu, 1 Jan 2015 05:59:04 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 60788a13-5c74-4eb8-bb15-a7f43e1cbfb5 X-Archives-Hash: 8ccc55b89f55e91038574755960d4d39 commit: a1c2785438593104d7c3d2ec20dd4fb25cb19884 Author: Brian Dolbec gentoo org> AuthorDate: Sat Jun 1 06:23:46 2013 +0000 Commit: Brian Dolbec gentoo org> CommitDate: Thu Jan 1 05:58:07 2015 +0000 URL: http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=a1c27854 Create AutoResume class to handle all file creation, deletion... Create catalyst/base/resume.py. Migrate all auto_resume operations to using the new class. --- catalyst/base/clearbase.py | 21 +---- catalyst/base/resume.py | 166 ++++++++++++++++++++++++++++++++++++++ catalyst/base/stagebase.py | 156 +++++++++++++++-------------------- catalyst/targets/grp.py | 4 +- catalyst/targets/livecd_stage1.py | 4 +- catalyst/targets/livecd_stage2.py | 14 ++-- catalyst/targets/netboot2.py | 19 +++-- 7 files changed, 250 insertions(+), 134 deletions(-) diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py index 8462a3c..0ebe299 100644 --- a/catalyst/base/clearbase.py +++ b/catalyst/base/clearbase.py @@ -13,31 +13,14 @@ class ClearBase(object): """ def __init__(self, myspec): self.settings = myspec - + self.resume = None def clear_autoresume(self): """ Clean resume points since they are no longer needed """ if "autoresume" in self.settings["options"]: print "Removing AutoResume Points: ..." - myemp=self.settings["autoresume_path"] - if os.path.isdir(myemp): - if "autoresume" in self.settings["options"]: - print "Emptying directory",myemp - """ - stat the dir, delete the dir, recreate the dir and set - the proper perms and ownership - """ - mystat=os.stat(myemp) - if os.uname()[0] == "FreeBSD": - cmd("chflags -R noschg "+myemp,\ - "Could not remove immutable flag for file "\ - +myemp) - #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env) - shutil.rmtree(myemp) - ensure_dirs(myemp, 0755) - os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) - os.chmod(myemp,mystat[ST_MODE]) + self.resume.clear_all() def clear_chroot(self): diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py new file mode 100644 index 0000000..e42c7dc --- /dev/null +++ b/catalyst/base/resume.py @@ -0,0 +1,166 @@ +#!/usr/bin/python + +# Maintained in full by: +# Catalyst Team +# Release Engineering Team +# Copyright 2013 Brian Dolbec + +'''resume.py + +Performs autoresume tracking file operations such as +set, unset, is_set, is_unset, enabled, clear_all +''' + +import os +import shutil +from stat import ST_UID, ST_GID, ST_MODE +import traceback + +from catalyst.fileops import ensure_dirs, pjoin, listdir_files +from catalyst.support import touch + + +class AutoResumeError(Exception): + def __init__(self, message, print_traceback=False): + if message: + if print_traceback: + (type,value)=sys.exc_info()[:2] + if value!=None: + print + print "Traceback values found. listing..." + print traceback.print_exc(file=sys.stdout) + print "!!! catalyst: AutoResumeError " + message + print + + +class AutoResume(object): + '''Class for tracking and handling all aspects of + the autoresume option and related files. + ''' + + + def __init__(self, basedir, mode=0755): + self.basedir = basedir + ensure_dirs(basedir, mode=mode, fatal=True) + self._points = {} + self._init_points_() + + + def _init_points_(self): + '''Internal function which reads the autoresume directory and + for existing autoresume points and adds them to our _points variable + ''' + existing = listdir_files(self.basedir, False) + for point in existing: + self._points[point] = pjoin(self.basedir, point) + + + def enable(self, point, data=None): + '''Sets the resume point 'ON' + + @param point: string. name of the resume point to enable + @param data: string of information to store, or None + @return boolean + ''' + if point in self._points and not data: + return True + fname = pjoin(self.basedir, point) + if data: + with open(fname,"w") as myf: + myf.write(data) + else: + try: + touch(fname) + self._points[point] = fname + except Exception as e: + print AutoResumeError(str(e)) + return False + return True + + + def get(self, point, no_lf=True): + '''Gets any data stored inside a resume point + + @param point: string. name of the resume point to enable + @return data: string of information stored, or None + ''' + if point in self._points: + try: + with open(self._points[point], 'r') as myf: + data = myf.read() + if data and no_lf: + data = data.replace('\n', '') + except OSError as e: + print AutoResumeError(str(e)) + return None + return data + return None + + + def disable(self, point): + '''Sets the resume point 'OFF' + + @param point: string. name of the resume point to disable + @return boolean + ''' + if point not in self._points: + return True + try: + os.unlink(self._points[point]) + self._points.pop(point) + except Exception as e: + print AutoResumeError(str(e)) + return False + return True + + + def is_enabled(self, point): + '''Returns True if the resume point 'ON' + + @param point: string. name of the resume point enabled + @return boolean + ''' + return point in self._points + + + def is_disabled(self, point): + '''Returns True if the resume point 'OFF' + + @param point: string. name of the resume point not enabled + @return boolean + ''' + return point not in self._points + + + @property + def enabled(self): + '''Returns a list of enabled points + ''' + return list(self._points) + + + def clear_all(self): + '''Clear all active resume points + + @return boolean + ''' + try: + print "Emptying directory---", self.basedir + """ + stat the dir, delete the dir, recreate the dir and set + the proper perms and ownership + """ + mystat=os.stat(self.basedir) + if os.uname()[0] == "FreeBSD": + cmd("chflags -R noschg " + self.basedir,\ + "Could not remove immutable flag for file "\ + + self.basedir) + shutil.rmtree(self.basedir) + ensure_dirs(self.basedir, 0755) + os.chown(self.basedir,mystat[ST_UID],mystat[ST_GID]) + os.chmod(self.basedir,mystat[ST_MODE]) + self._points = {} + except Exception as e: + print AutoResumeError(str(e)) + return False + return True diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 40c7362..f3cb3e3 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -19,6 +19,7 @@ from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase from catalyst.lock import LockDir from catalyst.fileops import ensure_dirs, pjoin +from catalyst.base.resume import AutoResume class StageBase(TargetBase, ClearBase, GenBase): @@ -323,10 +324,8 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_target_path(self): self.settings["target_path"]=normpath(self.settings["storedir"]+\ "/builds/"+self.settings["target_subpath"]+".tar.bz2") - setup_target_path_resume = pjoin(self.settings["autoresume_path"], - "setup_target_path") - if "autoresume" in self.settings["options"] and \ - os.path.exists(setup_target_path_resume): + if "autoresume" in self.settings["options"]\ + and self.resume.is_enabled("setup_target_path"): print \ "Resume point detected, skipping target path setup operation..." else: @@ -338,7 +337,7 @@ class StageBase(TargetBase, ClearBase, GenBase): # cmd("rm -f "+self.settings["target_path"],\ # "Could not remove existing file: "\ # +self.settings["target_path"],env=self.env) - touch(setup_target_path_resume) + self.resume.enable("setup_target_path") ensure_dirs(self.settings["storedir"] + "/builds/") @@ -486,7 +485,7 @@ class StageBase(TargetBase, ClearBase, GenBase): )) if "autoresume" in self.settings["options"]: print "The autoresume path is " + self.settings["autoresume_path"] - ensure_dirs(self.settings["autoresume_path"], mode=0755) + self.resume = AutoResume(self.settings["autoresume_path"], mode=0755) def set_controller_file(self): self.settings["controller_file"]=normpath(self.settings["sharedir"]+\ @@ -667,8 +666,7 @@ class StageBase(TargetBase, ClearBase, GenBase): def unpack(self): unpack=True - unpack_resume = pjoin(self.settings["autoresume_path"], "unpack") - clst_unpack_hash=read_from_clst(unpack_resume) + clst_unpack_hash = self.resume.get("unpack") if "seedcache" in self.settings["options"]: if os.path.isdir(self.settings["source_path"]): @@ -714,7 +712,7 @@ class StageBase(TargetBase, ClearBase, GenBase): if "autoresume" in self.settings["options"]: if os.path.isdir(self.settings["source_path"]) \ - and os.path.exists(unpack_resume): + and self.resume.is_enabled("unpack"): """ Autoresume is valid, SEEDCACHE is valid """ unpack=False invalid_snapshot=False @@ -726,7 +724,7 @@ class StageBase(TargetBase, ClearBase, GenBase): invalid_snapshot=True elif os.path.isdir(self.settings["source_path"]) \ - and not os.path.exists(unpack_resume): + and self.resume.is_disabled("unpack"): """ Autoresume is invalid, SEEDCACHE """ unpack=True invalid_snapshot=False @@ -782,19 +780,16 @@ class StageBase(TargetBase, ClearBase, GenBase): cmd(unpack_cmd,error_msg,env=self.env) if "source_path_hash" in self.settings: - myf=open(unpack_resume,"w") - myf.write(self.settings["source_path_hash"]) - myf.close() + self.resume.enable("unpack", + data=self.settings["source_path_hash"]) else: - touch(unpack_resume) + self.resume.enable("unpack") else: print "Resume point detected, skipping unpack operation..." def unpack_snapshot(self): unpack=True - unpack_portage_resume = pjoin(self.settings["autoresume_path"], - "unpack_portage") - snapshot_hash=read_from_clst(unpack_portage_resume) + snapshot_hash = self.resume.get("unpack_portage") if "snapcache" in self.settings["options"]: snapshot_cache_hash=\ @@ -831,7 +826,7 @@ class StageBase(TargetBase, ClearBase, GenBase): if "autoresume" in self.settings["options"] \ and os.path.exists(self.settings["chroot_path"]+\ self.settings["portdir"]) \ - and os.path.exists(unpack_portage_resume) \ + and self.resume.is_enabled("unpack_portage") \ and self.settings["snapshot_path_hash"] == snapshot_hash: print \ "Valid Resume point detected, skipping unpack of portage tree..." @@ -856,18 +851,15 @@ class StageBase(TargetBase, ClearBase, GenBase): myf.close() else: print "Setting snapshot autoresume point" - myf=open(unpack_portage_resume,"w") - myf.write(self.settings["snapshot_path_hash"]) - myf.close() + self.resume.enable("unpack_portage", + data=self.settings["snapshot_path_hash"]) if "snapcache" in self.settings["options"]: self.snapshot_lock_object.unlock() def config_profile_link(self): - config_protect_link_resume = pjoin(self.settings["autoresume_path"], - "config_profile_link") if "autoresume" in self.settings["options"] \ - and os.path.exists(config_protect_link_resume): + and self.resume.is_enabled("config_profile_link"): print \ "Resume point detected, skipping config_profile_link operation..." else: @@ -881,21 +873,20 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["target_profile"]+" "+\ self.settings["chroot_path"]+"/etc/portage/make.profile",\ "Error creating profile link",env=self.env) - touch(config_protect_link_resume) + self.resume.enable("config_profile_link") def setup_confdir(self): - setup_confdir_resume = pjoin(self.settings["autoresume_path"], - "setup_confdir") if "autoresume" in self.settings["options"] \ - and os.path.exists(setup_confdir_resume): + and self.resume.is_enabled("setup_confdir"): print "Resume point detected, skipping setup_confdir operation..." else: if "portage_confdir" in self.settings: - print "Configuring /etc/portage..." - cmd("rsync -a "+self.settings["portage_confdir"]+"/ "+\ - self.settings["chroot_path"]+"/etc/portage/",\ - "Error copying /etc/portage",env=self.env) - touch(setup_confdir_resume) + print "Configuring %s..." % self.settings["port_conf"] + cmd("rsync -a " + self.settings["portage_confdir"] + "/ " + + self.settings["chroot_path"] + self.settings["port_conf"], + "Error copying %s" % self.settings["port_conf"], + env=self.env) + self.resume.enable("setup_confdir") def portage_overlay(self): """ We copy the contents of our overlays to /usr/local/portage """ @@ -1018,10 +1009,8 @@ class StageBase(TargetBase, ClearBase, GenBase): self.override_cflags() self.override_cxxflags() self.override_ldflags() - chroot_setup_resume = pjoin(self.settings["autoresume_path"], - "chroot_setup") if "autoresume" in self.settings["options"] \ - and os.path.exists(chroot_setup_resume): + and self.resume.is_enabled("chroot_setup"): print "Resume point detected, skipping chroot_setup operation..." else: print "Setting up chroot..." @@ -1118,35 +1107,32 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["make_conf"]) cmd("cp " + makepath + " " + makepath + ".catalyst",\ "Could not backup " + self.settings["make_conf"],env=self.env) - touch(self.settings["autoresume_path"]+"chroot_setup") + self.resume.enable("chroot_setup") def fsscript(self): - fsscript_resume = pjoin(self.settings["autoresume_path"], "fsscript") if "autoresume" in self.settings["options"] \ - and os.path.exists(fsscript_resume): + and self.resume.is_enabled("fsscript"): print "Resume point detected, skipping fsscript operation..." else: if "fsscript" in self.settings: if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+\ " fsscript","fsscript script failed.",env=self.env) - touch(fsscript_resume) + self.resume.enable("fsscript") def rcupdate(self): - rcupdate_resume = pjoin(self.settings["autoresume_path"], "rcupdate") if "autoresume" in self.settings["options"] \ - and os.path.exists(rcupdate_resume): + and self.resume.is_enabled("rcupdate"): print "Resume point detected, skipping rcupdate operation..." else: if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+" rc-update",\ "rc-update script failed.",env=self.env) - touch(rcupdate_resume) + self.resume.enable("rcupdate") def clean(self): - clean_resume = pjoin(self.settings["autoresume_path"], "clean") if "autoresume" in self.settings["options"] \ - and os.path.exists(clean_resume): + and self.resume.is_enabled("clean"): print "Resume point detected, skipping clean operation..." else: for x in self.settings["cleanables"]: @@ -1177,12 +1163,11 @@ class StageBase(TargetBase, ClearBase, GenBase): if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+" clean",\ "clean script failed.",env=self.env) - touch(clean_resume) + self.resume.enable("clean") def empty(self): - empty_resume = pjoin(self.settings["autoresume_path"], "empty") if "autoresume" in self.settings["options"] \ - and os.path.exists(empty_resume): + and self.resume.is_enabled("empty"): print "Resume point detected, skipping empty operation..." else: if self.settings["spec_prefix"]+"/empty" in self.settings: @@ -1206,12 +1191,11 @@ class StageBase(TargetBase, ClearBase, GenBase): ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) - touch(empty_resume) + self.resume.enable("empty") def remove(self): - remove_resume = pjoin(self.settings["autoresume_path"], "remove") if "autoresume" in self.settings["options"] \ - and os.path.exists(remove_resume): + and self.resume.is_enabled("remove"): print "Resume point detected, skipping remove operation..." else: if self.settings["spec_prefix"]+"/rm" in self.settings: @@ -1226,31 +1210,29 @@ class StageBase(TargetBase, ClearBase, GenBase): if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+\ " clean","Clean failed.",env=self.env) - touch(remove_resume) + self.resume.enable("remove") except: self.unbind() raise def preclean(self): - preclean_resume = pjoin(self.settings["autoresume_path"], "preclean") if "autoresume" in self.settings["options"] \ - and os.path.exists(preclean_resume): + and self.resume.is_enabled("preclean"): print "Resume point detected, skipping preclean operation..." else: try: if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+\ " preclean","preclean script failed.",env=self.env) - touch(preclean_resume) + self.resume.enable("preclean") except: self.unbind() raise CatalystError("Build failed, could not execute preclean") def capture(self): - capture_resume = pjoin(self.settings["autoresume_path"], "capture") if "autoresume" in self.settings["options"] \ - and os.path.exists(capture_resume): + and self.resume.is_enabled("capture"): print "Resume point detected, skipping capture operation..." else: """ Capture target in a tarball """ @@ -1270,12 +1252,11 @@ class StageBase(TargetBase, ClearBase, GenBase): self.gen_contents_file(self.settings["target_path"]) self.gen_digest_file(self.settings["target_path"]) - touch(capture_resume) + self.resume.enable("capture") def run_local(self): - run_local_resume = pjoin(self.settings["autoresume_path"], "run_local") if "autoresume" in self.settings["options"] \ - and os.path.exists(run_local_resume): + and self.resume.is_enabled("run_local"): print "Resume point detected, skipping run_local operation..." else: try: @@ -1283,7 +1264,9 @@ class StageBase(TargetBase, ClearBase, GenBase): print "run_local() starting controller script..." cmd(self.settings["controller_file"]+" run",\ "run script failed.",env=self.env) - touch(run_local_resume) + self.resume.enable("run_local") + else: + print "run_local() no controller_file found...", self.settings["controller_file"] except CatalystError: self.unbind() @@ -1359,9 +1342,8 @@ class StageBase(TargetBase, ClearBase, GenBase): self.chroot_lock.unlock() def unmerge(self): - unmerge_resume = pjoin(self.settings["autoresume_path"], "unmerge") if "autoresume" in self.settings["options"] \ - and os.path.exists(unmerge_resume): + and self.resume.is_enabled("unmerge"): print "Resume point detected, skipping unmerge operation..." else: if self.settings["spec_prefix"]+"/unmerge" in self.settings: @@ -1389,26 +1371,22 @@ class StageBase(TargetBase, ClearBase, GenBase): except CatalystError: self.unbind() raise - touch(unmerge_resume) + self.resume.enable("unmerge") def target_setup(self): - target_setup_resume = pjoin(self.settings["autoresume_path"], - "target_setup") if "autoresume" in self.settings["options"] \ - and os.path.exists(target_setup_resume): + and self.resume.is_enabled("target_setup"): print "Resume point detected, skipping target_setup operation..." else: print "Setting up filesystems per filesystem type" cmd(self.settings["controller_file"]+\ " target_image_setup "+ self.settings["target_path"],\ "target_image_setup script failed.",env=self.env) - touch(target_setup_resume) + self.resume.enable("target_setup") def setup_overlay(self): - setup_overlay_resume = pjoin(self.settings["autoresume_path"], - "setup_overlay") if "autoresume" in self.settings["options"] \ - and os.path.exists(setup_overlay_resume): + and self.resume.is_enabled("setup_overlay"): print "Resume point detected, skipping setup_overlay operation..." else: if self.settings["spec_prefix"]+"/overlay" in self.settings: @@ -1418,12 +1396,11 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["target_path"],\ self.settings["spec_prefix"]+"overlay: "+x+\ " copy failed.",env=self.env) - touch(setup_overlay_resume) + self.resume.enable("setup_overlay") def create_iso(self): - create_iso_resume = pjoin(self.settings["autoresume_path"], "create_iso") if "autoresume" in self.settings["options"] \ - and os.path.exists(create_iso_resume): + and self.resume.is_enabled("create_iso"): print "Resume point detected, skipping create_iso operation..." else: """ Create the ISO """ @@ -1433,7 +1410,7 @@ class StageBase(TargetBase, ClearBase, GenBase): env=self.env) self.gen_contents_file(self.settings["iso"]) self.gen_digest_file(self.settings["iso"]) - touch(create_iso_resume) + self.resume.enable("create_iso") else: print "WARNING: livecd/iso was not defined." print "An ISO Image will not be created." @@ -1442,13 +1419,12 @@ class StageBase(TargetBase, ClearBase, GenBase): build_packages_resume = pjoin(self.settings["autoresume_path"], "build_packages") if "autoresume" in self.settings["options"] \ - and os.path.exists(build_packages_resume): + and self.resume.is_enabled("build_packages"): print "Resume point detected, skipping build_packages operation..." else: if self.settings["spec_prefix"]+"/packages" in self.settings: if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+\ - "build_packages"): + and self.resume.is_enabled("build_packages"): print "Resume point detected, skipping build_packages operation..." else: mypack=\ @@ -1459,6 +1435,7 @@ class StageBase(TargetBase, ClearBase, GenBase): " build_packages "+mypack,\ "Error in attempt to build packages",env=self.env) touch(build_packages_resume) + self.resume.enable("build_packages") except CatalystError: self.unbind() raise CatalystError(self.settings["spec_prefix"]+\ @@ -1466,10 +1443,8 @@ class StageBase(TargetBase, ClearBase, GenBase): def build_kernel(self): '''Build all configured kernels''' - build_kernel_resume = pjoin(self.settings["autoresume_path"], - "build_kernel") if "autoresume" in self.settings["options"] \ - and os.path.exists(build_kernel_resume): + and self.resume.is_enabled("build_kernel"): print "Resume point detected, skipping build_kernel operation..." else: if "boot/kernel" in self.settings: @@ -1485,7 +1460,7 @@ class StageBase(TargetBase, ClearBase, GenBase): env=self.env) for kname in mynames: self._build_kernel(kname=kname) - touch(build_kernel_resume) + self.resume.enable("build_kernel") except CatalystError: self.unbind() raise CatalystError( @@ -1494,10 +1469,8 @@ class StageBase(TargetBase, ClearBase, GenBase): def _build_kernel(self, kname): "Build a single configured kernel by name" - kname_resume = pjoin(self.settings["autoresume_path"], - "build_kernel_" + kname) if "autoresume" in self.settings["options"] \ - and os.path.exists(kname_resume): + and self.resume.is_enabled("build_kernel_"+kname): print "Resume point detected, skipping build_kernel for "+kname+" operation..." return self._copy_kernel_config(kname=kname) @@ -1539,7 +1512,7 @@ class StageBase(TargetBase, ClearBase, GenBase): cmd("rm -R "+self.settings["chroot_path"]+\ "/tmp/initramfs_overlay/",env=self.env) - touch(kname_resume) + self.resume.is_enabled("build_kernel_"+kname) """ Execute the script that cleans up the kernel build @@ -1592,31 +1565,28 @@ class StageBase(TargetBase, ClearBase, GenBase): "/initramfs_overlay"],env=self.env) def bootloader(self): - bootloader_resume = pjoin(self.settings["autoresume_path"], "bootloader") if "autoresume" in self.settings["options"] \ - and os.path.exists(bootloader_resume): + and self.resume.is_enabled("bootloader"): print "Resume point detected, skipping bootloader operation..." else: try: cmd(self.settings["controller_file"]+\ " bootloader " + self.settings["target_path"],\ "Bootloader script failed.",env=self.env) - touch(bootloader_resume) + self.resume.enable("bootloader") except CatalystError: self.unbind() raise CatalystError("Script aborting due to error.") def livecd_update(self): - livecd_update_resume = pjoin(self.settings["autoresume_path"], - "livecd_update") if "autoresume" in self.settings["options"] \ - and os.path.exists(livecd_update_resume): + and self.resume.is_enabled("livecd_update"): print "Resume point detected, skipping build_packages operation..." else: try: cmd(self.settings["controller_file"]+\ " livecd-update","livecd-update failed.",env=self.env) - touch(livecd_update_resume) + self.resume.enable("livecd_update") except CatalystError: self.unbind() diff --git a/catalyst/targets/grp.py b/catalyst/targets/grp.py index a17f9bd..14cffe0 100644 --- a/catalyst/targets/grp.py +++ b/catalyst/targets/grp.py @@ -44,7 +44,7 @@ class grp(StageBase): def set_target_path(self): self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]+"/") if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"): + and self.resume.is_enabled("setup_target_path"): print "Resume point detected, skipping target path setup operation..." else: # first clean up any existing target stuff @@ -53,7 +53,7 @@ class grp(StageBase): #"Could not remove existing directory: "+self.settings["target_path"],env=self.env) ensure_dirs(self.settings["target_path"]) - touch(self.settings["autoresume_path"]+"setup_target_path") + self.resume.enable("setup_target_path") def run_local(self): for pkgset in self.settings["grp"]: diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index 668960b..55859bc 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -35,14 +35,14 @@ class livecd_stage1(StageBase): def set_target_path(self): self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]) if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"): + and self.resume.is_enabled("setup_target_path"): print "Resume point detected, skipping target path setup operation..." else: # first clean up any existing target stuff if os.path.exists(self.settings["target_path"]): cmd("rm -rf "+self.settings["target_path"],\ "Could not remove existing directory: "+self.settings["target_path"],env=self.env) - touch(self.settings["autoresume_path"]+"setup_target_path") + self.resume.enable("setup_target_path") ensure_dirs(self.settings["target_path"]) diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index 94da670..973a734 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -55,14 +55,14 @@ class livecd_stage2(StageBase): def set_target_path(self): self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]+"/") if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"): + and self.resume.is_enabled("setup_target_path"): print "Resume point detected, skipping target path setup operation..." else: # first clean up any existing target stuff if os.path.isdir(self.settings["target_path"]): cmd("rm -rf "+self.settings["target_path"], "Could not remove existing directory: "+self.settings["target_path"],env=self.env) - touch(self.settings["autoresume_path"]+"setup_target_path") + self.resume.enable("setup_target_path") ensure_dirs(self.settings["target_path"]) def run_local(self): @@ -89,7 +89,7 @@ class livecd_stage2(StageBase): unpack=True display_msg=None - clst_unpack_hash=read_from_clst(self.settings["autoresume_path"]+"unpack") + clst_unpack_hash = self.resume.get("unpack") if os.path.isdir(self.settings["source_path"]): unpack_cmd="rsync -a --delete "+self.settings["source_path"]+" "+self.settings["chroot_path"] @@ -100,7 +100,7 @@ class livecd_stage2(StageBase): if "autoresume" in self.settings["options"]: if os.path.isdir(self.settings["source_path"]) and \ - os.path.exists(self.settings["autoresume_path"]+"unpack"): + self.resume.is_enabled("unpack"): print "Resume point detected, skipping unpack operation..." unpack=False elif "source_path_hash" in self.settings: @@ -131,11 +131,9 @@ class livecd_stage2(StageBase): cmd(unpack_cmd,error_msg,env=self.env) if "source_path_hash" in self.settings: - myf=open(self.settings["autoresume_path"]+"unpack","w") - myf.write(self.settings["source_path_hash"]) - myf.close() + self.resume.enable("unpack", data=self.settings["source_path_hash"]) else: - touch(self.settings["autoresume_path"]+"unpack") + self.resume.enable("unpack") def set_action_sequence(self): self.settings["action_sequence"]=["unpack","unpack_snapshot",\ diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py index 7edfc1f..acffb05 100644 --- a/catalyst/targets/netboot2.py +++ b/catalyst/targets/netboot2.py @@ -54,15 +54,14 @@ class netboot2(StageBase): self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\ self.settings["target_subpath"]+"/") if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"): + and self.resume.is_enabled("setup_target_path"): print "Resume point detected, skipping target path setup operation..." else: # first clean up any existing target stuff if os.path.isfile(self.settings["target_path"]): cmd("rm -f "+self.settings["target_path"], \ "Could not remove existing file: "+self.settings["target_path"],env=self.env) - touch(self.settings["autoresume_path"]+"setup_target_path") - + self.resume.enable("setup_target_path") ensure_dirs(self.settings["storedir"]+"/builds/") def copy_files_to_image(self): @@ -71,7 +70,7 @@ class netboot2(StageBase): # check for autoresume point if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"copy_files_to_image"): + and self.resume.is_enabled("copy_files_to_image"): print "Resume point detected, skipping target path setup operation..." else: if "netboot2/packages" in self.settings: @@ -101,11 +100,11 @@ class netboot2(StageBase): raise CatalystError("Failed to copy files to image!", print_traceback=True) - touch(self.settings["autoresume_path"]+"copy_files_to_image") + self.resume.enable("copy_files_to_image") def setup_overlay(self): if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_overlay"): + and self.resume.is_enabled("setup_overlay"): print "Resume point detected, skipping setup_overlay operation..." else: if "netboot2/overlay" in self.settings: @@ -113,7 +112,7 @@ class netboot2(StageBase): if os.path.exists(x): cmd("rsync -a "+x+"/ "+\ self.settings["chroot_path"] + self.settings["merge_path"], "netboot2/overlay: "+x+" copy failed.",env=self.env) - touch(self.settings["autoresume_path"]+"setup_overlay") + self.resume.enable("setup_overlay") def move_kernels(self): # we're done, move the kernels to builds/* @@ -130,7 +129,7 @@ class netboot2(StageBase): def remove(self): if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"remove"): + and self.resume.is_enabled("remove"): print "Resume point detected, skipping remove operation..." else: if self.settings["spec_prefix"]+"/rm" in self.settings: @@ -142,7 +141,7 @@ class netboot2(StageBase): def empty(self): if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"empty"): + and self.resume.is_enabled("empty"): print "Resume point detected, skipping empty operation..." else: if "netboot2/empty" in self.settings: @@ -161,7 +160,7 @@ class netboot2(StageBase): ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) - touch(self.settings["autoresume_path"]+"empty") + self.resume.enable("empty") def set_action_sequence(self): self.settings["action_sequence"]=["unpack","unpack_snapshot","config_profile_link", From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 45B5F138ADA for ; Thu, 26 Feb 2015 20:13:11 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 72F19E08DC; Thu, 26 Feb 2015 20:13:01 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id C765CE08DC for ; Thu, 26 Feb 2015 20:13:00 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 3E837340C4C for ; Thu, 26 Feb 2015 20:12:59 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 7873D129DF for ; Thu, 26 Feb 2015 20:12:43 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1420091887.a1c2785438593104d7c3d2ec20dd4fb25cb19884.dolsen@gentoo> Subject: [gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/targets/ X-VCS-Repository: proj/catalyst X-VCS-Files: catalyst/base/clearbase.py catalyst/base/resume.py catalyst/base/stagebase.py catalyst/targets/grp.py catalyst/targets/livecd_stage1.py catalyst/targets/livecd_stage2.py catalyst/targets/netboot2.py X-VCS-Directories: catalyst/base/ catalyst/targets/ X-VCS-Committer: dolsen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: a1c2785438593104d7c3d2ec20dd4fb25cb19884 X-VCS-Branch: master Date: Thu, 26 Feb 2015 20:12:43 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 656f0860-e660-4b0a-b3f9-4b343e9618fb X-Archives-Hash: ea9e4b9ef09007de4db07cb85160c222 Message-ID: <20150226201243._jzlscSHLZLJ-ksPb80Q6xnxK2EIFmYlLgQF3XOfOvk@z> commit: a1c2785438593104d7c3d2ec20dd4fb25cb19884 Author: Brian Dolbec gentoo org> AuthorDate: Sat Jun 1 06:23:46 2013 +0000 Commit: Brian Dolbec gentoo org> CommitDate: Thu Jan 1 05:58:07 2015 +0000 URL: http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=a1c27854 Create AutoResume class to handle all file creation, deletion... Create catalyst/base/resume.py. Migrate all auto_resume operations to using the new class. --- catalyst/base/clearbase.py | 21 +---- catalyst/base/resume.py | 166 ++++++++++++++++++++++++++++++++++++++ catalyst/base/stagebase.py | 156 +++++++++++++++-------------------- catalyst/targets/grp.py | 4 +- catalyst/targets/livecd_stage1.py | 4 +- catalyst/targets/livecd_stage2.py | 14 ++-- catalyst/targets/netboot2.py | 19 +++-- 7 files changed, 250 insertions(+), 134 deletions(-) diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py index 8462a3c..0ebe299 100644 --- a/catalyst/base/clearbase.py +++ b/catalyst/base/clearbase.py @@ -13,31 +13,14 @@ class ClearBase(object): """ def __init__(self, myspec): self.settings = myspec - + self.resume = None def clear_autoresume(self): """ Clean resume points since they are no longer needed """ if "autoresume" in self.settings["options"]: print "Removing AutoResume Points: ..." - myemp=self.settings["autoresume_path"] - if os.path.isdir(myemp): - if "autoresume" in self.settings["options"]: - print "Emptying directory",myemp - """ - stat the dir, delete the dir, recreate the dir and set - the proper perms and ownership - """ - mystat=os.stat(myemp) - if os.uname()[0] == "FreeBSD": - cmd("chflags -R noschg "+myemp,\ - "Could not remove immutable flag for file "\ - +myemp) - #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env) - shutil.rmtree(myemp) - ensure_dirs(myemp, 0755) - os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) - os.chmod(myemp,mystat[ST_MODE]) + self.resume.clear_all() def clear_chroot(self): diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py new file mode 100644 index 0000000..e42c7dc --- /dev/null +++ b/catalyst/base/resume.py @@ -0,0 +1,166 @@ +#!/usr/bin/python + +# Maintained in full by: +# Catalyst Team +# Release Engineering Team +# Copyright 2013 Brian Dolbec + +'''resume.py + +Performs autoresume tracking file operations such as +set, unset, is_set, is_unset, enabled, clear_all +''' + +import os +import shutil +from stat import ST_UID, ST_GID, ST_MODE +import traceback + +from catalyst.fileops import ensure_dirs, pjoin, listdir_files +from catalyst.support import touch + + +class AutoResumeError(Exception): + def __init__(self, message, print_traceback=False): + if message: + if print_traceback: + (type,value)=sys.exc_info()[:2] + if value!=None: + print + print "Traceback values found. listing..." + print traceback.print_exc(file=sys.stdout) + print "!!! catalyst: AutoResumeError " + message + print + + +class AutoResume(object): + '''Class for tracking and handling all aspects of + the autoresume option and related files. + ''' + + + def __init__(self, basedir, mode=0755): + self.basedir = basedir + ensure_dirs(basedir, mode=mode, fatal=True) + self._points = {} + self._init_points_() + + + def _init_points_(self): + '''Internal function which reads the autoresume directory and + for existing autoresume points and adds them to our _points variable + ''' + existing = listdir_files(self.basedir, False) + for point in existing: + self._points[point] = pjoin(self.basedir, point) + + + def enable(self, point, data=None): + '''Sets the resume point 'ON' + + @param point: string. name of the resume point to enable + @param data: string of information to store, or None + @return boolean + ''' + if point in self._points and not data: + return True + fname = pjoin(self.basedir, point) + if data: + with open(fname,"w") as myf: + myf.write(data) + else: + try: + touch(fname) + self._points[point] = fname + except Exception as e: + print AutoResumeError(str(e)) + return False + return True + + + def get(self, point, no_lf=True): + '''Gets any data stored inside a resume point + + @param point: string. name of the resume point to enable + @return data: string of information stored, or None + ''' + if point in self._points: + try: + with open(self._points[point], 'r') as myf: + data = myf.read() + if data and no_lf: + data = data.replace('\n', '') + except OSError as e: + print AutoResumeError(str(e)) + return None + return data + return None + + + def disable(self, point): + '''Sets the resume point 'OFF' + + @param point: string. name of the resume point to disable + @return boolean + ''' + if point not in self._points: + return True + try: + os.unlink(self._points[point]) + self._points.pop(point) + except Exception as e: + print AutoResumeError(str(e)) + return False + return True + + + def is_enabled(self, point): + '''Returns True if the resume point 'ON' + + @param point: string. name of the resume point enabled + @return boolean + ''' + return point in self._points + + + def is_disabled(self, point): + '''Returns True if the resume point 'OFF' + + @param point: string. name of the resume point not enabled + @return boolean + ''' + return point not in self._points + + + @property + def enabled(self): + '''Returns a list of enabled points + ''' + return list(self._points) + + + def clear_all(self): + '''Clear all active resume points + + @return boolean + ''' + try: + print "Emptying directory---", self.basedir + """ + stat the dir, delete the dir, recreate the dir and set + the proper perms and ownership + """ + mystat=os.stat(self.basedir) + if os.uname()[0] == "FreeBSD": + cmd("chflags -R noschg " + self.basedir,\ + "Could not remove immutable flag for file "\ + + self.basedir) + shutil.rmtree(self.basedir) + ensure_dirs(self.basedir, 0755) + os.chown(self.basedir,mystat[ST_UID],mystat[ST_GID]) + os.chmod(self.basedir,mystat[ST_MODE]) + self._points = {} + except Exception as e: + print AutoResumeError(str(e)) + return False + return True diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 40c7362..f3cb3e3 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -19,6 +19,7 @@ from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase from catalyst.lock import LockDir from catalyst.fileops import ensure_dirs, pjoin +from catalyst.base.resume import AutoResume class StageBase(TargetBase, ClearBase, GenBase): @@ -323,10 +324,8 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_target_path(self): self.settings["target_path"]=normpath(self.settings["storedir"]+\ "/builds/"+self.settings["target_subpath"]+".tar.bz2") - setup_target_path_resume = pjoin(self.settings["autoresume_path"], - "setup_target_path") - if "autoresume" in self.settings["options"] and \ - os.path.exists(setup_target_path_resume): + if "autoresume" in self.settings["options"]\ + and self.resume.is_enabled("setup_target_path"): print \ "Resume point detected, skipping target path setup operation..." else: @@ -338,7 +337,7 @@ class StageBase(TargetBase, ClearBase, GenBase): # cmd("rm -f "+self.settings["target_path"],\ # "Could not remove existing file: "\ # +self.settings["target_path"],env=self.env) - touch(setup_target_path_resume) + self.resume.enable("setup_target_path") ensure_dirs(self.settings["storedir"] + "/builds/") @@ -486,7 +485,7 @@ class StageBase(TargetBase, ClearBase, GenBase): )) if "autoresume" in self.settings["options"]: print "The autoresume path is " + self.settings["autoresume_path"] - ensure_dirs(self.settings["autoresume_path"], mode=0755) + self.resume = AutoResume(self.settings["autoresume_path"], mode=0755) def set_controller_file(self): self.settings["controller_file"]=normpath(self.settings["sharedir"]+\ @@ -667,8 +666,7 @@ class StageBase(TargetBase, ClearBase, GenBase): def unpack(self): unpack=True - unpack_resume = pjoin(self.settings["autoresume_path"], "unpack") - clst_unpack_hash=read_from_clst(unpack_resume) + clst_unpack_hash = self.resume.get("unpack") if "seedcache" in self.settings["options"]: if os.path.isdir(self.settings["source_path"]): @@ -714,7 +712,7 @@ class StageBase(TargetBase, ClearBase, GenBase): if "autoresume" in self.settings["options"]: if os.path.isdir(self.settings["source_path"]) \ - and os.path.exists(unpack_resume): + and self.resume.is_enabled("unpack"): """ Autoresume is valid, SEEDCACHE is valid """ unpack=False invalid_snapshot=False @@ -726,7 +724,7 @@ class StageBase(TargetBase, ClearBase, GenBase): invalid_snapshot=True elif os.path.isdir(self.settings["source_path"]) \ - and not os.path.exists(unpack_resume): + and self.resume.is_disabled("unpack"): """ Autoresume is invalid, SEEDCACHE """ unpack=True invalid_snapshot=False @@ -782,19 +780,16 @@ class StageBase(TargetBase, ClearBase, GenBase): cmd(unpack_cmd,error_msg,env=self.env) if "source_path_hash" in self.settings: - myf=open(unpack_resume,"w") - myf.write(self.settings["source_path_hash"]) - myf.close() + self.resume.enable("unpack", + data=self.settings["source_path_hash"]) else: - touch(unpack_resume) + self.resume.enable("unpack") else: print "Resume point detected, skipping unpack operation..." def unpack_snapshot(self): unpack=True - unpack_portage_resume = pjoin(self.settings["autoresume_path"], - "unpack_portage") - snapshot_hash=read_from_clst(unpack_portage_resume) + snapshot_hash = self.resume.get("unpack_portage") if "snapcache" in self.settings["options"]: snapshot_cache_hash=\ @@ -831,7 +826,7 @@ class StageBase(TargetBase, ClearBase, GenBase): if "autoresume" in self.settings["options"] \ and os.path.exists(self.settings["chroot_path"]+\ self.settings["portdir"]) \ - and os.path.exists(unpack_portage_resume) \ + and self.resume.is_enabled("unpack_portage") \ and self.settings["snapshot_path_hash"] == snapshot_hash: print \ "Valid Resume point detected, skipping unpack of portage tree..." @@ -856,18 +851,15 @@ class StageBase(TargetBase, ClearBase, GenBase): myf.close() else: print "Setting snapshot autoresume point" - myf=open(unpack_portage_resume,"w") - myf.write(self.settings["snapshot_path_hash"]) - myf.close() + self.resume.enable("unpack_portage", + data=self.settings["snapshot_path_hash"]) if "snapcache" in self.settings["options"]: self.snapshot_lock_object.unlock() def config_profile_link(self): - config_protect_link_resume = pjoin(self.settings["autoresume_path"], - "config_profile_link") if "autoresume" in self.settings["options"] \ - and os.path.exists(config_protect_link_resume): + and self.resume.is_enabled("config_profile_link"): print \ "Resume point detected, skipping config_profile_link operation..." else: @@ -881,21 +873,20 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["target_profile"]+" "+\ self.settings["chroot_path"]+"/etc/portage/make.profile",\ "Error creating profile link",env=self.env) - touch(config_protect_link_resume) + self.resume.enable("config_profile_link") def setup_confdir(self): - setup_confdir_resume = pjoin(self.settings["autoresume_path"], - "setup_confdir") if "autoresume" in self.settings["options"] \ - and os.path.exists(setup_confdir_resume): + and self.resume.is_enabled("setup_confdir"): print "Resume point detected, skipping setup_confdir operation..." else: if "portage_confdir" in self.settings: - print "Configuring /etc/portage..." - cmd("rsync -a "+self.settings["portage_confdir"]+"/ "+\ - self.settings["chroot_path"]+"/etc/portage/",\ - "Error copying /etc/portage",env=self.env) - touch(setup_confdir_resume) + print "Configuring %s..." % self.settings["port_conf"] + cmd("rsync -a " + self.settings["portage_confdir"] + "/ " + + self.settings["chroot_path"] + self.settings["port_conf"], + "Error copying %s" % self.settings["port_conf"], + env=self.env) + self.resume.enable("setup_confdir") def portage_overlay(self): """ We copy the contents of our overlays to /usr/local/portage """ @@ -1018,10 +1009,8 @@ class StageBase(TargetBase, ClearBase, GenBase): self.override_cflags() self.override_cxxflags() self.override_ldflags() - chroot_setup_resume = pjoin(self.settings["autoresume_path"], - "chroot_setup") if "autoresume" in self.settings["options"] \ - and os.path.exists(chroot_setup_resume): + and self.resume.is_enabled("chroot_setup"): print "Resume point detected, skipping chroot_setup operation..." else: print "Setting up chroot..." @@ -1118,35 +1107,32 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["make_conf"]) cmd("cp " + makepath + " " + makepath + ".catalyst",\ "Could not backup " + self.settings["make_conf"],env=self.env) - touch(self.settings["autoresume_path"]+"chroot_setup") + self.resume.enable("chroot_setup") def fsscript(self): - fsscript_resume = pjoin(self.settings["autoresume_path"], "fsscript") if "autoresume" in self.settings["options"] \ - and os.path.exists(fsscript_resume): + and self.resume.is_enabled("fsscript"): print "Resume point detected, skipping fsscript operation..." else: if "fsscript" in self.settings: if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+\ " fsscript","fsscript script failed.",env=self.env) - touch(fsscript_resume) + self.resume.enable("fsscript") def rcupdate(self): - rcupdate_resume = pjoin(self.settings["autoresume_path"], "rcupdate") if "autoresume" in self.settings["options"] \ - and os.path.exists(rcupdate_resume): + and self.resume.is_enabled("rcupdate"): print "Resume point detected, skipping rcupdate operation..." else: if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+" rc-update",\ "rc-update script failed.",env=self.env) - touch(rcupdate_resume) + self.resume.enable("rcupdate") def clean(self): - clean_resume = pjoin(self.settings["autoresume_path"], "clean") if "autoresume" in self.settings["options"] \ - and os.path.exists(clean_resume): + and self.resume.is_enabled("clean"): print "Resume point detected, skipping clean operation..." else: for x in self.settings["cleanables"]: @@ -1177,12 +1163,11 @@ class StageBase(TargetBase, ClearBase, GenBase): if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+" clean",\ "clean script failed.",env=self.env) - touch(clean_resume) + self.resume.enable("clean") def empty(self): - empty_resume = pjoin(self.settings["autoresume_path"], "empty") if "autoresume" in self.settings["options"] \ - and os.path.exists(empty_resume): + and self.resume.is_enabled("empty"): print "Resume point detected, skipping empty operation..." else: if self.settings["spec_prefix"]+"/empty" in self.settings: @@ -1206,12 +1191,11 @@ class StageBase(TargetBase, ClearBase, GenBase): ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) - touch(empty_resume) + self.resume.enable("empty") def remove(self): - remove_resume = pjoin(self.settings["autoresume_path"], "remove") if "autoresume" in self.settings["options"] \ - and os.path.exists(remove_resume): + and self.resume.is_enabled("remove"): print "Resume point detected, skipping remove operation..." else: if self.settings["spec_prefix"]+"/rm" in self.settings: @@ -1226,31 +1210,29 @@ class StageBase(TargetBase, ClearBase, GenBase): if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+\ " clean","Clean failed.",env=self.env) - touch(remove_resume) + self.resume.enable("remove") except: self.unbind() raise def preclean(self): - preclean_resume = pjoin(self.settings["autoresume_path"], "preclean") if "autoresume" in self.settings["options"] \ - and os.path.exists(preclean_resume): + and self.resume.is_enabled("preclean"): print "Resume point detected, skipping preclean operation..." else: try: if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+\ " preclean","preclean script failed.",env=self.env) - touch(preclean_resume) + self.resume.enable("preclean") except: self.unbind() raise CatalystError("Build failed, could not execute preclean") def capture(self): - capture_resume = pjoin(self.settings["autoresume_path"], "capture") if "autoresume" in self.settings["options"] \ - and os.path.exists(capture_resume): + and self.resume.is_enabled("capture"): print "Resume point detected, skipping capture operation..." else: """ Capture target in a tarball """ @@ -1270,12 +1252,11 @@ class StageBase(TargetBase, ClearBase, GenBase): self.gen_contents_file(self.settings["target_path"]) self.gen_digest_file(self.settings["target_path"]) - touch(capture_resume) + self.resume.enable("capture") def run_local(self): - run_local_resume = pjoin(self.settings["autoresume_path"], "run_local") if "autoresume" in self.settings["options"] \ - and os.path.exists(run_local_resume): + and self.resume.is_enabled("run_local"): print "Resume point detected, skipping run_local operation..." else: try: @@ -1283,7 +1264,9 @@ class StageBase(TargetBase, ClearBase, GenBase): print "run_local() starting controller script..." cmd(self.settings["controller_file"]+" run",\ "run script failed.",env=self.env) - touch(run_local_resume) + self.resume.enable("run_local") + else: + print "run_local() no controller_file found...", self.settings["controller_file"] except CatalystError: self.unbind() @@ -1359,9 +1342,8 @@ class StageBase(TargetBase, ClearBase, GenBase): self.chroot_lock.unlock() def unmerge(self): - unmerge_resume = pjoin(self.settings["autoresume_path"], "unmerge") if "autoresume" in self.settings["options"] \ - and os.path.exists(unmerge_resume): + and self.resume.is_enabled("unmerge"): print "Resume point detected, skipping unmerge operation..." else: if self.settings["spec_prefix"]+"/unmerge" in self.settings: @@ -1389,26 +1371,22 @@ class StageBase(TargetBase, ClearBase, GenBase): except CatalystError: self.unbind() raise - touch(unmerge_resume) + self.resume.enable("unmerge") def target_setup(self): - target_setup_resume = pjoin(self.settings["autoresume_path"], - "target_setup") if "autoresume" in self.settings["options"] \ - and os.path.exists(target_setup_resume): + and self.resume.is_enabled("target_setup"): print "Resume point detected, skipping target_setup operation..." else: print "Setting up filesystems per filesystem type" cmd(self.settings["controller_file"]+\ " target_image_setup "+ self.settings["target_path"],\ "target_image_setup script failed.",env=self.env) - touch(target_setup_resume) + self.resume.enable("target_setup") def setup_overlay(self): - setup_overlay_resume = pjoin(self.settings["autoresume_path"], - "setup_overlay") if "autoresume" in self.settings["options"] \ - and os.path.exists(setup_overlay_resume): + and self.resume.is_enabled("setup_overlay"): print "Resume point detected, skipping setup_overlay operation..." else: if self.settings["spec_prefix"]+"/overlay" in self.settings: @@ -1418,12 +1396,11 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["target_path"],\ self.settings["spec_prefix"]+"overlay: "+x+\ " copy failed.",env=self.env) - touch(setup_overlay_resume) + self.resume.enable("setup_overlay") def create_iso(self): - create_iso_resume = pjoin(self.settings["autoresume_path"], "create_iso") if "autoresume" in self.settings["options"] \ - and os.path.exists(create_iso_resume): + and self.resume.is_enabled("create_iso"): print "Resume point detected, skipping create_iso operation..." else: """ Create the ISO """ @@ -1433,7 +1410,7 @@ class StageBase(TargetBase, ClearBase, GenBase): env=self.env) self.gen_contents_file(self.settings["iso"]) self.gen_digest_file(self.settings["iso"]) - touch(create_iso_resume) + self.resume.enable("create_iso") else: print "WARNING: livecd/iso was not defined." print "An ISO Image will not be created." @@ -1442,13 +1419,12 @@ class StageBase(TargetBase, ClearBase, GenBase): build_packages_resume = pjoin(self.settings["autoresume_path"], "build_packages") if "autoresume" in self.settings["options"] \ - and os.path.exists(build_packages_resume): + and self.resume.is_enabled("build_packages"): print "Resume point detected, skipping build_packages operation..." else: if self.settings["spec_prefix"]+"/packages" in self.settings: if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+\ - "build_packages"): + and self.resume.is_enabled("build_packages"): print "Resume point detected, skipping build_packages operation..." else: mypack=\ @@ -1459,6 +1435,7 @@ class StageBase(TargetBase, ClearBase, GenBase): " build_packages "+mypack,\ "Error in attempt to build packages",env=self.env) touch(build_packages_resume) + self.resume.enable("build_packages") except CatalystError: self.unbind() raise CatalystError(self.settings["spec_prefix"]+\ @@ -1466,10 +1443,8 @@ class StageBase(TargetBase, ClearBase, GenBase): def build_kernel(self): '''Build all configured kernels''' - build_kernel_resume = pjoin(self.settings["autoresume_path"], - "build_kernel") if "autoresume" in self.settings["options"] \ - and os.path.exists(build_kernel_resume): + and self.resume.is_enabled("build_kernel"): print "Resume point detected, skipping build_kernel operation..." else: if "boot/kernel" in self.settings: @@ -1485,7 +1460,7 @@ class StageBase(TargetBase, ClearBase, GenBase): env=self.env) for kname in mynames: self._build_kernel(kname=kname) - touch(build_kernel_resume) + self.resume.enable("build_kernel") except CatalystError: self.unbind() raise CatalystError( @@ -1494,10 +1469,8 @@ class StageBase(TargetBase, ClearBase, GenBase): def _build_kernel(self, kname): "Build a single configured kernel by name" - kname_resume = pjoin(self.settings["autoresume_path"], - "build_kernel_" + kname) if "autoresume" in self.settings["options"] \ - and os.path.exists(kname_resume): + and self.resume.is_enabled("build_kernel_"+kname): print "Resume point detected, skipping build_kernel for "+kname+" operation..." return self._copy_kernel_config(kname=kname) @@ -1539,7 +1512,7 @@ class StageBase(TargetBase, ClearBase, GenBase): cmd("rm -R "+self.settings["chroot_path"]+\ "/tmp/initramfs_overlay/",env=self.env) - touch(kname_resume) + self.resume.is_enabled("build_kernel_"+kname) """ Execute the script that cleans up the kernel build @@ -1592,31 +1565,28 @@ class StageBase(TargetBase, ClearBase, GenBase): "/initramfs_overlay"],env=self.env) def bootloader(self): - bootloader_resume = pjoin(self.settings["autoresume_path"], "bootloader") if "autoresume" in self.settings["options"] \ - and os.path.exists(bootloader_resume): + and self.resume.is_enabled("bootloader"): print "Resume point detected, skipping bootloader operation..." else: try: cmd(self.settings["controller_file"]+\ " bootloader " + self.settings["target_path"],\ "Bootloader script failed.",env=self.env) - touch(bootloader_resume) + self.resume.enable("bootloader") except CatalystError: self.unbind() raise CatalystError("Script aborting due to error.") def livecd_update(self): - livecd_update_resume = pjoin(self.settings["autoresume_path"], - "livecd_update") if "autoresume" in self.settings["options"] \ - and os.path.exists(livecd_update_resume): + and self.resume.is_enabled("livecd_update"): print "Resume point detected, skipping build_packages operation..." else: try: cmd(self.settings["controller_file"]+\ " livecd-update","livecd-update failed.",env=self.env) - touch(livecd_update_resume) + self.resume.enable("livecd_update") except CatalystError: self.unbind() diff --git a/catalyst/targets/grp.py b/catalyst/targets/grp.py index a17f9bd..14cffe0 100644 --- a/catalyst/targets/grp.py +++ b/catalyst/targets/grp.py @@ -44,7 +44,7 @@ class grp(StageBase): def set_target_path(self): self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]+"/") if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"): + and self.resume.is_enabled("setup_target_path"): print "Resume point detected, skipping target path setup operation..." else: # first clean up any existing target stuff @@ -53,7 +53,7 @@ class grp(StageBase): #"Could not remove existing directory: "+self.settings["target_path"],env=self.env) ensure_dirs(self.settings["target_path"]) - touch(self.settings["autoresume_path"]+"setup_target_path") + self.resume.enable("setup_target_path") def run_local(self): for pkgset in self.settings["grp"]: diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index 668960b..55859bc 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -35,14 +35,14 @@ class livecd_stage1(StageBase): def set_target_path(self): self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]) if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"): + and self.resume.is_enabled("setup_target_path"): print "Resume point detected, skipping target path setup operation..." else: # first clean up any existing target stuff if os.path.exists(self.settings["target_path"]): cmd("rm -rf "+self.settings["target_path"],\ "Could not remove existing directory: "+self.settings["target_path"],env=self.env) - touch(self.settings["autoresume_path"]+"setup_target_path") + self.resume.enable("setup_target_path") ensure_dirs(self.settings["target_path"]) diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index 94da670..973a734 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -55,14 +55,14 @@ class livecd_stage2(StageBase): def set_target_path(self): self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]+"/") if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"): + and self.resume.is_enabled("setup_target_path"): print "Resume point detected, skipping target path setup operation..." else: # first clean up any existing target stuff if os.path.isdir(self.settings["target_path"]): cmd("rm -rf "+self.settings["target_path"], "Could not remove existing directory: "+self.settings["target_path"],env=self.env) - touch(self.settings["autoresume_path"]+"setup_target_path") + self.resume.enable("setup_target_path") ensure_dirs(self.settings["target_path"]) def run_local(self): @@ -89,7 +89,7 @@ class livecd_stage2(StageBase): unpack=True display_msg=None - clst_unpack_hash=read_from_clst(self.settings["autoresume_path"]+"unpack") + clst_unpack_hash = self.resume.get("unpack") if os.path.isdir(self.settings["source_path"]): unpack_cmd="rsync -a --delete "+self.settings["source_path"]+" "+self.settings["chroot_path"] @@ -100,7 +100,7 @@ class livecd_stage2(StageBase): if "autoresume" in self.settings["options"]: if os.path.isdir(self.settings["source_path"]) and \ - os.path.exists(self.settings["autoresume_path"]+"unpack"): + self.resume.is_enabled("unpack"): print "Resume point detected, skipping unpack operation..." unpack=False elif "source_path_hash" in self.settings: @@ -131,11 +131,9 @@ class livecd_stage2(StageBase): cmd(unpack_cmd,error_msg,env=self.env) if "source_path_hash" in self.settings: - myf=open(self.settings["autoresume_path"]+"unpack","w") - myf.write(self.settings["source_path_hash"]) - myf.close() + self.resume.enable("unpack", data=self.settings["source_path_hash"]) else: - touch(self.settings["autoresume_path"]+"unpack") + self.resume.enable("unpack") def set_action_sequence(self): self.settings["action_sequence"]=["unpack","unpack_snapshot",\ diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py index 7edfc1f..acffb05 100644 --- a/catalyst/targets/netboot2.py +++ b/catalyst/targets/netboot2.py @@ -54,15 +54,14 @@ class netboot2(StageBase): self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\ self.settings["target_subpath"]+"/") if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"): + and self.resume.is_enabled("setup_target_path"): print "Resume point detected, skipping target path setup operation..." else: # first clean up any existing target stuff if os.path.isfile(self.settings["target_path"]): cmd("rm -f "+self.settings["target_path"], \ "Could not remove existing file: "+self.settings["target_path"],env=self.env) - touch(self.settings["autoresume_path"]+"setup_target_path") - + self.resume.enable("setup_target_path") ensure_dirs(self.settings["storedir"]+"/builds/") def copy_files_to_image(self): @@ -71,7 +70,7 @@ class netboot2(StageBase): # check for autoresume point if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"copy_files_to_image"): + and self.resume.is_enabled("copy_files_to_image"): print "Resume point detected, skipping target path setup operation..." else: if "netboot2/packages" in self.settings: @@ -101,11 +100,11 @@ class netboot2(StageBase): raise CatalystError("Failed to copy files to image!", print_traceback=True) - touch(self.settings["autoresume_path"]+"copy_files_to_image") + self.resume.enable("copy_files_to_image") def setup_overlay(self): if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"setup_overlay"): + and self.resume.is_enabled("setup_overlay"): print "Resume point detected, skipping setup_overlay operation..." else: if "netboot2/overlay" in self.settings: @@ -113,7 +112,7 @@ class netboot2(StageBase): if os.path.exists(x): cmd("rsync -a "+x+"/ "+\ self.settings["chroot_path"] + self.settings["merge_path"], "netboot2/overlay: "+x+" copy failed.",env=self.env) - touch(self.settings["autoresume_path"]+"setup_overlay") + self.resume.enable("setup_overlay") def move_kernels(self): # we're done, move the kernels to builds/* @@ -130,7 +129,7 @@ class netboot2(StageBase): def remove(self): if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"remove"): + and self.resume.is_enabled("remove"): print "Resume point detected, skipping remove operation..." else: if self.settings["spec_prefix"]+"/rm" in self.settings: @@ -142,7 +141,7 @@ class netboot2(StageBase): def empty(self): if "autoresume" in self.settings["options"] \ - and os.path.exists(self.settings["autoresume_path"]+"empty"): + and self.resume.is_enabled("empty"): print "Resume point detected, skipping empty operation..." else: if "netboot2/empty" in self.settings: @@ -161,7 +160,7 @@ class netboot2(StageBase): ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) - touch(self.settings["autoresume_path"]+"empty") + self.resume.enable("empty") def set_action_sequence(self): self.settings["action_sequence"]=["unpack","unpack_snapshot","config_profile_link",