From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id ABFF5139B2A for ; Tue, 8 Sep 2015 14:14:50 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 2B58C14229; Tue, 8 Sep 2015 14:14:49 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 43144141F9 for ; Tue, 8 Sep 2015 14:14:48 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id CE8F53407B8 for ; Tue, 8 Sep 2015 14:14:46 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 8171918F for ; Tue, 8 Sep 2015 14:14:45 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1441721508.7536857c1a14d2eec224e80ba96ab9f3091e0d1c.dolsen@gentoo> Subject: [gentoo-commits] proj/catalyst:pending commit in: catalyst/base/, catalyst/ X-VCS-Repository: proj/catalyst X-VCS-Files: catalyst/base/stagebase.py catalyst/support.py X-VCS-Directories: catalyst/ catalyst/base/ X-VCS-Committer: dolsen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: 7536857c1a14d2eec224e80ba96ab9f3091e0d1c X-VCS-Branch: pending Date: Tue, 8 Sep 2015 14:14:45 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: c3e15231-536d-456c-b340-d8b988042d9e X-Archives-Hash: 39877b9324c70734d605aeaebeec8dd1 commit: 7536857c1a14d2eec224e80ba96ab9f3091e0d1c Author: Brian Dolbec gentoo org> AuthorDate: Tue Sep 8 00:35:41 2015 +0000 Commit: Brian Dolbec gentoo org> CommitDate: Tue Sep 8 14:11:48 2015 +0000 URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=7536857c Make the new compress code handling compatible with existing specs Clean out old snapshot setting code. Make file_check() not traceback after printing it's error message. Fix source auto extension handling in unpack() catalyst/base/stagebase.py | 53 ++++++++++++++++++++-------------------------- catalyst/support.py | 21 ++++++++++++++++++ 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index bd6938c..31fbe8b 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -15,7 +15,8 @@ from DeComp.compress import CompressMap from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN) from catalyst.support import (CatalystError, msg, file_locate, normpath, - touch, cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount) + touch, cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount, + file_check) from catalyst.base.targetbase import TargetBase from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase @@ -193,7 +194,7 @@ class StageBase(TargetBase, ClearBase, GenBase): on disk. """ #pdb.set_trace() - file_locate(self.settings,["source_path","snapshot_path","distdir"],\ + file_locate(self.settings,["distdir"],\ expand=0) """ If we are using portage_confdir, check that as well. """ if "portage_confdir" in self.settings: @@ -430,8 +431,10 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["source_path"] = normpath(self.settings["storedir"] + "/tmp/" + self.settings["source_subpath"] + "/") else: - self.settings["source_path"]=normpath(self.settings["storedir"]+\ - "/builds/"+self.settings["source_subpath"]) + self.settings["source_path"] = file_check( + normpath(self.settings["storedir"] + "/builds/" + + self.settings["source_subpath"]) + ) if os.path.isfile(self.settings["source_path"]): # XXX: Is this even necessary if the previous check passes? if os.path.exists(self.settings["source_path"]): @@ -460,27 +463,15 @@ class StageBase(TargetBase, ClearBase, GenBase): "/root/*", self.settings["portdir"]] def set_snapshot_path(self): - self.settings["snapshot_path"]=normpath(self.settings["storedir"]+\ + self.settings["snapshot_path"]= file_check(normpath(self.settings["storedir"]+\ "/snapshots/" + self.settings["snapshot_name"] + - self.settings["snapshot"].rstrip('/')+".tar.xz") - - if os.path.exists(self.settings["snapshot_path"]): - self.settings["snapshot_path_hash"] = \ - self.settings["hash_map"].generate_hash( - self.settings["snapshot_path"], - hash_ = self.settings["hash_function"], - verbose = False) - else: - self.settings["snapshot_path"]=normpath(self.settings["storedir"]+\ - "/snapshots/" + self.settings["snapshot_name"] + - self.settings["snapshot"]) - - if os.path.exists(self.settings["snapshot_path"]): - self.settings["snapshot_path_hash"] = \ - self.settings["hash_map"].generate_hash( - self.settings["snapshot_path"], - hash_ = self.settings["hash_function"], - verbose = False) + self.settings["snapshot"])) + print "*** SNAPSHOT_PATH set to:", self.settings["snapshot_path"] + self.settings["snapshot_path_hash"] = \ + self.settings["hash_map"].generate_hash( + self.settings["snapshot_path"], + hash_ = self.settings["hash_function"], + verbose = False) def set_snapcache_path(self): self.settings["snapshot_cache_path"]=\ @@ -723,12 +714,10 @@ class StageBase(TargetBase, ClearBase, GenBase): else: """ SEEDCACHE is a not a directory, try untar'ing """ print "Referenced SEEDCACHE does not appear to be a directory, trying to untar..." - unpack_info['mode'] = self.decompressor.determine_mode( - self.settings["source_path"]) + unpack_info['source'] = file_check(unpack_info['source']) else: """ No SEEDCACHE, use tar """ - unpack_info['mode'] = self.decompressor.determine_mode( - unpack_info["source"]) + unpack_info['source'] = file_check(unpack_info['source']) # endif "seedcache" if "autoresume" in self.settings["options"]: @@ -742,19 +731,23 @@ class StageBase(TargetBase, ClearBase, GenBase): and self.settings["source_path_hash"]==clst_unpack_hash: """ Autoresume is valid, tarball is valid """ _unpack=False - invalid_snapshot=True + invalid_snapshot=False elif os.path.isdir(self.settings["source_path"]) \ and self.resume.is_disabled("unpack"): """ Autoresume is invalid, SEEDCACHE """ _unpack=True - invalid_snapshot=False + invalid_snapshot=True + # check and reset the unpack_info['source'] + unpack_info['source'] = file_check(unpack_info['source']) elif os.path.isfile(self.settings["source_path"]) \ and self.settings["source_path_hash"]!=clst_unpack_hash: """ Autoresume is invalid, tarball """ _unpack=True invalid_snapshot=True + unpack_info['source'] = file_check(unpack_info['source']) + else: """ No autoresume, SEEDCACHE """ if "seedcache" in self.settings["options"]: diff --git a/catalyst/support.py b/catalyst/support.py index 2ac4816..b6705c9 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -1,4 +1,5 @@ +import glob import sys import string import os @@ -144,6 +145,25 @@ def cmd(mycmd, myexc="", env={}, debug=False, fail_func=None): print_traceback=False) +def file_check(filepath): + '''Check for the files existence and that only one exists + if others are found with various extensions + ''' + if os.path.exists(filepath): + return filepath + # it didn't exist + # so check if there are files of that name with an extension + files = glob.glob("%s.*" % filepath) + # remove any false positive files + files = [x for x in files if not x.endswith(".CONTENTS") and not x.endswith(".DIGESTS")] + if len(files) is 1: + return files[0] + elif len(files) > 1: + msg = "Ambiguos Filename: %s\nPlease specify the correct extension as well" % filepath + raise CatalystError(msg, print_traceback=False) + raise CatalystError("File Not Found: %s" % filepath) + + def file_locate(settings,filelist,expand=1): #if expand=1, non-absolute paths will be accepted and # expanded to os.getcwd()+"/"+localpath if file exists @@ -186,6 +206,7 @@ defined are not preserved. In other words, "foo", "bar", "oni" ordering is prese """, print_traceback=True) + def parse_makeconf(mylines): mymakeconf={} pos=0 From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 74307139B2A for ; Tue, 8 Sep 2015 14:17:19 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id CA3E114229; Tue, 8 Sep 2015 14:17:17 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 0BA7A141F9 for ; Tue, 8 Sep 2015 14:17:17 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 23175340775 for ; Tue, 8 Sep 2015 14:17:16 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 1352818E for ; Tue, 8 Sep 2015 14:17:14 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1441721508.7536857c1a14d2eec224e80ba96ab9f3091e0d1c.dolsen@gentoo> Subject: [gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/ X-VCS-Repository: proj/catalyst X-VCS-Files: catalyst/base/stagebase.py catalyst/support.py X-VCS-Directories: catalyst/base/ catalyst/ X-VCS-Committer: dolsen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: 7536857c1a14d2eec224e80ba96ab9f3091e0d1c X-VCS-Branch: master Date: Tue, 8 Sep 2015 14:17:14 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 6248b2ff-964e-4985-9518-a14abc3a6526 X-Archives-Hash: 055ee1f18cfda96e3a4a94470c1f03f5 Message-ID: <20150908141714.9G5NM9coaOzio9y6fOhnLHL5TF4KmONlOHRaJgHMw7M@z> commit: 7536857c1a14d2eec224e80ba96ab9f3091e0d1c Author: Brian Dolbec gentoo org> AuthorDate: Tue Sep 8 00:35:41 2015 +0000 Commit: Brian Dolbec gentoo org> CommitDate: Tue Sep 8 14:11:48 2015 +0000 URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=7536857c Make the new compress code handling compatible with existing specs Clean out old snapshot setting code. Make file_check() not traceback after printing it's error message. Fix source auto extension handling in unpack() catalyst/base/stagebase.py | 53 ++++++++++++++++++++-------------------------- catalyst/support.py | 21 ++++++++++++++++++ 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index bd6938c..31fbe8b 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -15,7 +15,8 @@ from DeComp.compress import CompressMap from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN) from catalyst.support import (CatalystError, msg, file_locate, normpath, - touch, cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount) + touch, cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount, + file_check) from catalyst.base.targetbase import TargetBase from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase @@ -193,7 +194,7 @@ class StageBase(TargetBase, ClearBase, GenBase): on disk. """ #pdb.set_trace() - file_locate(self.settings,["source_path","snapshot_path","distdir"],\ + file_locate(self.settings,["distdir"],\ expand=0) """ If we are using portage_confdir, check that as well. """ if "portage_confdir" in self.settings: @@ -430,8 +431,10 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["source_path"] = normpath(self.settings["storedir"] + "/tmp/" + self.settings["source_subpath"] + "/") else: - self.settings["source_path"]=normpath(self.settings["storedir"]+\ - "/builds/"+self.settings["source_subpath"]) + self.settings["source_path"] = file_check( + normpath(self.settings["storedir"] + "/builds/" + + self.settings["source_subpath"]) + ) if os.path.isfile(self.settings["source_path"]): # XXX: Is this even necessary if the previous check passes? if os.path.exists(self.settings["source_path"]): @@ -460,27 +463,15 @@ class StageBase(TargetBase, ClearBase, GenBase): "/root/*", self.settings["portdir"]] def set_snapshot_path(self): - self.settings["snapshot_path"]=normpath(self.settings["storedir"]+\ + self.settings["snapshot_path"]= file_check(normpath(self.settings["storedir"]+\ "/snapshots/" + self.settings["snapshot_name"] + - self.settings["snapshot"].rstrip('/')+".tar.xz") - - if os.path.exists(self.settings["snapshot_path"]): - self.settings["snapshot_path_hash"] = \ - self.settings["hash_map"].generate_hash( - self.settings["snapshot_path"], - hash_ = self.settings["hash_function"], - verbose = False) - else: - self.settings["snapshot_path"]=normpath(self.settings["storedir"]+\ - "/snapshots/" + self.settings["snapshot_name"] + - self.settings["snapshot"]) - - if os.path.exists(self.settings["snapshot_path"]): - self.settings["snapshot_path_hash"] = \ - self.settings["hash_map"].generate_hash( - self.settings["snapshot_path"], - hash_ = self.settings["hash_function"], - verbose = False) + self.settings["snapshot"])) + print "*** SNAPSHOT_PATH set to:", self.settings["snapshot_path"] + self.settings["snapshot_path_hash"] = \ + self.settings["hash_map"].generate_hash( + self.settings["snapshot_path"], + hash_ = self.settings["hash_function"], + verbose = False) def set_snapcache_path(self): self.settings["snapshot_cache_path"]=\ @@ -723,12 +714,10 @@ class StageBase(TargetBase, ClearBase, GenBase): else: """ SEEDCACHE is a not a directory, try untar'ing """ print "Referenced SEEDCACHE does not appear to be a directory, trying to untar..." - unpack_info['mode'] = self.decompressor.determine_mode( - self.settings["source_path"]) + unpack_info['source'] = file_check(unpack_info['source']) else: """ No SEEDCACHE, use tar """ - unpack_info['mode'] = self.decompressor.determine_mode( - unpack_info["source"]) + unpack_info['source'] = file_check(unpack_info['source']) # endif "seedcache" if "autoresume" in self.settings["options"]: @@ -742,19 +731,23 @@ class StageBase(TargetBase, ClearBase, GenBase): and self.settings["source_path_hash"]==clst_unpack_hash: """ Autoresume is valid, tarball is valid """ _unpack=False - invalid_snapshot=True + invalid_snapshot=False elif os.path.isdir(self.settings["source_path"]) \ and self.resume.is_disabled("unpack"): """ Autoresume is invalid, SEEDCACHE """ _unpack=True - invalid_snapshot=False + invalid_snapshot=True + # check and reset the unpack_info['source'] + unpack_info['source'] = file_check(unpack_info['source']) elif os.path.isfile(self.settings["source_path"]) \ and self.settings["source_path_hash"]!=clst_unpack_hash: """ Autoresume is invalid, tarball """ _unpack=True invalid_snapshot=True + unpack_info['source'] = file_check(unpack_info['source']) + else: """ No autoresume, SEEDCACHE """ if "seedcache" in self.settings["options"]: diff --git a/catalyst/support.py b/catalyst/support.py index 2ac4816..b6705c9 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -1,4 +1,5 @@ +import glob import sys import string import os @@ -144,6 +145,25 @@ def cmd(mycmd, myexc="", env={}, debug=False, fail_func=None): print_traceback=False) +def file_check(filepath): + '''Check for the files existence and that only one exists + if others are found with various extensions + ''' + if os.path.exists(filepath): + return filepath + # it didn't exist + # so check if there are files of that name with an extension + files = glob.glob("%s.*" % filepath) + # remove any false positive files + files = [x for x in files if not x.endswith(".CONTENTS") and not x.endswith(".DIGESTS")] + if len(files) is 1: + return files[0] + elif len(files) > 1: + msg = "Ambiguos Filename: %s\nPlease specify the correct extension as well" % filepath + raise CatalystError(msg, print_traceback=False) + raise CatalystError("File Not Found: %s" % filepath) + + def file_locate(settings,filelist,expand=1): #if expand=1, non-absolute paths will be accepted and # expanded to os.getcwd()+"/"+localpath if file exists @@ -186,6 +206,7 @@ defined are not preserved. In other words, "foo", "bar", "oni" ordering is prese """, print_traceback=True) + def parse_makeconf(mylines): mymakeconf={} pos=0