* [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, etc/, catalyst/
@ 2020-05-19 23:39 Matt Turner
0 siblings, 0 replies; 3+ messages in thread
From: Matt Turner @ 2020-05-19 23:39 UTC (permalink / raw
To: gentoo-commits
commit: ec3cddef28cf7cf9ca3d6719e743dbc43e8d9626
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat May 16 03:18:21 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Tue May 19 23:39:36 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ec3cddef
catalyst: Convert catalyst.conf to TOML
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 4 +--
catalyst/main.py | 47 +++++++------------------
etc/catalyst.conf | 88 +++++++++++++++++++++++++++-------------------
3 files changed, 67 insertions(+), 72 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 537ab752..5c82a46d 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -857,8 +857,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
if 'var_tmpfs_portage' not in self.settings:
continue
- mount += ['-t', 'tmpfs', '-o', 'size=' +
- self.settings['var_tmpfs_portage'] + 'G']
+ mount += ['-t', 'tmpfs', '-o',
+ f"size={self.settings['var_tmpfs_portage']}G"]
elif source == 'tmpfs':
mount += ['-t', 'tmpfs']
elif source == 'shm':
diff --git a/catalyst/main.py b/catalyst/main.py
index be06ccd7..159fe454 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -5,6 +5,8 @@ import os
import sys
import textwrap
+import toml
+
from snakeoil.process import namespaces
from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
@@ -17,8 +19,7 @@ from catalyst.defaults import confdefaults, option_messages, DEFAULT_CONFIG_FILE
from catalyst.support import CatalystError
from catalyst.version import get_version
-
-conf_values = {}
+conf_values = confdefaults
def version():
@@ -30,42 +31,20 @@ def version():
def parse_config(config_files):
- # search a couple of different areas for the main config file
- myconf = {}
-
- # try and parse the config file "config_file"
for config_file in config_files:
log.notice('Loading configuration file: %s', config_file)
try:
- config = catalyst.config.ConfigParser(config_file)
- myconf.update(config.get_values())
+ conf_values.update(toml.load(config_file))
except Exception as e:
log.critical('Could not find parse configuration file: %s: %s',
config_file, e)
- # now, load up the values into conf_values so that we can use them
- for x in list(confdefaults):
- if x in myconf:
- if x == 'options':
- conf_values[x] = set(myconf[x].split())
- elif x in ["decompressor_search_order"]:
- conf_values[x] = myconf[x].split()
- else:
- conf_values[x] = myconf[x]
- else:
- conf_values[x] = confdefaults[x]
-
# print out any options messages
for opt in conf_values['options']:
if opt in option_messages:
log.info(option_messages[opt])
- for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir",
- "local_overlay", "repos"]:
- if key in myconf:
- conf_values[key] = myconf[key]
-
- if "envscript" in myconf:
+ if "envscript" in conf_values:
log.info('Envscript support enabled.')
# take care of any variable substitutions that may be left
@@ -297,17 +276,17 @@ def _main(parser, opts):
conf_values['DEBUG'] = opts.debug
conf_values['VERBOSE'] = opts.debug or opts.verbose
- options = set()
+ options = []
if opts.fetchonly:
- options.add('fetch')
+ options.append('fetch')
if opts.purge:
- options.add('purge')
+ options.append('purge')
if opts.purgeonly:
- options.add('purgeonly')
+ options.append('purgeonly')
if opts.purgetmponly:
- options.add('purgetmponly')
+ options.append('purgetmponly')
if opts.clear_autoresume:
- options.add('clear-autoresume')
+ options.append('clear-autoresume')
# Make sure we have some work before moving further.
if not myspecfile and not mycmdline:
@@ -318,7 +297,7 @@ def _main(parser, opts):
# import configuration file and import our main module using those settings
parse_config(myconfigs)
- conf_values["options"].update(options)
+ conf_values["options"].extend(options)
log.notice('conf_values[options] = %s', conf_values['options'])
# initialize our contents generator
@@ -335,7 +314,7 @@ def _main(parser, opts):
if "digests" in conf_values:
valid_digests = hashlib.algorithms_available
- digests = set(conf_values['digests'].split())
+ digests = set(conf_values['digests'])
conf_values['digests'] = digests
# First validate all the requested digests are valid keys.
diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index d33be15f..2272cb86 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -10,53 +10,69 @@
#
# $ python3 -c 'import hashlib; print(hashlib.algorithms_available)'
#
-digests="blake2b sha512"
+digests = ["blake2b", "sha512"]
# envscript allows users to set options such as http proxies, MAKEOPTS,
# GENTOO_MIRRORS, or any other environment variables needed for building.
# The envscript file sets environment variables like so:
# export FOO="bar"
-envscript="/etc/catalyst/catalystrc"
-
-# options set different build-time options for catalyst. Some examples are:
-# autoresume = Attempt to resume a failed build, clear the autoresume flags with
-# the -a option to the catalyst cmdline. -p will clear the autoresume flags
-# as well as your pkgcache and kerncache
-# ( This option is not fully tested, bug reports welcome )
-# bindist = enables the bindist USE flag, please see package specific definition,
-# however, it is suggested to enable this if redistributing builds.
-# This optional USE flag is normally cleaned from the make.conf file on
-# completion of the stage. For a non-cleaned version,
-# use sticky-config also (see below)
-# ccache = enables build time ccache support
-# distcc = enable distcc support for building. You have to set distcc_hosts in
-# your spec file.
-# icecream = enables icecream compiler cluster support for building
-# keepwork = Prevents the removal of the working chroot path and any autoresume
-# files or points.
-# kerncache = keeps a tbz2 of your built kernel and modules (useful if your
-# build stops in livecd-stage2)
-# pkgcache = keeps a tbz2 of every built package (useful if your build stops
-# prematurely)
-# seedcache = use the build output of a previous target if it exists to speed up
-# the copy
-# sticky-config = enables the code that will keep any internal 'catalyst_use' flags
-# added to the USE= for building the stage. These are usually added for legal
-# or specific needs in building the the early stage. Mostly it is the
-# 'bindist' USE flag option that is used for legal reasons, please see its
-# specific definition. It will also keep any /etc/portage/package.*
-# files or directories.
-#
-# (These options can be used together)
-options="autoresume bindist kerncache pkgcache seedcache"
+envscript = "/etc/catalyst/catalystrc"
+
+# options set different build-time options for catalyst.
+options = [
+ # Attempt to resume a failed build, clear the autoresume flags with the
+ # -a option to the catalyst cmdline. -p will clear the autoresume
+ # flags as well as your pkgcache and kerncache
+ "autoresume",
+
+ # Enables the bindist USE flag, please see package specific definition,
+ # however, it is suggested to enable this if redistributing builds.
+ # This optional USE flag is normally cleaned from the make.conf file on
+ # completion of the stage. For a non-cleaned version, use
+ # sticky-config also (see below)
+ "bindist",
+
+ # Enable FEATURES=ccache
+ # "ccache",
+
+ # Enable FEATURES=distcc. You have to set distcc_hosts in your spec
+ # file.
+ # "distcc",
+
+ # Enable FEATURES=icecream
+ # "icecream",
+
+ # Prevents the removal of the working chroot path and any autoresume
+ # files or points.
+ # "keepwork",
+
+ # keeps a tbz2 of your built kernel and modules (useful if your
+ # build stops in livecd-stage2)
+ "kerncache",
+
+ # Build and use binary packages
+ "pkgcache",
+
+ # Use the build output of a previous target if it exists rather than
+ # the tarball
+ "seedcache",
+
+ # enables the code that will keep any internal 'catalyst_use' flags
+ # added to the USE= for building the stage. These are usually added
+ # for legal or specific needs in building the the early stage. Mostly
+ # it is the 'bindist' USE flag option that is used for legal reasons,
+ # please see its specific definition. It will also keep any
+ # /etc/portage/package.* files or directories.
+ # "sticky-config",
+]
# port_logdir is where all build logs will be kept. This dir will be automatically cleaned
# of all logs over 30 days old. If left undefined the logs will remain in the build directory
# as usual and get cleaned every time a stage build is restarted.
-# port_logdir="/var/tmp/catalyst/tmp"
+# port_logdir = "/var/tmp/catalyst/tmp"
# var_tmpfs_portage will mount a tmpfs for /var/tmp/portage so building takes place in RAM
# this feature requires a pretty large tmpfs ({open,libre}office needs ~8GB to build)
# WARNING: If you use too much RAM everything will fail horribly and it is not our fault.
# set size of /var/tmp/portage tmpfs in gigabytes
-# var_tmpfs_portage=16
+# var_tmpfs_portage = 16
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, etc/, catalyst/
@ 2020-05-20 0:26 Matt Turner
0 siblings, 0 replies; 3+ messages in thread
From: Matt Turner @ 2020-05-20 0:26 UTC (permalink / raw
To: gentoo-commits
commit: da7e5a151eb2dfc6a1f4a0afd755d93582789ce6
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat May 16 03:18:21 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 00:24:54 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=da7e5a15
catalyst: Convert catalyst.conf to TOML
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 4 +--
catalyst/main.py | 47 +++++++------------------
etc/catalyst.conf | 88 +++++++++++++++++++++++++++-------------------
3 files changed, 67 insertions(+), 72 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index febaf969..9410f151 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -857,8 +857,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
if 'var_tmpfs_portage' not in self.settings:
continue
- mount += ['-t', 'tmpfs', '-o', 'size=' +
- self.settings['var_tmpfs_portage'] + 'G']
+ mount += ['-t', 'tmpfs', '-o',
+ f"size={self.settings['var_tmpfs_portage']}G"]
elif source == 'tmpfs':
mount += ['-t', 'tmpfs']
elif source == 'shm':
diff --git a/catalyst/main.py b/catalyst/main.py
index be06ccd7..159fe454 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -5,6 +5,8 @@ import os
import sys
import textwrap
+import toml
+
from snakeoil.process import namespaces
from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
@@ -17,8 +19,7 @@ from catalyst.defaults import confdefaults, option_messages, DEFAULT_CONFIG_FILE
from catalyst.support import CatalystError
from catalyst.version import get_version
-
-conf_values = {}
+conf_values = confdefaults
def version():
@@ -30,42 +31,20 @@ def version():
def parse_config(config_files):
- # search a couple of different areas for the main config file
- myconf = {}
-
- # try and parse the config file "config_file"
for config_file in config_files:
log.notice('Loading configuration file: %s', config_file)
try:
- config = catalyst.config.ConfigParser(config_file)
- myconf.update(config.get_values())
+ conf_values.update(toml.load(config_file))
except Exception as e:
log.critical('Could not find parse configuration file: %s: %s',
config_file, e)
- # now, load up the values into conf_values so that we can use them
- for x in list(confdefaults):
- if x in myconf:
- if x == 'options':
- conf_values[x] = set(myconf[x].split())
- elif x in ["decompressor_search_order"]:
- conf_values[x] = myconf[x].split()
- else:
- conf_values[x] = myconf[x]
- else:
- conf_values[x] = confdefaults[x]
-
# print out any options messages
for opt in conf_values['options']:
if opt in option_messages:
log.info(option_messages[opt])
- for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir",
- "local_overlay", "repos"]:
- if key in myconf:
- conf_values[key] = myconf[key]
-
- if "envscript" in myconf:
+ if "envscript" in conf_values:
log.info('Envscript support enabled.')
# take care of any variable substitutions that may be left
@@ -297,17 +276,17 @@ def _main(parser, opts):
conf_values['DEBUG'] = opts.debug
conf_values['VERBOSE'] = opts.debug or opts.verbose
- options = set()
+ options = []
if opts.fetchonly:
- options.add('fetch')
+ options.append('fetch')
if opts.purge:
- options.add('purge')
+ options.append('purge')
if opts.purgeonly:
- options.add('purgeonly')
+ options.append('purgeonly')
if opts.purgetmponly:
- options.add('purgetmponly')
+ options.append('purgetmponly')
if opts.clear_autoresume:
- options.add('clear-autoresume')
+ options.append('clear-autoresume')
# Make sure we have some work before moving further.
if not myspecfile and not mycmdline:
@@ -318,7 +297,7 @@ def _main(parser, opts):
# import configuration file and import our main module using those settings
parse_config(myconfigs)
- conf_values["options"].update(options)
+ conf_values["options"].extend(options)
log.notice('conf_values[options] = %s', conf_values['options'])
# initialize our contents generator
@@ -335,7 +314,7 @@ def _main(parser, opts):
if "digests" in conf_values:
valid_digests = hashlib.algorithms_available
- digests = set(conf_values['digests'].split())
+ digests = set(conf_values['digests'])
conf_values['digests'] = digests
# First validate all the requested digests are valid keys.
diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index d33be15f..2272cb86 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -10,53 +10,69 @@
#
# $ python3 -c 'import hashlib; print(hashlib.algorithms_available)'
#
-digests="blake2b sha512"
+digests = ["blake2b", "sha512"]
# envscript allows users to set options such as http proxies, MAKEOPTS,
# GENTOO_MIRRORS, or any other environment variables needed for building.
# The envscript file sets environment variables like so:
# export FOO="bar"
-envscript="/etc/catalyst/catalystrc"
-
-# options set different build-time options for catalyst. Some examples are:
-# autoresume = Attempt to resume a failed build, clear the autoresume flags with
-# the -a option to the catalyst cmdline. -p will clear the autoresume flags
-# as well as your pkgcache and kerncache
-# ( This option is not fully tested, bug reports welcome )
-# bindist = enables the bindist USE flag, please see package specific definition,
-# however, it is suggested to enable this if redistributing builds.
-# This optional USE flag is normally cleaned from the make.conf file on
-# completion of the stage. For a non-cleaned version,
-# use sticky-config also (see below)
-# ccache = enables build time ccache support
-# distcc = enable distcc support for building. You have to set distcc_hosts in
-# your spec file.
-# icecream = enables icecream compiler cluster support for building
-# keepwork = Prevents the removal of the working chroot path and any autoresume
-# files or points.
-# kerncache = keeps a tbz2 of your built kernel and modules (useful if your
-# build stops in livecd-stage2)
-# pkgcache = keeps a tbz2 of every built package (useful if your build stops
-# prematurely)
-# seedcache = use the build output of a previous target if it exists to speed up
-# the copy
-# sticky-config = enables the code that will keep any internal 'catalyst_use' flags
-# added to the USE= for building the stage. These are usually added for legal
-# or specific needs in building the the early stage. Mostly it is the
-# 'bindist' USE flag option that is used for legal reasons, please see its
-# specific definition. It will also keep any /etc/portage/package.*
-# files or directories.
-#
-# (These options can be used together)
-options="autoresume bindist kerncache pkgcache seedcache"
+envscript = "/etc/catalyst/catalystrc"
+
+# options set different build-time options for catalyst.
+options = [
+ # Attempt to resume a failed build, clear the autoresume flags with the
+ # -a option to the catalyst cmdline. -p will clear the autoresume
+ # flags as well as your pkgcache and kerncache
+ "autoresume",
+
+ # Enables the bindist USE flag, please see package specific definition,
+ # however, it is suggested to enable this if redistributing builds.
+ # This optional USE flag is normally cleaned from the make.conf file on
+ # completion of the stage. For a non-cleaned version, use
+ # sticky-config also (see below)
+ "bindist",
+
+ # Enable FEATURES=ccache
+ # "ccache",
+
+ # Enable FEATURES=distcc. You have to set distcc_hosts in your spec
+ # file.
+ # "distcc",
+
+ # Enable FEATURES=icecream
+ # "icecream",
+
+ # Prevents the removal of the working chroot path and any autoresume
+ # files or points.
+ # "keepwork",
+
+ # keeps a tbz2 of your built kernel and modules (useful if your
+ # build stops in livecd-stage2)
+ "kerncache",
+
+ # Build and use binary packages
+ "pkgcache",
+
+ # Use the build output of a previous target if it exists rather than
+ # the tarball
+ "seedcache",
+
+ # enables the code that will keep any internal 'catalyst_use' flags
+ # added to the USE= for building the stage. These are usually added
+ # for legal or specific needs in building the the early stage. Mostly
+ # it is the 'bindist' USE flag option that is used for legal reasons,
+ # please see its specific definition. It will also keep any
+ # /etc/portage/package.* files or directories.
+ # "sticky-config",
+]
# port_logdir is where all build logs will be kept. This dir will be automatically cleaned
# of all logs over 30 days old. If left undefined the logs will remain in the build directory
# as usual and get cleaned every time a stage build is restarted.
-# port_logdir="/var/tmp/catalyst/tmp"
+# port_logdir = "/var/tmp/catalyst/tmp"
# var_tmpfs_portage will mount a tmpfs for /var/tmp/portage so building takes place in RAM
# this feature requires a pretty large tmpfs ({open,libre}office needs ~8GB to build)
# WARNING: If you use too much RAM everything will fail horribly and it is not our fault.
# set size of /var/tmp/portage tmpfs in gigabytes
-# var_tmpfs_portage=16
+# var_tmpfs_portage = 16
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, etc/, catalyst/
@ 2020-05-20 0:26 Matt Turner
0 siblings, 0 replies; 3+ messages in thread
From: Matt Turner @ 2020-05-20 0:26 UTC (permalink / raw
To: gentoo-commits
commit: 883744726a73ea1e97dd15e54c099444f4e7f538
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat May 16 21:53:11 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 20 00:24:54 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=88374472
catalyst: Remove support for source_matching="loose"
This does not seem like a useful feature to me.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 3 ---
catalyst/defaults.py | 1 -
catalyst/support.py | 6 +++---
etc/catalyst.conf | 12 ------------
4 files changed, 3 insertions(+), 19 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index f39895fe..febaf969 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -130,8 +130,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
decomp_opt=self.settings["decomp_opt"])
self.accepted_extensions = self.decompressor.search_order_extensions(
self.settings["decompressor_search_order"])
- log.notice("Source file specification matching setting is: %s",
- self.settings["source_matching"])
log.notice("Accepted source file extensions search order: %s",
self.accepted_extensions)
# save resources, it is not always needed
@@ -409,7 +407,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
normpath(self.settings["storedir"] + "/builds/" +
self.settings["source_subpath"]),
self.accepted_extensions,
- self.settings["source_matching"] in ["strict"]
)
log.debug('Source path returned from file_check is: %s',
self.settings["source_path"])
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 412cb956..14f671fe 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -45,7 +45,6 @@ confdefaults = {
"repos": "%(storedir)s/repos",
"sharedir": "/usr/share/catalyst",
"shdir": "%(sharedir)s/targets",
- "source_matching": "strict",
"storedir": "/var/tmp/catalyst",
"target_distdir": "/var/cache/distfiles",
"target_pkgdir": "/var/cache/binpkgs",
diff --git a/catalyst/support.py b/catalyst/support.py
index c4a5c797..a6a6854a 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -59,7 +59,7 @@ def cmd(mycmd, env=None, debug=False, fail_func=None):
print_traceback=False)
-def file_check(filepath, extensions=None, strict=True):
+def file_check(filepath, extensions=None):
'''Check for the files existence and that only one exists
if others are found with various extensions
'''
@@ -73,8 +73,8 @@ def file_check(filepath, extensions=None, strict=True):
".CONTENTS") and not x.endswith(".CONTENTS.gz") and not x.endswith(".DIGESTS")]
if len(files) == 1:
return files[0]
- if len(files) > 1 and strict:
- msg = "Ambiguos Filename: %s\nPlease specify the correct extension as well" % filepath
+ if len(files) > 1:
+ msg = "Ambiguous Filename: %s\nPlease specify the correct extension as well" % filepath
raise CatalystError(msg, print_traceback=False)
target_file = None
for ext in extensions:
diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index f64fe971..d33be15f 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -50,18 +50,6 @@ envscript="/etc/catalyst/catalystrc"
# (These options can be used together)
options="autoresume bindist kerncache pkgcache seedcache"
-# source_matching specifies how catalyst will match non-specific file names
-# if the filename is not found as an exact match.
-# ie: a filename without the extension specified. "/path/to/foo"
-#
-# possible values are:
-# "strict" meaning if more than one file of that name is present with any
-# file extension, then it will raise an exception.
-# "loose" meaning it will search for an existing filename with an added
-# extension from an ordered list of extensions determined from the
-# decompressor_search_order specification in the spec file or (default)
-source_matching="strict"
-
# port_logdir is where all build logs will be kept. This dir will be automatically cleaned
# of all logs over 30 days old. If left undefined the logs will remain in the build directory
# as usual and get cleaned every time a stage build is restarted.
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2020-05-20 0:26 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-05-20 0:26 [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, etc/, catalyst/ Matt Turner
-- strict thread matches above, loose matches on Subject: below --
2020-05-20 0:26 Matt Turner
2020-05-19 23:39 Matt Turner
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox