* [gentoo-commits] proj/catalyst:pending/mattst88 commit in: catalyst/, catalyst/base/
@ 2020-10-29 13:08 Matt Turner
0 siblings, 0 replies; 4+ messages in thread
From: Matt Turner @ 2020-10-29 13:08 UTC (permalink / raw
To: gentoo-commits
commit: befeef9cf3acaef27161a37197ec9f49e80ef4e0
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 28 21:59:17 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Oct 29 13:08:36 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=befeef9c
catalyst: Add and use namespace context manager
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 6 ++++--
catalyst/context.py | 33 +++++++++++++++++++++++++++++++++
catalyst/main.py | 17 +++++++----------
3 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index da133bf2..2bbbb987 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -14,6 +14,7 @@ from snakeoil.osutils import pjoin
from DeComp.compress import CompressMap
from catalyst import log
+from catalyst.context import namespace
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
cmd, read_makeconf, ismount, file_check,
@@ -1392,8 +1393,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not self.run_sequence(self.prepare_sequence):
return False
- if not self.run_sequence(self.build_sequence):
- return False
+ with namespace(mount=True):
+ if not self.run_sequence(self.build_sequence):
+ return False
if not self.run_sequence(self.finish_sequence):
return False
diff --git a/catalyst/context.py b/catalyst/context.py
new file mode 100644
index 00000000..f5d240ea
--- /dev/null
+++ b/catalyst/context.py
@@ -0,0 +1,33 @@
+
+import contextlib
+import os
+
+from snakeoil.process.namespaces import setns, simple_unshare
+
+@contextlib.contextmanager
+def namespace(mount=False, uts=False, ipc=False, net=False, pid=False,
+ user=False, hostname=None):
+ namespaces = {
+ (mount, "mnt"): None,
+ (uts, "uts"): None,
+ (ipc, "ipc"): None,
+ (net, "net"): None,
+ (pid, "pid"): None,
+ (user, "user"): None,
+ }
+ pid = os.getpid()
+
+ # Save fds of current namespaces
+ for ns in [ns for ns in namespaces if ns[0]]:
+ fp = open(f"/proc/{pid}/ns/{ns[1]}")
+ namespaces[ns] = fp
+
+ simple_unshare(mount=mount, uts=uts, ipc=ipc, net=net, pid=pid, user=user,
+ hostname=hostname)
+ try:
+ yield None
+ finally:
+ for ns in [ns for ns in namespaces if ns[0]]:
+ fp = namespaces[ns]
+ setns(fp.fileno(), 0)
+ fp.close()
diff --git a/catalyst/main.py b/catalyst/main.py
index 543895c6..5536471a 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -7,14 +7,13 @@ import textwrap
import toml
-from snakeoil.process import namespaces
-
from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
CONTENTS_DEFINITIONS)
from DeComp.contents import ContentsMap
from catalyst import log
import catalyst.config
+from catalyst.context import namespace
from catalyst.defaults import (confdefaults, option_messages,
DEFAULT_CONFIG_FILE, valid_config_file_values)
from catalyst.support import CatalystError
@@ -356,15 +355,13 @@ def _main(parser, opts):
# use pid & user namespaces, but snakeoil's namespace module has signal
# transfer issues (CTRL+C doesn't propagate), and user namespaces need
# more work due to Gentoo build process (uses sudo/root/portage).
- namespaces.simple_unshare(
- mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
- hostname='catalyst')
+ with namespace(uts=True, ipc=True, hostname='catalyst'):
+ # everything is setup, so the build is a go
+ try:
+ success = build_target(addlargs)
+ except KeyboardInterrupt:
+ log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
- # everything is setup, so the build is a go
- try:
- success = build_target(addlargs)
- except KeyboardInterrupt:
- log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
if not success:
sys.exit(2)
sys.exit(0)
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:pending/mattst88 commit in: catalyst/, catalyst/base/
@ 2020-10-29 14:24 Matt Turner
0 siblings, 0 replies; 4+ messages in thread
From: Matt Turner @ 2020-10-29 14:24 UTC (permalink / raw
To: gentoo-commits
commit: e042cf37bcb4f4c4e7c1665f69faf971fa8177e8
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 28 21:59:17 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Oct 29 14:24:15 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e042cf37
catalyst: Add and use namespace context manager
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 6 ++++--
catalyst/context.py | 33 +++++++++++++++++++++++++++++++++
catalyst/main.py | 17 +++++++----------
3 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index da133bf2..2bbbb987 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -14,6 +14,7 @@ from snakeoil.osutils import pjoin
from DeComp.compress import CompressMap
from catalyst import log
+from catalyst.context import namespace
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
cmd, read_makeconf, ismount, file_check,
@@ -1392,8 +1393,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not self.run_sequence(self.prepare_sequence):
return False
- if not self.run_sequence(self.build_sequence):
- return False
+ with namespace(mount=True):
+ if not self.run_sequence(self.build_sequence):
+ return False
if not self.run_sequence(self.finish_sequence):
return False
diff --git a/catalyst/context.py b/catalyst/context.py
new file mode 100644
index 00000000..b53be56e
--- /dev/null
+++ b/catalyst/context.py
@@ -0,0 +1,33 @@
+
+import contextlib
+import os
+
+from snakeoil.process.namespaces import setns, simple_unshare
+
+@contextlib.contextmanager
+def namespace(mount=False, uts=False, ipc=False, net=False, pid=False,
+ user=False, hostname=None):
+ namespaces = {
+ (mount, "mnt"): None,
+ (uts, "uts"): None,
+ (ipc, "ipc"): None,
+ (net, "net"): None,
+ (pid, "pid"): None,
+ (user, "user"): None,
+ }
+ process_id = os.getpid()
+
+ # Save fds of current namespaces
+ for ns in [ns for ns in namespaces if ns[0]]:
+ fp = open(f"/proc/{process_id}/ns/{ns[1]}")
+ namespaces[ns] = fp
+
+ simple_unshare(mount=mount, uts=uts, ipc=ipc, net=net, pid=pid, user=user,
+ hostname=hostname)
+ try:
+ yield None
+ finally:
+ for ns in [ns for ns in namespaces if ns[0]]:
+ fp = namespaces[ns]
+ setns(fp.fileno(), 0)
+ fp.close()
diff --git a/catalyst/main.py b/catalyst/main.py
index 543895c6..93a4a0d3 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -7,14 +7,13 @@ import textwrap
import toml
-from snakeoil.process import namespaces
-
from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
CONTENTS_DEFINITIONS)
from DeComp.contents import ContentsMap
from catalyst import log
import catalyst.config
+from catalyst.context import namespace
from catalyst.defaults import (confdefaults, option_messages,
DEFAULT_CONFIG_FILE, valid_config_file_values)
from catalyst.support import CatalystError
@@ -356,15 +355,13 @@ def _main(parser, opts):
# use pid & user namespaces, but snakeoil's namespace module has signal
# transfer issues (CTRL+C doesn't propagate), and user namespaces need
# more work due to Gentoo build process (uses sudo/root/portage).
- namespaces.simple_unshare(
- mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
- hostname='catalyst')
+ with namespace(mount=True, uts=True, ipc=True, hostname='catalyst'):
+ # everything is setup, so the build is a go
+ try:
+ success = build_target(addlargs)
+ except KeyboardInterrupt:
+ log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
- # everything is setup, so the build is a go
- try:
- success = build_target(addlargs)
- except KeyboardInterrupt:
- log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
if not success:
sys.exit(2)
sys.exit(0)
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:pending/mattst88 commit in: catalyst/, catalyst/base/
@ 2020-10-29 16:14 Matt Turner
0 siblings, 0 replies; 4+ messages in thread
From: Matt Turner @ 2020-10-29 16:14 UTC (permalink / raw
To: gentoo-commits
commit: e8a048bbc0cef96f2a65365aa73169e6eff0f405
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 29 15:00:42 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Oct 29 16:13:47 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e8a048bb
catalyst: Run the build sequence in new mount namespace
Catalyst has a lot of code to unmount the bind mounts it's made, and
then more to try harder when something fails. This is important because
if bind mounts still exist within the chroot when clean up happens,
files outside of the chroot on the host system can inadvertently be
deleted. E.g., distfiles, binpkgs, kerncache.
Running the build sequence (the steps that need bind mounts) within a
mount namespace and exiting the mount namespace when finished ensures
that clean up can never accidentally delete files outside the chroot.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 8 +++++---
catalyst/main.py | 2 +-
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 06ec8727..ec9a8f06 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -15,6 +15,7 @@ from snakeoil.osutils import pjoin
from DeComp.compress import CompressMap
from catalyst import log
+from catalyst.context import namespace
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
cmd, read_makeconf, ismount, file_check,
@@ -1405,9 +1406,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not self.run_sequence(self.prepare_sequence):
return False
- if not self.run_sequence(self.build_sequence):
- self.unbind()
- return False
+ with namespace(mount=True):
+ if not self.run_sequence(self.build_sequence):
+ self.unbind()
+ return False
if not self.run_sequence(self.finish_sequence):
return False
diff --git a/catalyst/main.py b/catalyst/main.py
index 93a4a0d3..5536471a 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -355,7 +355,7 @@ def _main(parser, opts):
# use pid & user namespaces, but snakeoil's namespace module has signal
# transfer issues (CTRL+C doesn't propagate), and user namespaces need
# more work due to Gentoo build process (uses sudo/root/portage).
- with namespace(mount=True, uts=True, ipc=True, hostname='catalyst'):
+ with namespace(uts=True, ipc=True, hostname='catalyst'):
# everything is setup, so the build is a go
try:
success = build_target(addlargs)
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:pending/mattst88 commit in: catalyst/, catalyst/base/
@ 2021-01-18 19:53 Matt Turner
0 siblings, 0 replies; 4+ messages in thread
From: Matt Turner @ 2021-01-18 19:53 UTC (permalink / raw
To: gentoo-commits
commit: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6
Author: Felix Bier <Felix.Bier <AT> rohde-schwarz <DOT> com>
AuthorDate: Tue Nov 10 01:03:03 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sat Nov 14 16:34:57 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=37a38605
Move from PORTDIR_OVERLAY to repos.conf
This commit fixes the following issues:
* The PORTDIR_OVERLAY variable has been deprecated by Gentoo.
With this commit, the variable is no longer written to the
generated make.conf. Instead, a config file
/etc/portage/repos.conf/<repo-name>.conf
is generated for each overlay. The repo name is read from the
overlay using the portage API. Internally, portage parses
metadata/layout.conf and profiles/repo_name to obtain the name.
References:
https://wiki.gentoo.org/wiki//etc/portage/make.conf
https://wiki.gentoo.org/wiki//etc/portage/repos.conf
* All overlays were copied into the same target directory. If the
same file name occurred in multiple overlays, the last overlay
would overwrite all previous files with this name. In particular,
only the metadata/layout.conf of the last overlay was retained,
so it was not possible to reference the other overlays e.g. via
the masters entry in the layout.conf or the portage-2 syntax
for specifying a parent profile from another overlay. Also,
this created problems when the overlays contained ebuilds
for the same package, but with differing versions, because
after copying, the target directory contained both versions of the
ebuild but only the manifest file of the last overlay.
With this commit, each overlay is copied into a separate
sub-directory, e.g. /var/db/repos/<repo-name>.
This directory is referenced via the location entry in the
generated /etc/portage/repos.conf/<repo-name>.conf.
Signed-off-by: Felix Bier <felix.bier <AT> rohde-schwarz.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 84 ++++++++++++++++++++++++++++++++++------------
catalyst/defaults.py | 2 +-
catalyst/support.py | 18 ++++++++++
3 files changed, 81 insertions(+), 23 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 21cf96a0..fe79b55a 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1,4 +1,5 @@
+import configparser
import copy
import os
import platform
@@ -19,8 +20,8 @@ from catalyst import log
from catalyst.context import namespace
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
- cmd, read_makeconf, ismount, file_check,
- sanitize_name)
+ cmd, read_makeconf, get_repo_name, ismount,
+ file_check, sanitize_name)
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
@@ -786,17 +787,55 @@ class StageBase(TargetBase, ClearBase, GenBase):
env=self.env)
self.resume.enable("setup_confdir")
+ def to_chroot(self, path):
+ """ Prepend chroot path to the given path. """
+
+ chroot = Path(self.settings['chroot_path'])
+ return chroot / path.relative_to(path.anchor)
+
+ def get_repo_conf_path(self, repo_name):
+ """ Construct repo conf path: {repos_conf}/{name}.conf """
+ return Path(self.settings['repos_conf'], repo_name + ".conf")
+
+ def get_repo_location(self, repo_name):
+ """ Construct overlay repo path: {repo_basedir}/{name} """
+ return Path(self.settings['repo_basedir'], repo_name)
+
+ def write_repo_conf(self, repo_name, config):
+ """ Write ConfigParser to {chroot}/{repos_conf}/{name}.conf """
+
+ repo_conf = self.get_repo_conf_path(repo_name)
+
+ repo_conf_chroot = self.to_chroot(repo_conf)
+ repo_conf_chroot.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+ log.info(f'Creating repo config {repo_conf_chroot}.')
+
+ try:
+ with open(repo_conf_chroot, 'w') as f:
+ config.write(f)
+ except OSError as e:
+ raise CatalystError(f'Could not write {repo_conf_chroot}: {e}') from e
+
def portage_overlay(self):
- """ We copy the contents of our overlays to /usr/local/portage """
+ """ We copy the contents of our repos to get_repo_location(repo_name) """
if "portage_overlay" in self.settings:
for x in self.settings["portage_overlay"]:
if os.path.exists(x):
- log.info('Copying overlay dir %s', x)
- ensure_dirs(
- self.settings['chroot_path'] + self.settings['local_overlay'])
- cmd("cp -a " + x + "/* " + self.settings["chroot_path"] +
- self.settings["local_overlay"],
- env=self.env)
+ name = get_repo_name(x)
+
+ location = self.get_repo_location(name)
+ config = configparser.ConfigParser()
+ config[name] = {'location': location}
+ self.write_repo_conf(name, config)
+
+ location_chroot = self.to_chroot(location)
+ location_chroot.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+ log.info(f'Copying overlay dir {x} to {location_chroot}')
+ cmd(f'cp -a {x}/* {location_chroot}', env=self.env)
+ else:
+ log.warning(f'Skipping missing overlay {x}.')
def root_overlay(self):
""" Copy over the root_overlay """
@@ -852,8 +891,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
cxt = libmount.Context(source=source, target=target,
fstype=fstype, options=options)
cxt.mount()
- except OSError as e:
- raise CatalystError(f"Couldn't mount: {source}, {e.strerror}")
+ except Exception as e:
+ raise CatalystError(f"Couldn't mount: {source}, {e}")
def chroot_setup(self):
self.makeconf = read_makeconf(normpath(self.settings["chroot_path"] +
@@ -1018,12 +1057,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
varname = x.split('_')[1].upper()
myf.write(f'{varname}="{self.settings[x]}"\n')
- if setup:
- # Setup the portage overlay
- if "portage_overlay" in self.settings:
- myf.write('PORTDIR_OVERLAY="%s"\n' %
- self.settings["local_overlay"])
-
# Set default locale for system responses. #478382
myf.write(
'\n'
@@ -1097,11 +1130,18 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.warning("You've been hacking. Clearing target patches: %s", target)
clear_path(target)
- # Remove our overlay
- overlay = normpath(
- self.settings["chroot_path"] + self.settings["local_overlay"])
- if os.path.exists(overlay):
- clear_path(overlay)
+ # Remove our overlays
+ if "portage_overlay" in self.settings:
+ for repo_path in self.settings["portage_overlay"]:
+ repo_name = get_repo_name(repo_path)
+
+ repo_conf = self.get_repo_conf_path(repo_name)
+ chroot_repo_conf = self.to_chroot(repo_conf)
+ chroot_repo_conf.unlink()
+
+ location = self.get_repo_location(repo_name)
+ chroot_location = self.to_chroot(location)
+ clear_path(str(chroot_location))
if "sticky-config" not in self.settings["options"]:
# re-write the make.conf to be sure it is clean
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 0f399b56..3f12b8d5 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -38,9 +38,9 @@ confdefaults = {
"distdir": portage.settings['DISTDIR'],
"icecream": "/var/cache/icecream",
'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'],
- "local_overlay": "/var/db/repos/local",
"port_conf": "/etc/portage",
"make_conf": "%(port_conf)s/make.conf",
+ "repos_conf": "%(port_conf)s/repos.conf",
"options": set(),
"pkgdir": "/var/cache/binpkgs",
"port_tmpdir": "/var/tmp/portage",
diff --git a/catalyst/support.py b/catalyst/support.py
index ddbd9ab9..f3a865a7 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -10,6 +10,8 @@ from subprocess import Popen
import libmount
+from portage.repository.config import RepoConfig
+
from catalyst import log
BASH_BINARY = "/bin/bash"
@@ -182,6 +184,22 @@ def read_makeconf(mymakeconffile):
return makeconf
+def get_repo_name(repo_path):
+ """ Get the name of the repo at the given repo_path.
+
+ References:
+ https://wiki.gentoo.org/wiki/Repository_format/profiles/repo_name
+ https://wiki.gentoo.org/wiki/Repository_format/metadata/layout.conf#repo-name
+ """
+
+ repo_config = RepoConfig(None, {"location": repo_path})
+
+ if repo_config.missing_repo_name:
+ raise CatalystError(f"Missing name in repository {repo_path}")
+
+ return repo_config.name
+
+
def ismount(path):
"""Like os.path.ismount, but also support bind mounts"""
path = Path(path)
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2021-01-18 19:53 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-10-29 13:08 [gentoo-commits] proj/catalyst:pending/mattst88 commit in: catalyst/, catalyst/base/ Matt Turner
-- strict thread matches above, loose matches on Subject: below --
2020-10-29 14:24 Matt Turner
2020-10-29 16:14 Matt Turner
2021-01-18 19:53 Matt Turner
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox