* [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/
@ 2020-05-27 6:20 Matt Turner
0 siblings, 0 replies; 4+ messages in thread
From: Matt Turner @ 2020-05-27 6:20 UTC (permalink / raw
To: gentoo-commits
commit: b11d30977dd90954357722f488e37ad3cd257ff4
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat May 16 21:44:37 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu May 21 22:07:41 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b11d3097
catalyst: Add and use support API for handling mounts
Handle the mounts/unmounts in all in process rather than shelling out
(pun intended!) to an external program.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 28 ++++++++++++++++++----------
catalyst/mount.py | 32 ++++++++++++++++++++++++++++++++
2 files changed, 50 insertions(+), 10 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 00efd252..aa5cafd0 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -15,6 +15,7 @@ from DeComp.compress import CompressMap
from catalyst import log
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
+from catalyst.mount import MS, mount, umount
from catalyst.support import (CatalystError, file_locate, normpath,
cmd, read_makeconf, ismount, file_check)
from catalyst.base.targetbase import TargetBase
@@ -847,7 +848,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
source = str(self.mount[x]['source'])
target = self.settings['chroot_path'] + str(self.mount[x]['target'])
- mount = ['mount']
+ filesystem = ''
+ flags = MS.NONE
+ options = ''
log.debug('bind %s: "%s" -> "%s"', x, source, target)
@@ -855,18 +858,19 @@ class StageBase(TargetBase, ClearBase, GenBase):
if 'var_tmpfs_portage' not in self.settings:
continue
- mount += ['-t', 'tmpfs', '-o',
- f"size={self.settings['var_tmpfs_portage']}G"]
+ filesystem = 'tmpfs'
+ options = f"size={self.settings['var_tmpfs_portage']}G"
elif source == 'tmpfs':
- mount += ['-t', 'tmpfs']
+ filesystem = 'tmpfs'
elif source == 'shm':
- mount += ['-t', 'tmpfs', '-o', 'noexec,nosuid,nodev']
+ filesystem = 'tmpfs'
+ flags = MS.NOEXEC | MS.NOSUID | MS.NODEV
else:
source_path = Path(self.mount[x]['source'])
if source_path.suffix == '.sqfs':
- mount += ['-o', 'ro']
+ flags = MS.RDONLY
else:
- mount.append('--bind')
+ flags = MS.BIND
# We may need to create the source of the bind mount. E.g., in the
# case of an empty package cache we must create the directory that
@@ -875,7 +879,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
Path(target).mkdir(mode=0o755, parents=True, exist_ok=True)
- cmd(mount + [source, target], env=self.env, fail_func=self.unbind)
+ try:
+ mount(source, target, filesystem, flags, options)
+ except OSError as e:
+ self.unbind()
+ raise CatalystError
def unbind(self):
ouch = 0
@@ -893,7 +901,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
continue
try:
- cmd(['umount', target], env=self.env)
+ umount(target)
except CatalystError:
log.warning('First attempt to unmount failed: %s', target)
log.warning('Killing any pids still running in the chroot')
@@ -901,7 +909,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.kill_chroot_pids()
try:
- cmd(['umount', target], env=self.env)
+ umount(target)
except CatalystError:
ouch = 1
log.warning("Couldn't umount bind mount: %s", target)
diff --git a/catalyst/mount.py b/catalyst/mount.py
new file mode 100644
index 00000000..58b953fc
--- /dev/null
+++ b/catalyst/mount.py
@@ -0,0 +1,32 @@
+import ctypes
+import ctypes.util
+import enum
+import os
+
+libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
+libc.mount.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulong, ctypes.c_char_p)
+libc.umount.argtypes = (ctypes.c_char_p)
+
+class MS(enum.IntFlag):
+ NONE = 0x0
+ RDONLY = 0x1
+ NOSUID = 0x2
+ NODEV = 0x4
+ NOEXEC = 0x8
+ BIND = 0x1000
+
+def mount(source: str, target: str, fs: str, flags: MS = MS.NONE, options: str = '') -> None:
+ ret = libc.mount(source.encode(), target.encode(), fs.encode(), flags,
+ options.encode())
+ if ret < 0:
+ errno = ctypes.get_errno()
+ raise OSError(errno,
+ f"Error mounting {source} ({fs}) on {target} with options"
+ f" {options}': {os.strerror(errno)}")
+
+def umount(target: str) -> None:
+ ret = libc.umount(target.encode())
+ if ret < 0:
+ errno = ctypes.get_errno()
+ raise OSError(errno,
+ f"Error unmounting {target}': {os.strerror(errno)}"
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/
@ 2020-05-27 6:21 Matt Turner
0 siblings, 0 replies; 4+ messages in thread
From: Matt Turner @ 2020-05-27 6:21 UTC (permalink / raw
To: gentoo-commits
commit: fcf1829e26eb901354958e92d48f214f2475aff4
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat May 16 21:44:37 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed May 27 06:21:10 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=fcf1829e
catalyst: Add and use support API for handling mounts
Handle the mounts/unmounts in all in process rather than shelling out
(pun intended!) to an external program.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 28 ++++++++++++++++++----------
catalyst/mount.py | 32 ++++++++++++++++++++++++++++++++
2 files changed, 50 insertions(+), 10 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 00efd252..aa5cafd0 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -15,6 +15,7 @@ from DeComp.compress import CompressMap
from catalyst import log
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
+from catalyst.mount import MS, mount, umount
from catalyst.support import (CatalystError, file_locate, normpath,
cmd, read_makeconf, ismount, file_check)
from catalyst.base.targetbase import TargetBase
@@ -847,7 +848,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
source = str(self.mount[x]['source'])
target = self.settings['chroot_path'] + str(self.mount[x]['target'])
- mount = ['mount']
+ filesystem = ''
+ flags = MS.NONE
+ options = ''
log.debug('bind %s: "%s" -> "%s"', x, source, target)
@@ -855,18 +858,19 @@ class StageBase(TargetBase, ClearBase, GenBase):
if 'var_tmpfs_portage' not in self.settings:
continue
- mount += ['-t', 'tmpfs', '-o',
- f"size={self.settings['var_tmpfs_portage']}G"]
+ filesystem = 'tmpfs'
+ options = f"size={self.settings['var_tmpfs_portage']}G"
elif source == 'tmpfs':
- mount += ['-t', 'tmpfs']
+ filesystem = 'tmpfs'
elif source == 'shm':
- mount += ['-t', 'tmpfs', '-o', 'noexec,nosuid,nodev']
+ filesystem = 'tmpfs'
+ flags = MS.NOEXEC | MS.NOSUID | MS.NODEV
else:
source_path = Path(self.mount[x]['source'])
if source_path.suffix == '.sqfs':
- mount += ['-o', 'ro']
+ flags = MS.RDONLY
else:
- mount.append('--bind')
+ flags = MS.BIND
# We may need to create the source of the bind mount. E.g., in the
# case of an empty package cache we must create the directory that
@@ -875,7 +879,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
Path(target).mkdir(mode=0o755, parents=True, exist_ok=True)
- cmd(mount + [source, target], env=self.env, fail_func=self.unbind)
+ try:
+ mount(source, target, filesystem, flags, options)
+ except OSError as e:
+ self.unbind()
+ raise CatalystError
def unbind(self):
ouch = 0
@@ -893,7 +901,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
continue
try:
- cmd(['umount', target], env=self.env)
+ umount(target)
except CatalystError:
log.warning('First attempt to unmount failed: %s', target)
log.warning('Killing any pids still running in the chroot')
@@ -901,7 +909,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.kill_chroot_pids()
try:
- cmd(['umount', target], env=self.env)
+ umount(target)
except CatalystError:
ouch = 1
log.warning("Couldn't umount bind mount: %s", target)
diff --git a/catalyst/mount.py b/catalyst/mount.py
new file mode 100644
index 00000000..0125353b
--- /dev/null
+++ b/catalyst/mount.py
@@ -0,0 +1,32 @@
+import ctypes
+import ctypes.util
+import enum
+import os
+
+libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
+libc.mount.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulong, ctypes.c_char_p)
+libc.umount.argtypes = (ctypes.c_char_p)
+
+class MS(enum.IntFlag):
+ NONE = 0x0
+ RDONLY = 0x1
+ NOSUID = 0x2
+ NODEV = 0x4
+ NOEXEC = 0x8
+ BIND = 0x1000
+
+def mount(source: str, target: str, fs: str, flags: MS = MS.NONE, options: str = '') -> None:
+ ret = libc.mount(source.encode(), target.encode(), fs.encode(), flags,
+ options.encode())
+ if ret < 0:
+ errno = ctypes.get_errno()
+ raise OSError(errno,
+ f"Error mounting {source} ({fs}) on {target} with options"
+ f" {options}': {os.strerror(errno)}")
+
+def umount(target: str) -> None:
+ ret = libc.umount(target.encode())
+ if ret < 0:
+ errno = ctypes.get_errno()
+ raise OSError(errno,
+ f"Error unmounting {target}': {os.strerror(errno)}")
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/
2020-10-30 22:41 [gentoo-commits] proj/catalyst:master " Matt Turner
@ 2020-12-19 19:56 ` Matt Turner
0 siblings, 0 replies; 4+ messages in thread
From: Matt Turner @ 2020-12-19 19:56 UTC (permalink / raw
To: gentoo-commits
commit: 488b06bf5dbe1eba68ac11de95f56feeb6cead83
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 29 15:00:42 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Oct 30 22:40:52 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=488b06bf
catalyst: Run the build sequence in new mount namespace
Catalyst has a lot of code to unmount the bind mounts it's made, and
then more to try harder when something fails. This is important because
if bind mounts still exist within the chroot when clean up happens,
files outside of the chroot on the host system can inadvertently be
deleted. E.g., distfiles, binpkgs, kerncache.
Running the build sequence (the steps that need bind mounts) within a
mount namespace and exiting the mount namespace when finished ensures
that clean up can never accidentally delete files outside the chroot.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 7 ++++---
catalyst/main.py | 2 +-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 06ec8727..caec5935 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -15,6 +15,7 @@ from snakeoil.osutils import pjoin
from DeComp.compress import CompressMap
from catalyst import log
+from catalyst.context import namespace
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
cmd, read_makeconf, ismount, file_check,
@@ -1405,9 +1406,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not self.run_sequence(self.prepare_sequence):
return False
- if not self.run_sequence(self.build_sequence):
- self.unbind()
- return False
+ with namespace(mount=True):
+ if not self.run_sequence(self.build_sequence):
+ return False
if not self.run_sequence(self.finish_sequence):
return False
diff --git a/catalyst/main.py b/catalyst/main.py
index 93a4a0d3..5536471a 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -355,7 +355,7 @@ def _main(parser, opts):
# use pid & user namespaces, but snakeoil's namespace module has signal
# transfer issues (CTRL+C doesn't propagate), and user namespaces need
# more work due to Gentoo build process (uses sudo/root/portage).
- with namespace(mount=True, uts=True, ipc=True, hostname='catalyst'):
+ with namespace(uts=True, ipc=True, hostname='catalyst'):
# everything is setup, so the build is a go
try:
success = build_target(addlargs)
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/
2021-02-20 21:27 [gentoo-commits] proj/catalyst:master " Matt Turner
@ 2021-06-10 0:48 ` Matt Turner
0 siblings, 0 replies; 4+ messages in thread
From: Matt Turner @ 2021-06-10 0:48 UTC (permalink / raw
To: gentoo-commits
commit: 65d49f1028b49fed6e011526429e553ebb6c903e
Author: Felix Bier <Felix.Bier <AT> rohde-schwarz <DOT> com>
AuthorDate: Thu Feb 4 00:38:36 2021 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sat Feb 20 21:27:29 2021 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=65d49f10
Unify handling of main repo and other repos
This commit unifies the handling of the main repo and the other repos.
All are stored in one common list. A mount entry is created for
each entry in the list (previously a mount entry was only created
for the main repo and the other repos were copied into the chroot).
This means each non-main repo can now be either a directory or a
squash files (previously the non-main repos had to be directories).
The existing mount logic will bind-mount the repos that are stored as
directories, removing the need for copying.
A repos.conf entry will be created for each entry in the list.
This means a repos.conf entry for the main repo can now be created
(previously repos.conf entries were only created for the non-main
repos). The repos.conf entry will only be created if the target
location for the main repo is non-default, i.e. unequal to
/var/db/repos/gentoo. This mirrors the behavior of write_make_conf,
which only writes the PORTDIR variable to make.conf if the location
of the main repo is non-default.
As a side effect, the PORTDIR variable is now no longer needed,
and will be removed from write_make_conf in a separate commit.
Signed-off-by: Felix Bier <felix.bier <AT> rohde-schwarz.com>
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 88 +++++++++++++++++++++++++++++++---------------
catalyst/defaults.py | 5 ---
2 files changed, 60 insertions(+), 33 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index fedc8f87..97e2318c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -219,8 +219,17 @@ class StageBase(TargetBase, ClearBase, GenBase):
# Setup our mount points.
self.mount = copy.deepcopy(MOUNT_DEFAULTS)
- self.mount['portdir']['source'] = self.snapshot
- self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/' + self.settings['repo_name']
+ # Create mount entry for each repository
+ for path, name, _ in self.repos:
+ name = get_repo_name(path)
+ mount_id = f'repo_{name}'
+
+ self.mount[mount_id] = {
+ 'enable': True,
+ 'source': path,
+ 'target': self.get_repo_location(name)
+ }
+
self.mount['distdir']['source'] = self.settings['distdir']
self.mount["distdir"]['target'] = self.settings['target_distdir']
@@ -587,13 +596,41 @@ class StageBase(TargetBase, ClearBase, GenBase):
"/busybox_config"]
def set_repos(self):
+
+ # Each entry in this list will be a tuple of the form
+ # (source, name, default)
+ #
+ # source: the location of the repo on the host system,
+ # either a directory or a squashfs file.
+ #
+ # name: the repository name parsed from the repo.
+ # This is just a caching mechanism to avoid parsing the name
+ # every time the source is processed.
+ #
+ # default: Default location where the repo is expected in the
+ # target system. If this matches the path where we mount the repo to
+ # (as per get_repo_location), then we can skip generating a repos.conf
+ # entry for that repo. Currently this mechanism is only used for
+ # the main repo, which has a default location hard-coded in
+ # /usr/share/portage/config/repos.conf. For the other repos,
+ # the default is set to None.
+ self.repos = []
+
+ # Create entry for snapshot
+ default_location = Path(confdefaults['repo_basedir'], confdefaults['repo_name'])
+ self.repos.append((self.snapshot, get_repo_name(self.snapshot), default_location))
+
+ # Create entry for every other repo
if 'repos' in self.settings:
if isinstance(self.settings['repos'], str):
self.settings['repos'] = \
self.settings['repos'].split()
- log.info('repos directories are set to: %s',
+ log.info('repos are set to: %s',
' '.join(self.settings['repos']))
+ get_info = lambda repo: (repo, get_repo_name(repo), None)
+ self.repos.extend(map(get_info, self.settings['repos']))
+
def set_overlay(self):
if self.settings["spec_prefix"] + "/overlay" in self.settings:
if isinstance(self.settings[self.settings['spec_prefix'] + '/overlay'], str):
@@ -832,24 +869,19 @@ class StageBase(TargetBase, ClearBase, GenBase):
raise CatalystError(f'Could not write {repo_conf_chroot}: {e}') from e
def process_repos(self):
- """ We copy the contents of our repos to get_repo_location(repo_name) """
- if 'repos' in self.settings:
- for x in self.settings['repos']:
- if os.path.exists(x):
- name = get_repo_name(x)
+ """ Create repos.conf entry for every repo """
- location = self.get_repo_location(name)
- config = configparser.ConfigParser()
- config[name] = {'location': location}
- self.write_repo_conf(name, config)
+ for _, name, default in self.repos:
+ location = self.get_repo_location(name)
- location_chroot = self.to_chroot(location)
- location_chroot.mkdir(mode=0o755, parents=True, exist_ok=True)
+ if default == location:
+ log.debug('Skipping repos.conf entry for repo %s '
+ 'with default location %s.', name, location)
+ continue
- log.info('Copying overlay dir %s to %s', x, location_chroot)
- cmd(f'cp -a {x}/* {location_chroot}', env=self.env)
- else:
- log.warning('Skipping missing overlay %s.', x)
+ config = configparser.ConfigParser()
+ config[name] = {'location': location}
+ self.write_repo_conf(name, config)
def root_overlay(self):
""" Copy over the root_overlay """
@@ -1144,18 +1176,18 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.warning("You've been hacking. Clearing target patches: %s", target)
clear_path(target)
- # Remove our overlays
- if 'repos' in self.settings:
- for repo_path in self.settings['repos']:
- repo_name = get_repo_name(repo_path)
+ # Remove repo data
+ for _, name, _ in self.repos:
- repo_conf = self.get_repo_conf_path(repo_name)
- chroot_repo_conf = self.to_chroot(repo_conf)
- chroot_repo_conf.unlink()
+ # Remove repos.conf entry
+ repo_conf = self.get_repo_conf_path(name)
+ chroot_repo_conf = self.to_chroot(repo_conf)
+ chroot_repo_conf.unlink(missing_ok=True)
- location = self.get_repo_location(repo_name)
- chroot_location = self.to_chroot(location)
- clear_path(str(chroot_location))
+ # The repo has already been unmounted, remove the mount point
+ location = self.get_repo_location(name)
+ chroot_location = self.to_chroot(location)
+ clear_path(str(chroot_location))
if "sticky-config" not in self.settings["options"]:
# re-write the make.conf to be sure it is clean
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 3d5c0a7f..ccb0a584 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -85,11 +85,6 @@ MOUNT_DEFAULTS = OrderedDict([
'source': 'tmpfs',
'target': '/run',
}),
- ('portdir', {
- 'enable': True,
- 'source': 'config',
- 'target': 'config',
- }),
('distdir', {
'enable': True,
'source': 'config',
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2021-06-10 0:48 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-05-27 6:21 [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/ Matt Turner
-- strict thread matches above, loose matches on Subject: below --
2021-02-20 21:27 [gentoo-commits] proj/catalyst:master " Matt Turner
2021-06-10 0:48 ` [gentoo-commits] proj/catalyst:wip/mattst88 " Matt Turner
2020-10-30 22:41 [gentoo-commits] proj/catalyst:master " Matt Turner
2020-12-19 19:56 ` [gentoo-commits] proj/catalyst:wip/mattst88 " Matt Turner
2020-05-27 6:20 Matt Turner
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox