* [gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/
@ 2020-04-10 21:04 Matt Turner
0 siblings, 0 replies; 3+ messages in thread
From: Matt Turner @ 2020-04-10 21:04 UTC (permalink / raw
To: gentoo-commits
commit: 8ef6e85a57ecefb4eb7b4b50796dbb98ab1a3f2a
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 10 07:40:48 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Apr 10 07:42:08 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=8ef6e85a
catalyst: Drop BSD remnants
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/clearbase.py | 4 ++--
catalyst/base/resume.py | 2 +-
catalyst/base/stagebase.py | 19 ++++++-------------
catalyst/builder.py | 3 +--
catalyst/defaults.py | 14 ++++----------
catalyst/fileops.py | 6 +-----
catalyst/main.py | 17 +++++++----------
catalyst/targets/stage1.py | 14 +-------------
8 files changed, 23 insertions(+), 56 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index f1d4d1ba..8dfffb06 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -30,13 +30,13 @@ class ClearBase():
def clear_chroot(self):
self.chroot_lock.unlock()
log.notice('Clearing the chroot path ...')
- clear_dir(self.settings["chroot_path"], mode=0o755, chg_flags=True)
+ clear_dir(self.settings["chroot_path"], mode=0o755)
def remove_chroot(self):
self.chroot_lock.unlock()
log.notice('Removing the chroot path ...')
- clear_dir(self.settings["chroot_path"], mode=0o755, chg_flags=True, remove=True)
+ clear_dir(self.settings["chroot_path"], mode=0o755, remove=True)
def clear_packages(self, remove=False):
diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index 7732dd5d..a0770b9d 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -126,7 +126,7 @@ class AutoResume():
@remove: boolean, passed through to clear_dir()
@return boolean
'''
- if clear_dir(self.basedir, mode=0o755, chg_flags=True, remove=remove):
+ if clear_dir(self.basedir, mode=0o755, remove=remove):
self._points = {}
return True
return False
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 320e9d53..021f3beb 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -224,10 +224,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["snapshot_cache_path"],
self.settings["repo_name"],
]))
- if os.uname()[0] == "Linux":
- self.mounts.append("devpts")
- self.mounts.append("shm")
- self.mounts.append("run")
+ self.mounts.append("devpts")
+ self.mounts.append("shm")
+ self.mounts.append("run")
self.set_mounts()
@@ -983,16 +982,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
elif src == "tmpfs":
_cmd = ['mount', '-t', 'tmpfs', src, target]
else:
- if os.uname()[0] == "FreeBSD":
- if src == "/dev":
- _cmd = ['mount', '-t', 'devfs', 'none', target]
- else:
- _cmd = ['mount_nullfs', src, target]
+ if src == "shmfs":
+ _cmd = ['mount', '-t', 'tmpfs', '-o', 'noexec,nosuid,nodev', 'shm', target]
else:
- if src == "shmfs":
- _cmd = ['mount', '-t', 'tmpfs', '-o', 'noexec,nosuid,nodev', 'shm', target]
- else:
- _cmd = ['mount', '--bind', src, target]
+ _cmd = ['mount', '--bind', src, target]
if _cmd:
log.debug('bind(); _cmd = %s', _cmd)
cmd(_cmd, env=self.env, fail_func=self.unbind)
diff --git a/catalyst/builder.py b/catalyst/builder.py
index 5d7f76c1..e9f468d7 100644
--- a/catalyst/builder.py
+++ b/catalyst/builder.py
@@ -10,8 +10,7 @@ class generic():
Useful for building x86-on-amd64 and such.
"""
- if os.uname()[0] == 'Linux':
- self.settings['CHROOT'] = 'setarch %s %s' % (arch, self.settings['CHROOT'])
+ self.settings['CHROOT'] = 'setarch %s %s' % (arch, self.settings['CHROOT'])
def mount_safety_check(self):
"""
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 5b4ae83e..bcb59796 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -5,12 +5,6 @@ from DeComp.definitions import DECOMPRESSOR_SEARCH_ORDER
from DeComp.definitions import COMPRESSOR_PROGRAM_OPTIONS, XATTRS_OPTIONS
from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS, LIST_XATTRS_OPTIONS
-# Used for the (de)compressor definitions
-if os.uname()[0] in ["Linux", "linux"]:
- TAR = 'linux'
-else:
- TAR = 'bsd'
-
# these should never be touched
required_build_targets = ["targetbase", "generic_stage_target"]
@@ -41,16 +35,16 @@ PORTDIR = REPODIR + "/" + MAINREPO
confdefaults={
"archdir": "%(PythonDir)s/arch",
- "comp_prog": COMPRESSOR_PROGRAM_OPTIONS[TAR],
+ "comp_prog": COMPRESSOR_PROGRAM_OPTIONS['linux'],
"compression_mode": 'lbzip2',
"compressor_arch": None,
- "compressor_options": XATTRS_OPTIONS[TAR],
- "decomp_opt": DECOMPRESSOR_PROGRAM_OPTIONS[TAR],
+ "compressor_options": XATTRS_OPTIONS['linux'],
+ "decomp_opt": DECOMPRESSOR_PROGRAM_OPTIONS['linux'],
"decompressor_search_order": DECOMPRESSOR_SEARCH_ORDER,
"distdir": DISTDIR[:],
"hash_function": "crc32",
"icecream": "/var/cache/icecream",
- 'list_xattrs_opt': LIST_XATTRS_OPTIONS[TAR],
+ 'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'],
"local_overlay": REPODIR[:] + "/local",
"port_conf": "/etc/portage",
"make_conf": "%(port_conf)s/make.conf",
diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index f7df4a01..9fa23c52 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -50,13 +50,12 @@ def ensure_dirs(path, gid=-1, uid=-1, mode=0o755, minimal=True,
return succeeded
-def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
+def clear_dir(target, mode=0o755, remove=False,
clear_nondir=True):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
- @chg_flags: boolean used for FreeBSD hosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
@@ -73,9 +72,6 @@ def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
try:
log.debug('os.stat()')
mystat = os.stat(target)
- # There's no easy way to change flags recursively in python
- if chg_flags and os.uname()[0] == "FreeBSD":
- cmd(['chflags', '-R', 'noschg', target])
log.debug('shutil.rmtree()')
shutil.rmtree(target)
except Exception:
diff --git a/catalyst/main.py b/catalyst/main.py
index 1d52dffe..14733540 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -428,16 +428,13 @@ def _main(parser, opts):
# catalyst cannot be run as a normal user due to chroots, mounts, etc
log.critical('This script requires root privileges to operate')
- # Namespaces aren't supported on *BSDs at the moment. So let's check
- # whether we're on Linux.
- if os.uname().sysname in ["Linux", "linux"]:
- # Start off by creating unique namespaces to run in. Would be nice to
- # use pid & user namespaces, but snakeoil's namespace module has signal
- # transfer issues (CTRL+C doesn't propagate), and user namespaces need
- # more work due to Gentoo build process (uses sudo/root/portage).
- namespaces.simple_unshare(
- mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
- hostname='catalyst')
+ # Start off by creating unique namespaces to run in. Would be nice to
+ # use pid & user namespaces, but snakeoil's namespace module has signal
+ # transfer issues (CTRL+C doesn't propagate), and user namespaces need
+ # more work due to Gentoo build process (uses sudo/root/portage).
+ namespaces.simple_unshare(
+ mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
+ hostname='catalyst')
# everything is setup, so the build is a go
try:
diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py
index 59d6d49b..ca6c1e0d 100644
--- a/catalyst/targets/stage1.py
+++ b/catalyst/targets/stage1.py
@@ -78,19 +78,7 @@ class stage1(StageBase):
"Don't say we did not warn you.")
def base_dirs(self):
- if os.uname()[0] == "FreeBSD":
- # baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
- # would create them too late...we need them earlier before bind mounting filesystems
- # since proc and dev are not writeable, so...create them here
- ensure_dirs(self.settings["stage_path"]+"/proc")
- ensure_dirs(self.settings["stage_path"]+"/dev")
- for f in ('/proc', '/dev'):
- f = self.settings['stage_path'] + f + '/.keep'
- if not os.path.isfile(f):
- try:
- fileutils.touch(f)
- except IOError:
- log.error('Failed to create %s', f)
+ pass
def set_mounts(self):
# stage_path/proc probably doesn't exist yet, so create it
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/
@ 2021-06-11 3:30 Matt Turner
2022-01-30 20:42 ` [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/, catalyst/targets/ Matt Turner
0 siblings, 1 reply; 3+ messages in thread
From: Matt Turner @ 2021-06-11 3:30 UTC (permalink / raw
To: gentoo-commits
commit: b3c917f7fa73d11c69b7e55dc7a00bc18a18edc7
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Jun 9 06:17:31 2021 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Jun 10 04:23:54 2021 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b3c917f7
catalyst: Replace snakeoil's locks with fasteners
To no great surprise, the existing locking was broken. For example,
clear_chroot() releases the lock. It is called by unpack(), which is
part of prepare_sequence. The result is that the whole build could be
done without holding the lock.
Just lock around run(). It's not apparent that finer-grained locking
does anything for us.
Bug: https://bugs.gentoo.org/791583
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/clearbase.py | 2 --
catalyst/base/stagebase.py | 10 +++-----
catalyst/lock.py | 58 --------------------------------------------
catalyst/targets/snapshot.py | 6 ++---
4 files changed, 7 insertions(+), 69 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index dcf6c523..6218330e 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -27,12 +27,10 @@ class ClearBase():
self.resume.clear_all(remove=True)
def clear_chroot(self):
- self.chroot_lock.unlock()
log.notice('Clearing the chroot path ...')
clear_dir(self.settings["chroot_path"], mode=0o755)
def remove_chroot(self):
- self.chroot_lock.unlock()
log.notice('Removing the chroot path ...')
clear_dir(self.settings["chroot_path"], mode=0o755, remove=True)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 448d6265..10cffd4c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -8,6 +8,7 @@ import sys
from pathlib import Path
+import fasteners
import libmount
import toml
@@ -25,7 +26,6 @@ from catalyst.support import (CatalystError, file_locate, normpath,
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
-from catalyst.lock import LockDir, LockInUse
from catalyst.fileops import ensure_dirs, clear_dir, clear_path
from catalyst.base.resume import AutoResume
@@ -36,9 +36,6 @@ def run_sequence(sequence):
sys.stdout.flush()
try:
func()
- except LockInUse:
- log.error('Unable to aquire the lock...')
- return False
except Exception:
log.error('Exception running action sequence %s',
func.__name__, exc_info=True)
@@ -478,7 +475,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"""
self.settings["chroot_path"] = normpath(self.settings["storedir"] +
"/tmp/" + self.settings["target_subpath"].rstrip('/'))
- self.chroot_lock = LockDir(self.settings["chroot_path"])
def set_autoresume_path(self):
self.settings["autoresume_path"] = normpath(pjoin(
@@ -1366,8 +1362,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
pass
def run(self):
- self.chroot_lock.write_lock()
+ with fasteners.InterProcessLock(self.settings["chroot_path"] + '.lock'):
+ return self._run()
+ def _run(self):
if "clear-autoresume" in self.settings["options"]:
self.clear_autoresume()
diff --git a/catalyst/lock.py b/catalyst/lock.py
deleted file mode 100644
index e31745b2..00000000
--- a/catalyst/lock.py
+++ /dev/null
@@ -1,58 +0,0 @@
-
-import os
-
-from contextlib import contextmanager
-
-from snakeoil import fileutils
-from snakeoil import osutils
-from catalyst.fileops import ensure_dirs
-
-
-LockInUse = osutils.LockException
-
-class Lock:
- """
- A fnctl-based filesystem lock
- """
- def __init__(self, lockfile):
- fileutils.touch(lockfile, mode=0o664)
- os.chown(lockfile, uid=-1, gid=250)
- self.lock = osutils.FsLock(lockfile)
-
- def read_lock(self):
- self.lock.acquire_read_lock()
-
- def write_lock(self):
- self.lock.acquire_write_lock()
-
- def unlock(self):
- # Releasing a write lock is the same as a read lock.
- self.lock.release_write_lock()
-
-class LockDir(Lock):
- """
- A fnctl-based filesystem lock in a directory
- """
- def __init__(self, lockdir):
- ensure_dirs(lockdir)
- lockfile = os.path.join(lockdir, '.catalyst_lock')
-
- Lock.__init__(self, lockfile)
-
-@contextmanager
-def read_lock(filename):
- lock = Lock(filename)
- lock.read_lock()
- try:
- yield
- finally:
- lock.unlock()
-
-@contextmanager
-def write_lock(filename):
- lock = Lock(filename)
- lock.write_lock()
- try:
- yield
- finally:
- lock.unlock()
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index b494575a..ef68765d 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -5,11 +5,12 @@ Snapshot target
import subprocess
import sys
+import fasteners
+
from pathlib import Path
from catalyst import log
from catalyst.base.targetbase import TargetBase
-from catalyst.lock import write_lock
from catalyst.support import CatalystError, command
class snapshot(TargetBase):
@@ -93,8 +94,7 @@ class snapshot(TargetBase):
log.notice('>>> ' + ' '.join([*git_cmd, '|']))
log.notice(' ' + ' '.join(tar2sqfs_cmd))
- lockfile = self.snapshot.with_suffix('.lock')
- with write_lock(lockfile):
+ with fasteners.InterProcessLock(self.snapshot.with_suffix('.lock')):
git = subprocess.Popen(git_cmd,
stdout=subprocess.PIPE,
stderr=sys.stderr,
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/, catalyst/targets/
2021-06-11 3:30 [gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/ Matt Turner
@ 2022-01-30 20:42 ` Matt Turner
0 siblings, 0 replies; 3+ messages in thread
From: Matt Turner @ 2022-01-30 20:42 UTC (permalink / raw
To: gentoo-commits
commit: b3c917f7fa73d11c69b7e55dc7a00bc18a18edc7
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Jun 9 06:17:31 2021 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Jun 10 04:23:54 2021 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b3c917f7
catalyst: Replace snakeoil's locks with fasteners
To no great surprise, the existing locking was broken. For example,
clear_chroot() releases the lock. It is called by unpack(), which is
part of prepare_sequence. The result is that the whole build could be
done without holding the lock.
Just lock around run(). It's not apparent that finer-grained locking
does anything for us.
Bug: https://bugs.gentoo.org/791583
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/clearbase.py | 2 --
catalyst/base/stagebase.py | 10 +++-----
catalyst/lock.py | 58 --------------------------------------------
catalyst/targets/snapshot.py | 6 ++---
4 files changed, 7 insertions(+), 69 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index dcf6c523..6218330e 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -27,12 +27,10 @@ class ClearBase():
self.resume.clear_all(remove=True)
def clear_chroot(self):
- self.chroot_lock.unlock()
log.notice('Clearing the chroot path ...')
clear_dir(self.settings["chroot_path"], mode=0o755)
def remove_chroot(self):
- self.chroot_lock.unlock()
log.notice('Removing the chroot path ...')
clear_dir(self.settings["chroot_path"], mode=0o755, remove=True)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 448d6265..10cffd4c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -8,6 +8,7 @@ import sys
from pathlib import Path
+import fasteners
import libmount
import toml
@@ -25,7 +26,6 @@ from catalyst.support import (CatalystError, file_locate, normpath,
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
-from catalyst.lock import LockDir, LockInUse
from catalyst.fileops import ensure_dirs, clear_dir, clear_path
from catalyst.base.resume import AutoResume
@@ -36,9 +36,6 @@ def run_sequence(sequence):
sys.stdout.flush()
try:
func()
- except LockInUse:
- log.error('Unable to aquire the lock...')
- return False
except Exception:
log.error('Exception running action sequence %s',
func.__name__, exc_info=True)
@@ -478,7 +475,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"""
self.settings["chroot_path"] = normpath(self.settings["storedir"] +
"/tmp/" + self.settings["target_subpath"].rstrip('/'))
- self.chroot_lock = LockDir(self.settings["chroot_path"])
def set_autoresume_path(self):
self.settings["autoresume_path"] = normpath(pjoin(
@@ -1366,8 +1362,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
pass
def run(self):
- self.chroot_lock.write_lock()
+ with fasteners.InterProcessLock(self.settings["chroot_path"] + '.lock'):
+ return self._run()
+ def _run(self):
if "clear-autoresume" in self.settings["options"]:
self.clear_autoresume()
diff --git a/catalyst/lock.py b/catalyst/lock.py
deleted file mode 100644
index e31745b2..00000000
--- a/catalyst/lock.py
+++ /dev/null
@@ -1,58 +0,0 @@
-
-import os
-
-from contextlib import contextmanager
-
-from snakeoil import fileutils
-from snakeoil import osutils
-from catalyst.fileops import ensure_dirs
-
-
-LockInUse = osutils.LockException
-
-class Lock:
- """
- A fnctl-based filesystem lock
- """
- def __init__(self, lockfile):
- fileutils.touch(lockfile, mode=0o664)
- os.chown(lockfile, uid=-1, gid=250)
- self.lock = osutils.FsLock(lockfile)
-
- def read_lock(self):
- self.lock.acquire_read_lock()
-
- def write_lock(self):
- self.lock.acquire_write_lock()
-
- def unlock(self):
- # Releasing a write lock is the same as a read lock.
- self.lock.release_write_lock()
-
-class LockDir(Lock):
- """
- A fnctl-based filesystem lock in a directory
- """
- def __init__(self, lockdir):
- ensure_dirs(lockdir)
- lockfile = os.path.join(lockdir, '.catalyst_lock')
-
- Lock.__init__(self, lockfile)
-
-@contextmanager
-def read_lock(filename):
- lock = Lock(filename)
- lock.read_lock()
- try:
- yield
- finally:
- lock.unlock()
-
-@contextmanager
-def write_lock(filename):
- lock = Lock(filename)
- lock.write_lock()
- try:
- yield
- finally:
- lock.unlock()
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index b494575a..ef68765d 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -5,11 +5,12 @@ Snapshot target
import subprocess
import sys
+import fasteners
+
from pathlib import Path
from catalyst import log
from catalyst.base.targetbase import TargetBase
-from catalyst.lock import write_lock
from catalyst.support import CatalystError, command
class snapshot(TargetBase):
@@ -93,8 +94,7 @@ class snapshot(TargetBase):
log.notice('>>> ' + ' '.join([*git_cmd, '|']))
log.notice(' ' + ' '.join(tar2sqfs_cmd))
- lockfile = self.snapshot.with_suffix('.lock')
- with write_lock(lockfile):
+ with fasteners.InterProcessLock(self.snapshot.with_suffix('.lock')):
git = subprocess.Popen(git_cmd,
stdout=subprocess.PIPE,
stderr=sys.stderr,
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2022-01-30 20:43 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-06-11 3:30 [gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/ Matt Turner
2022-01-30 20:42 ` [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/, catalyst/targets/ Matt Turner
-- strict thread matches above, loose matches on Subject: below --
2020-04-10 21:04 [gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/ Matt Turner
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox