From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 888A61382C5 for ; Mon, 18 Jan 2021 19:53:49 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id B9C5AE07D7; Mon, 18 Jan 2021 19:53:48 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 8B61DE07D7 for ; Mon, 18 Jan 2021 19:53:48 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id EA007340E3A for ; Mon, 18 Jan 2021 19:53:46 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 431D992 for ; Mon, 18 Jan 2021 19:53:45 +0000 (UTC) From: "Matt Turner" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Matt Turner" Message-ID: <1605371697.37a386056f77f7cc8f1c2bdfe680b13bd806b4e6.mattst88@gentoo> Subject: [gentoo-commits] proj/catalyst:pending/mattst88 commit in: catalyst/, catalyst/base/ X-VCS-Repository: proj/catalyst X-VCS-Files: catalyst/base/stagebase.py catalyst/defaults.py catalyst/support.py X-VCS-Directories: catalyst/ catalyst/base/ X-VCS-Committer: mattst88 X-VCS-Committer-Name: Matt Turner X-VCS-Revision: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6 X-VCS-Branch: pending/mattst88 Date: Mon, 18 Jan 2021 19:53:45 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 23181238-a962-4078-ba8a-237c772571cf X-Archives-Hash: 262e2084e0667973df729ebc2e98e9f0 commit: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6 Author: Felix Bier rohde-schwarz com> AuthorDate: Tue Nov 10 01:03:03 2020 +0000 Commit: Matt Turner gentoo org> CommitDate: Sat Nov 14 16:34:57 2020 +0000 URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=37a38605 Move from PORTDIR_OVERLAY to repos.conf This commit fixes the following issues: * The PORTDIR_OVERLAY variable has been deprecated by Gentoo. With this commit, the variable is no longer written to the generated make.conf. Instead, a config file /etc/portage/repos.conf/.conf is generated for each overlay. The repo name is read from the overlay using the portage API. Internally, portage parses metadata/layout.conf and profiles/repo_name to obtain the name. References: https://wiki.gentoo.org/wiki//etc/portage/make.conf https://wiki.gentoo.org/wiki//etc/portage/repos.conf * All overlays were copied into the same target directory. If the same file name occurred in multiple overlays, the last overlay would overwrite all previous files with this name. In particular, only the metadata/layout.conf of the last overlay was retained, so it was not possible to reference the other overlays e.g. via the masters entry in the layout.conf or the portage-2 syntax for specifying a parent profile from another overlay. Also, this created problems when the overlays contained ebuilds for the same package, but with differing versions, because after copying, the target directory contained both versions of the ebuild but only the manifest file of the last overlay. With this commit, each overlay is copied into a separate sub-directory, e.g. /var/db/repos/. This directory is referenced via the location entry in the generated /etc/portage/repos.conf/.conf. Signed-off-by: Felix Bier rohde-schwarz.com> Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 84 ++++++++++++++++++++++++++++++++++------------ catalyst/defaults.py | 2 +- catalyst/support.py | 18 ++++++++++ 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 21cf96a0..fe79b55a 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -1,4 +1,5 @@ +import configparser import copy import os import platform @@ -19,8 +20,8 @@ from catalyst import log from catalyst.context import namespace from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN) from catalyst.support import (CatalystError, file_locate, normpath, - cmd, read_makeconf, ismount, file_check, - sanitize_name) + cmd, read_makeconf, get_repo_name, ismount, + file_check, sanitize_name) from catalyst.base.targetbase import TargetBase from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase @@ -786,17 +787,55 @@ class StageBase(TargetBase, ClearBase, GenBase): env=self.env) self.resume.enable("setup_confdir") + def to_chroot(self, path): + """ Prepend chroot path to the given path. """ + + chroot = Path(self.settings['chroot_path']) + return chroot / path.relative_to(path.anchor) + + def get_repo_conf_path(self, repo_name): + """ Construct repo conf path: {repos_conf}/{name}.conf """ + return Path(self.settings['repos_conf'], repo_name + ".conf") + + def get_repo_location(self, repo_name): + """ Construct overlay repo path: {repo_basedir}/{name} """ + return Path(self.settings['repo_basedir'], repo_name) + + def write_repo_conf(self, repo_name, config): + """ Write ConfigParser to {chroot}/{repos_conf}/{name}.conf """ + + repo_conf = self.get_repo_conf_path(repo_name) + + repo_conf_chroot = self.to_chroot(repo_conf) + repo_conf_chroot.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + + log.info(f'Creating repo config {repo_conf_chroot}.') + + try: + with open(repo_conf_chroot, 'w') as f: + config.write(f) + except OSError as e: + raise CatalystError(f'Could not write {repo_conf_chroot}: {e}') from e + def portage_overlay(self): - """ We copy the contents of our overlays to /usr/local/portage """ + """ We copy the contents of our repos to get_repo_location(repo_name) """ if "portage_overlay" in self.settings: for x in self.settings["portage_overlay"]: if os.path.exists(x): - log.info('Copying overlay dir %s', x) - ensure_dirs( - self.settings['chroot_path'] + self.settings['local_overlay']) - cmd("cp -a " + x + "/* " + self.settings["chroot_path"] + - self.settings["local_overlay"], - env=self.env) + name = get_repo_name(x) + + location = self.get_repo_location(name) + config = configparser.ConfigParser() + config[name] = {'location': location} + self.write_repo_conf(name, config) + + location_chroot = self.to_chroot(location) + location_chroot.mkdir(mode=0o755, parents=True, exist_ok=True) + + log.info(f'Copying overlay dir {x} to {location_chroot}') + cmd(f'cp -a {x}/* {location_chroot}', env=self.env) + else: + log.warning(f'Skipping missing overlay {x}.') def root_overlay(self): """ Copy over the root_overlay """ @@ -852,8 +891,8 @@ class StageBase(TargetBase, ClearBase, GenBase): cxt = libmount.Context(source=source, target=target, fstype=fstype, options=options) cxt.mount() - except OSError as e: - raise CatalystError(f"Couldn't mount: {source}, {e.strerror}") + except Exception as e: + raise CatalystError(f"Couldn't mount: {source}, {e}") def chroot_setup(self): self.makeconf = read_makeconf(normpath(self.settings["chroot_path"] + @@ -1018,12 +1057,6 @@ class StageBase(TargetBase, ClearBase, GenBase): varname = x.split('_')[1].upper() myf.write(f'{varname}="{self.settings[x]}"\n') - if setup: - # Setup the portage overlay - if "portage_overlay" in self.settings: - myf.write('PORTDIR_OVERLAY="%s"\n' % - self.settings["local_overlay"]) - # Set default locale for system responses. #478382 myf.write( '\n' @@ -1097,11 +1130,18 @@ class StageBase(TargetBase, ClearBase, GenBase): log.warning("You've been hacking. Clearing target patches: %s", target) clear_path(target) - # Remove our overlay - overlay = normpath( - self.settings["chroot_path"] + self.settings["local_overlay"]) - if os.path.exists(overlay): - clear_path(overlay) + # Remove our overlays + if "portage_overlay" in self.settings: + for repo_path in self.settings["portage_overlay"]: + repo_name = get_repo_name(repo_path) + + repo_conf = self.get_repo_conf_path(repo_name) + chroot_repo_conf = self.to_chroot(repo_conf) + chroot_repo_conf.unlink() + + location = self.get_repo_location(repo_name) + chroot_location = self.to_chroot(location) + clear_path(str(chroot_location)) if "sticky-config" not in self.settings["options"]: # re-write the make.conf to be sure it is clean diff --git a/catalyst/defaults.py b/catalyst/defaults.py index 0f399b56..3f12b8d5 100644 --- a/catalyst/defaults.py +++ b/catalyst/defaults.py @@ -38,9 +38,9 @@ confdefaults = { "distdir": portage.settings['DISTDIR'], "icecream": "/var/cache/icecream", 'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'], - "local_overlay": "/var/db/repos/local", "port_conf": "/etc/portage", "make_conf": "%(port_conf)s/make.conf", + "repos_conf": "%(port_conf)s/repos.conf", "options": set(), "pkgdir": "/var/cache/binpkgs", "port_tmpdir": "/var/tmp/portage", diff --git a/catalyst/support.py b/catalyst/support.py index ddbd9ab9..f3a865a7 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -10,6 +10,8 @@ from subprocess import Popen import libmount +from portage.repository.config import RepoConfig + from catalyst import log BASH_BINARY = "/bin/bash" @@ -182,6 +184,22 @@ def read_makeconf(mymakeconffile): return makeconf +def get_repo_name(repo_path): + """ Get the name of the repo at the given repo_path. + + References: + https://wiki.gentoo.org/wiki/Repository_format/profiles/repo_name + https://wiki.gentoo.org/wiki/Repository_format/metadata/layout.conf#repo-name + """ + + repo_config = RepoConfig(None, {"location": repo_path}) + + if repo_config.missing_repo_name: + raise CatalystError(f"Missing name in repository {repo_path}") + + return repo_config.name + + def ismount(path): """Like os.path.ismount, but also support bind mounts""" path = Path(path) From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id B1F1313835A for ; Sat, 19 Dec 2020 19:56:07 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id B34452BC01C; Sat, 19 Dec 2020 19:56:06 +0000 (UTC) Received: from smtp.gentoo.org (mail.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 7C1A52BC01B for ; Sat, 19 Dec 2020 19:56:06 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id EB988340F42 for ; Sat, 19 Dec 2020 19:56:04 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 7FCC43B7 for ; Sat, 19 Dec 2020 19:56:02 +0000 (UTC) From: "Matt Turner" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Matt Turner" Message-ID: <1605371697.37a386056f77f7cc8f1c2bdfe680b13bd806b4e6.mattst88@gentoo> Subject: [gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/, catalyst/base/ X-VCS-Repository: proj/catalyst X-VCS-Files: catalyst/base/stagebase.py catalyst/defaults.py catalyst/support.py X-VCS-Directories: catalyst/ catalyst/base/ X-VCS-Committer: mattst88 X-VCS-Committer-Name: Matt Turner X-VCS-Revision: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6 X-VCS-Branch: wip/mattst88 Date: Sat, 19 Dec 2020 19:56:02 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 6178bfb8-4be0-48e5-a4a6-d58c61311cdd X-Archives-Hash: 191d6a6348798b96ab2efcb471c84f2a Message-ID: <20201219195602.IO3RjeFN7ifXLZWZpkdN8Dpz7_kYadL5r39hK35sfMc@z> commit: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6 Author: Felix Bier rohde-schwarz com> AuthorDate: Tue Nov 10 01:03:03 2020 +0000 Commit: Matt Turner gentoo org> CommitDate: Sat Nov 14 16:34:57 2020 +0000 URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=37a38605 Move from PORTDIR_OVERLAY to repos.conf This commit fixes the following issues: * The PORTDIR_OVERLAY variable has been deprecated by Gentoo. With this commit, the variable is no longer written to the generated make.conf. Instead, a config file /etc/portage/repos.conf/.conf is generated for each overlay. The repo name is read from the overlay using the portage API. Internally, portage parses metadata/layout.conf and profiles/repo_name to obtain the name. References: https://wiki.gentoo.org/wiki//etc/portage/make.conf https://wiki.gentoo.org/wiki//etc/portage/repos.conf * All overlays were copied into the same target directory. If the same file name occurred in multiple overlays, the last overlay would overwrite all previous files with this name. In particular, only the metadata/layout.conf of the last overlay was retained, so it was not possible to reference the other overlays e.g. via the masters entry in the layout.conf or the portage-2 syntax for specifying a parent profile from another overlay. Also, this created problems when the overlays contained ebuilds for the same package, but with differing versions, because after copying, the target directory contained both versions of the ebuild but only the manifest file of the last overlay. With this commit, each overlay is copied into a separate sub-directory, e.g. /var/db/repos/. This directory is referenced via the location entry in the generated /etc/portage/repos.conf/.conf. Signed-off-by: Felix Bier rohde-schwarz.com> Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 84 ++++++++++++++++++++++++++++++++++------------ catalyst/defaults.py | 2 +- catalyst/support.py | 18 ++++++++++ 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 21cf96a0..fe79b55a 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -1,4 +1,5 @@ +import configparser import copy import os import platform @@ -19,8 +20,8 @@ from catalyst import log from catalyst.context import namespace from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN) from catalyst.support import (CatalystError, file_locate, normpath, - cmd, read_makeconf, ismount, file_check, - sanitize_name) + cmd, read_makeconf, get_repo_name, ismount, + file_check, sanitize_name) from catalyst.base.targetbase import TargetBase from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase @@ -786,17 +787,55 @@ class StageBase(TargetBase, ClearBase, GenBase): env=self.env) self.resume.enable("setup_confdir") + def to_chroot(self, path): + """ Prepend chroot path to the given path. """ + + chroot = Path(self.settings['chroot_path']) + return chroot / path.relative_to(path.anchor) + + def get_repo_conf_path(self, repo_name): + """ Construct repo conf path: {repos_conf}/{name}.conf """ + return Path(self.settings['repos_conf'], repo_name + ".conf") + + def get_repo_location(self, repo_name): + """ Construct overlay repo path: {repo_basedir}/{name} """ + return Path(self.settings['repo_basedir'], repo_name) + + def write_repo_conf(self, repo_name, config): + """ Write ConfigParser to {chroot}/{repos_conf}/{name}.conf """ + + repo_conf = self.get_repo_conf_path(repo_name) + + repo_conf_chroot = self.to_chroot(repo_conf) + repo_conf_chroot.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + + log.info(f'Creating repo config {repo_conf_chroot}.') + + try: + with open(repo_conf_chroot, 'w') as f: + config.write(f) + except OSError as e: + raise CatalystError(f'Could not write {repo_conf_chroot}: {e}') from e + def portage_overlay(self): - """ We copy the contents of our overlays to /usr/local/portage """ + """ We copy the contents of our repos to get_repo_location(repo_name) """ if "portage_overlay" in self.settings: for x in self.settings["portage_overlay"]: if os.path.exists(x): - log.info('Copying overlay dir %s', x) - ensure_dirs( - self.settings['chroot_path'] + self.settings['local_overlay']) - cmd("cp -a " + x + "/* " + self.settings["chroot_path"] + - self.settings["local_overlay"], - env=self.env) + name = get_repo_name(x) + + location = self.get_repo_location(name) + config = configparser.ConfigParser() + config[name] = {'location': location} + self.write_repo_conf(name, config) + + location_chroot = self.to_chroot(location) + location_chroot.mkdir(mode=0o755, parents=True, exist_ok=True) + + log.info(f'Copying overlay dir {x} to {location_chroot}') + cmd(f'cp -a {x}/* {location_chroot}', env=self.env) + else: + log.warning(f'Skipping missing overlay {x}.') def root_overlay(self): """ Copy over the root_overlay """ @@ -852,8 +891,8 @@ class StageBase(TargetBase, ClearBase, GenBase): cxt = libmount.Context(source=source, target=target, fstype=fstype, options=options) cxt.mount() - except OSError as e: - raise CatalystError(f"Couldn't mount: {source}, {e.strerror}") + except Exception as e: + raise CatalystError(f"Couldn't mount: {source}, {e}") def chroot_setup(self): self.makeconf = read_makeconf(normpath(self.settings["chroot_path"] + @@ -1018,12 +1057,6 @@ class StageBase(TargetBase, ClearBase, GenBase): varname = x.split('_')[1].upper() myf.write(f'{varname}="{self.settings[x]}"\n') - if setup: - # Setup the portage overlay - if "portage_overlay" in self.settings: - myf.write('PORTDIR_OVERLAY="%s"\n' % - self.settings["local_overlay"]) - # Set default locale for system responses. #478382 myf.write( '\n' @@ -1097,11 +1130,18 @@ class StageBase(TargetBase, ClearBase, GenBase): log.warning("You've been hacking. Clearing target patches: %s", target) clear_path(target) - # Remove our overlay - overlay = normpath( - self.settings["chroot_path"] + self.settings["local_overlay"]) - if os.path.exists(overlay): - clear_path(overlay) + # Remove our overlays + if "portage_overlay" in self.settings: + for repo_path in self.settings["portage_overlay"]: + repo_name = get_repo_name(repo_path) + + repo_conf = self.get_repo_conf_path(repo_name) + chroot_repo_conf = self.to_chroot(repo_conf) + chroot_repo_conf.unlink() + + location = self.get_repo_location(repo_name) + chroot_location = self.to_chroot(location) + clear_path(str(chroot_location)) if "sticky-config" not in self.settings["options"]: # re-write the make.conf to be sure it is clean diff --git a/catalyst/defaults.py b/catalyst/defaults.py index 0f399b56..3f12b8d5 100644 --- a/catalyst/defaults.py +++ b/catalyst/defaults.py @@ -38,9 +38,9 @@ confdefaults = { "distdir": portage.settings['DISTDIR'], "icecream": "/var/cache/icecream", 'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'], - "local_overlay": "/var/db/repos/local", "port_conf": "/etc/portage", "make_conf": "%(port_conf)s/make.conf", + "repos_conf": "%(port_conf)s/repos.conf", "options": set(), "pkgdir": "/var/cache/binpkgs", "port_tmpdir": "/var/tmp/portage", diff --git a/catalyst/support.py b/catalyst/support.py index ddbd9ab9..f3a865a7 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -10,6 +10,8 @@ from subprocess import Popen import libmount +from portage.repository.config import RepoConfig + from catalyst import log BASH_BINARY = "/bin/bash" @@ -182,6 +184,22 @@ def read_makeconf(mymakeconffile): return makeconf +def get_repo_name(repo_path): + """ Get the name of the repo at the given repo_path. + + References: + https://wiki.gentoo.org/wiki/Repository_format/profiles/repo_name + https://wiki.gentoo.org/wiki/Repository_format/metadata/layout.conf#repo-name + """ + + repo_config = RepoConfig(None, {"location": repo_path}) + + if repo_config.missing_repo_name: + raise CatalystError(f"Missing name in repository {repo_path}") + + return repo_config.name + + def ismount(path): """Like os.path.ismount, but also support bind mounts""" path = Path(path) From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 2095013835B for ; Sat, 14 Nov 2020 16:37:34 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 1A391E05C1; Sat, 14 Nov 2020 16:37:33 +0000 (UTC) Received: from smtp.gentoo.org (mail.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id DA9E3E03EC for ; Sat, 14 Nov 2020 16:37:32 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 7DB3F34112B for ; Sat, 14 Nov 2020 16:37:31 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id BCE4744F for ; Sat, 14 Nov 2020 16:37:29 +0000 (UTC) From: "Matt Turner" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Matt Turner" Message-ID: <1605371697.37a386056f77f7cc8f1c2bdfe680b13bd806b4e6.mattst88@gentoo> Subject: [gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/ X-VCS-Repository: proj/catalyst X-VCS-Files: catalyst/base/stagebase.py catalyst/defaults.py catalyst/support.py X-VCS-Directories: catalyst/base/ catalyst/ X-VCS-Committer: mattst88 X-VCS-Committer-Name: Matt Turner X-VCS-Revision: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6 X-VCS-Branch: master Date: Sat, 14 Nov 2020 16:37:29 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: edb2f044-b8d5-4d44-85f9-a14685104e07 X-Archives-Hash: 449ed2bf304d2b6b1ecaa1e4a42ebfd0 Message-ID: <20201114163729.lf7Ya_f078ne1-VZX-wZ458qqvGUIvRbOnBX54MVg-A@z> commit: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6 Author: Felix Bier rohde-schwarz com> AuthorDate: Tue Nov 10 01:03:03 2020 +0000 Commit: Matt Turner gentoo org> CommitDate: Sat Nov 14 16:34:57 2020 +0000 URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=37a38605 Move from PORTDIR_OVERLAY to repos.conf This commit fixes the following issues: * The PORTDIR_OVERLAY variable has been deprecated by Gentoo. With this commit, the variable is no longer written to the generated make.conf. Instead, a config file /etc/portage/repos.conf/.conf is generated for each overlay. The repo name is read from the overlay using the portage API. Internally, portage parses metadata/layout.conf and profiles/repo_name to obtain the name. References: https://wiki.gentoo.org/wiki//etc/portage/make.conf https://wiki.gentoo.org/wiki//etc/portage/repos.conf * All overlays were copied into the same target directory. If the same file name occurred in multiple overlays, the last overlay would overwrite all previous files with this name. In particular, only the metadata/layout.conf of the last overlay was retained, so it was not possible to reference the other overlays e.g. via the masters entry in the layout.conf or the portage-2 syntax for specifying a parent profile from another overlay. Also, this created problems when the overlays contained ebuilds for the same package, but with differing versions, because after copying, the target directory contained both versions of the ebuild but only the manifest file of the last overlay. With this commit, each overlay is copied into a separate sub-directory, e.g. /var/db/repos/. This directory is referenced via the location entry in the generated /etc/portage/repos.conf/.conf. Signed-off-by: Felix Bier rohde-schwarz.com> Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 84 ++++++++++++++++++++++++++++++++++------------ catalyst/defaults.py | 2 +- catalyst/support.py | 18 ++++++++++ 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 21cf96a0..fe79b55a 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -1,4 +1,5 @@ +import configparser import copy import os import platform @@ -19,8 +20,8 @@ from catalyst import log from catalyst.context import namespace from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN) from catalyst.support import (CatalystError, file_locate, normpath, - cmd, read_makeconf, ismount, file_check, - sanitize_name) + cmd, read_makeconf, get_repo_name, ismount, + file_check, sanitize_name) from catalyst.base.targetbase import TargetBase from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase @@ -786,17 +787,55 @@ class StageBase(TargetBase, ClearBase, GenBase): env=self.env) self.resume.enable("setup_confdir") + def to_chroot(self, path): + """ Prepend chroot path to the given path. """ + + chroot = Path(self.settings['chroot_path']) + return chroot / path.relative_to(path.anchor) + + def get_repo_conf_path(self, repo_name): + """ Construct repo conf path: {repos_conf}/{name}.conf """ + return Path(self.settings['repos_conf'], repo_name + ".conf") + + def get_repo_location(self, repo_name): + """ Construct overlay repo path: {repo_basedir}/{name} """ + return Path(self.settings['repo_basedir'], repo_name) + + def write_repo_conf(self, repo_name, config): + """ Write ConfigParser to {chroot}/{repos_conf}/{name}.conf """ + + repo_conf = self.get_repo_conf_path(repo_name) + + repo_conf_chroot = self.to_chroot(repo_conf) + repo_conf_chroot.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + + log.info(f'Creating repo config {repo_conf_chroot}.') + + try: + with open(repo_conf_chroot, 'w') as f: + config.write(f) + except OSError as e: + raise CatalystError(f'Could not write {repo_conf_chroot}: {e}') from e + def portage_overlay(self): - """ We copy the contents of our overlays to /usr/local/portage """ + """ We copy the contents of our repos to get_repo_location(repo_name) """ if "portage_overlay" in self.settings: for x in self.settings["portage_overlay"]: if os.path.exists(x): - log.info('Copying overlay dir %s', x) - ensure_dirs( - self.settings['chroot_path'] + self.settings['local_overlay']) - cmd("cp -a " + x + "/* " + self.settings["chroot_path"] + - self.settings["local_overlay"], - env=self.env) + name = get_repo_name(x) + + location = self.get_repo_location(name) + config = configparser.ConfigParser() + config[name] = {'location': location} + self.write_repo_conf(name, config) + + location_chroot = self.to_chroot(location) + location_chroot.mkdir(mode=0o755, parents=True, exist_ok=True) + + log.info(f'Copying overlay dir {x} to {location_chroot}') + cmd(f'cp -a {x}/* {location_chroot}', env=self.env) + else: + log.warning(f'Skipping missing overlay {x}.') def root_overlay(self): """ Copy over the root_overlay """ @@ -852,8 +891,8 @@ class StageBase(TargetBase, ClearBase, GenBase): cxt = libmount.Context(source=source, target=target, fstype=fstype, options=options) cxt.mount() - except OSError as e: - raise CatalystError(f"Couldn't mount: {source}, {e.strerror}") + except Exception as e: + raise CatalystError(f"Couldn't mount: {source}, {e}") def chroot_setup(self): self.makeconf = read_makeconf(normpath(self.settings["chroot_path"] + @@ -1018,12 +1057,6 @@ class StageBase(TargetBase, ClearBase, GenBase): varname = x.split('_')[1].upper() myf.write(f'{varname}="{self.settings[x]}"\n') - if setup: - # Setup the portage overlay - if "portage_overlay" in self.settings: - myf.write('PORTDIR_OVERLAY="%s"\n' % - self.settings["local_overlay"]) - # Set default locale for system responses. #478382 myf.write( '\n' @@ -1097,11 +1130,18 @@ class StageBase(TargetBase, ClearBase, GenBase): log.warning("You've been hacking. Clearing target patches: %s", target) clear_path(target) - # Remove our overlay - overlay = normpath( - self.settings["chroot_path"] + self.settings["local_overlay"]) - if os.path.exists(overlay): - clear_path(overlay) + # Remove our overlays + if "portage_overlay" in self.settings: + for repo_path in self.settings["portage_overlay"]: + repo_name = get_repo_name(repo_path) + + repo_conf = self.get_repo_conf_path(repo_name) + chroot_repo_conf = self.to_chroot(repo_conf) + chroot_repo_conf.unlink() + + location = self.get_repo_location(repo_name) + chroot_location = self.to_chroot(location) + clear_path(str(chroot_location)) if "sticky-config" not in self.settings["options"]: # re-write the make.conf to be sure it is clean diff --git a/catalyst/defaults.py b/catalyst/defaults.py index 0f399b56..3f12b8d5 100644 --- a/catalyst/defaults.py +++ b/catalyst/defaults.py @@ -38,9 +38,9 @@ confdefaults = { "distdir": portage.settings['DISTDIR'], "icecream": "/var/cache/icecream", 'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'], - "local_overlay": "/var/db/repos/local", "port_conf": "/etc/portage", "make_conf": "%(port_conf)s/make.conf", + "repos_conf": "%(port_conf)s/repos.conf", "options": set(), "pkgdir": "/var/cache/binpkgs", "port_tmpdir": "/var/tmp/portage", diff --git a/catalyst/support.py b/catalyst/support.py index ddbd9ab9..f3a865a7 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -10,6 +10,8 @@ from subprocess import Popen import libmount +from portage.repository.config import RepoConfig + from catalyst import log BASH_BINARY = "/bin/bash" @@ -182,6 +184,22 @@ def read_makeconf(mymakeconffile): return makeconf +def get_repo_name(repo_path): + """ Get the name of the repo at the given repo_path. + + References: + https://wiki.gentoo.org/wiki/Repository_format/profiles/repo_name + https://wiki.gentoo.org/wiki/Repository_format/metadata/layout.conf#repo-name + """ + + repo_config = RepoConfig(None, {"location": repo_path}) + + if repo_config.missing_repo_name: + raise CatalystError(f"Missing name in repository {repo_path}") + + return repo_config.name + + def ismount(path): """Like os.path.ismount, but also support bind mounts""" path = Path(path)