From: Zac Medico <zmedico@gentoo.org>
To: gentoo-portage-dev@lists.gentoo.org
Cc: Zac Medico <zmedico@gentoo.org>
Subject: [gentoo-portage-dev] [PATCH 2/2] binrepos.conf: support fetchcommand customization (bug 661332)
Date: Sun, 6 Sep 2020 23:31:31 -0700 [thread overview]
Message-ID: <20200907063131.395951-3-zmedico@gentoo.org> (raw)
In-Reply-To: <20200907063131.395951-1-zmedico@gentoo.org>
Support customization of fetchcommand and resumecommand in
binrepos.conf, allowing customized authentication mechanisms for
each repository.
Bug: https://bugs.gentoo.org/661332
Signed-off-by: Zac Medico <zmedico@gentoo.org>
---
lib/_emerge/BinpkgFetcher.py | 29 +++++++++++++++++++----------
lib/portage/binrepo/config.py | 2 ++
lib/portage/dbapi/bintree.py | 34 +++++++++++++++++++++++++---------
man/portage.5 | 14 ++++++++++++++
4 files changed, 60 insertions(+), 19 deletions(-)
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 218d4d2ab..9a96bde28 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -96,14 +96,17 @@ class _BinpkgFetcherProcess(SpawnProcess):
# urljoin doesn't work correctly with
# unrecognized protocols like sftp
+ fetchcommand = None
+ resumecommand = None
if bintree._remote_has_index:
- instance_key = bintree.dbapi._instance_key(pkg.cpv)
- rel_uri = bintree._remotepkgs[instance_key].get("PATH")
+ remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(pkg.cpv)]
+ rel_uri = remote_metadata.get("PATH")
if not rel_uri:
rel_uri = pkg.cpv + ".tbz2"
- remote_base_uri = bintree._remotepkgs[
- instance_key]["BASE_URI"]
+ remote_base_uri = remote_metadata["BASE_URI"]
uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
+ fetchcommand = remote_metadata.get('FETCHCOMMAND')
+ resumecommand = remote_metadata.get('RESUMECOMMAND')
else:
uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
"/" + pkg.pf + ".tbz2"
@@ -114,13 +117,19 @@ class _BinpkgFetcherProcess(SpawnProcess):
self._async_wait()
return
- protocol = urllib_parse_urlparse(uri)[0]
- fcmd_prefix = "FETCHCOMMAND"
+ fcmd = None
if resume:
- fcmd_prefix = "RESUMECOMMAND"
- fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
- if not fcmd:
- fcmd = settings.get(fcmd_prefix)
+ fcmd = resumecommand
+ else:
+ fcmd = fetchcommand
+ if fcmd is None:
+ protocol = urllib_parse_urlparse(uri)[0]
+ fcmd_prefix = "FETCHCOMMAND"
+ if resume:
+ fcmd_prefix = "RESUMECOMMAND"
+ fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
+ if not fcmd:
+ fcmd = settings.get(fcmd_prefix)
fcmd_vars = {
"DISTDIR" : os.path.dirname(pkg_path),
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
index aa3ff7a77..0c01f6fae 100644
--- a/lib/portage/binrepo/config.py
+++ b/lib/portage/binrepo/config.py
@@ -15,7 +15,9 @@ class BinRepoConfig:
__slots__ = (
'name',
'name_fallback',
+ 'fetchcommand',
'priority',
+ 'resumecommand',
'sync_uri',
)
def __init__(self, opts):
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index a96183561..ebc6765b6 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -382,10 +382,10 @@ class binarytree:
self._pkgindex_keys.update(["CPV", "SIZE"])
self._pkgindex_aux_keys = \
["BASE_URI", "BDEPEND", "BUILD_ID", "BUILD_TIME", "CHOST",
- "DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI",
+ "DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI", "FETCHCOMMAND",
"IUSE", "KEYWORDS", "LICENSE", "PDEPEND",
"PKGINDEX_URI", "PROPERTIES", "PROVIDES",
- "RDEPEND", "repository", "REQUIRES", "RESTRICT",
+ "RDEPEND", "repository", "REQUIRES", "RESTRICT", "RESUMECOMMAND",
"SIZE", "SLOT", "USE"]
self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
self._pkgindex_use_evaluated_keys = \
@@ -972,7 +972,7 @@ class binarytree:
# Don't use urlopen for https, unless
# PEP 476 is supported (bug #469888).
- if parsed_url.scheme not in ('https',) or _have_pep_476():
+ if repo.fetchcommand is None and (parsed_url.scheme not in ('https',) or _have_pep_476()):
try:
f = _urlopen(url, if_modified_since=local_timestamp)
if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
@@ -997,7 +997,7 @@ class binarytree:
path = parsed_url.path.rstrip("/") + "/Packages"
- if parsed_url.scheme == 'ssh':
+ if repo.fetchcommand is None and parsed_url.scheme == 'ssh':
# Use a pipe so that we can terminate the download
# early if we detect that the TIMESTAMP header
# matches that of the cached Packages file.
@@ -1016,12 +1016,15 @@ class binarytree:
stdout=subprocess.PIPE)
f = proc.stdout
else:
- setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
- fcmd = self.settings.get(setting)
- if not fcmd:
- fcmd = self.settings.get('FETCHCOMMAND')
+ if repo.fetchcommand is None:
+ setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
+ fcmd = self.settings.get(setting)
if not fcmd:
- raise EnvironmentError("FETCHCOMMAND is unset")
+ fcmd = self.settings.get('FETCHCOMMAND')
+ if not fcmd:
+ raise EnvironmentError("FETCHCOMMAND is unset")
+ else:
+ fcmd = repo.fetchcommand
fd, tmp_filename = tempfile.mkstemp()
tmp_dirname, tmp_basename = os.path.split(tmp_filename)
@@ -1135,6 +1138,19 @@ class binarytree:
d["CPV"] = cpv
d["BASE_URI"] = remote_base_uri
d["PKGINDEX_URI"] = url
+ # FETCHCOMMAND and RESUMECOMMAND may be specified
+ # by binrepos.conf, and otherwise ensure that they
+ # do not propagate from the Packages index since
+ # it may be unsafe to execute remotely specified
+ # commands.
+ if repo.fetchcommand is None:
+ d.pop('FETCHCOMMAND', None)
+ else:
+ d['FETCHCOMMAND'] = repo.fetchcommand
+ if repo.resumecommand is None:
+ d.pop('RESUMECOMMAND', None)
+ else:
+ d['RESUMECOMMAND'] = repo.resumecommand
self._remotepkgs[self.dbapi._instance_key(cpv)] = d
self.dbapi.cpv_inject(cpv)
diff --git a/man/portage.5 b/man/portage.5
index 890a22adb..9c88bc3dc 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -634,6 +634,20 @@ is intended to be used as a replacement for the \fBmake.conf\fR(5)
\- attributes are specified in "${attribute} = ${value}" format
.fi
+.RS
+.I Attributes supported in DEFAULT section:
+.RS
+.TP
+.B fetchcommand
+Specifies a \fBFETCHCOMMAND\fR used to fetch files from a repository,
+overriding the value from \fBmake.conf\fR(5).
+.TP
+.B resumecommand
+Specifies a \fBRESUMECOMMAND\fR used to fetch files from a repository,
+overriding the value from \fBmake.conf\fR(5).
+.RE
+.RE
+
.RS
.I Attributes supported in sections of repositories:
.RS
--
2.25.3
next prev parent reply other threads:[~2020-09-07 6:35 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-09-07 6:31 [gentoo-portage-dev] [PATCH 0/2] Add binrepos.conf to support fetchcommand customization (bug 661332) Zac Medico
2020-09-07 6:31 ` [gentoo-portage-dev] [PATCH 1/2] Add binrepos.conf to replace PORTAGE_BINHOST (bug 668334) Zac Medico
2020-09-07 6:31 ` Zac Medico [this message]
2020-09-07 6:52 ` [gentoo-portage-dev] Re: [PATCH 0/2] Add binrepos.conf to support fetchcommand customization (bug 661332) Zac Medico
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200907063131.395951-3-zmedico@gentoo.org \
--to=zmedico@gentoo.org \
--cc=gentoo-portage-dev@lists.gentoo.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox