* [gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
@ 2019-10-20 8:34 Zac Medico
0 siblings, 0 replies; 4+ messages in thread
From: Zac Medico @ 2019-10-20 8:34 UTC (permalink / raw
To: gentoo-commits
commit: 52bc75a60b84d709712e91c68782f2f207bfce4e
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 00:55:09 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 20 08:33:09 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=52bc75a6
fetch: add force parameter (bug 697566)
Add a force parameter which forces download even when a file already
exists in DISTDIR (and no digests are available to verify it). This
avoids the need to remove the existing file in advance, which makes
it possible to atomically replace the file and avoid interference
with concurrent processes. This is useful when using FETCHCOMMAND to
fetch a mirror's layout.conf file, for the purposes of bug 697566.
Bug: https://bugs.gentoo.org/697566
Reviewed-by: Michał Górny <mgorny <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/package/ebuild/fetch.py | 22 +++++++++++++++----
lib/portage/tests/ebuild/test_fetch.py | 40 +++++++++++++++++++++++++++++++---
2 files changed, 55 insertions(+), 7 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
index 76e4636c2..05de12740 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -432,7 +432,7 @@ def get_mirror_url(mirror_url, filename, cache_path=None):
def fetch(myuris, mysettings, listonly=0, fetchonly=0,
locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None,
- allow_missing_digests=True):
+ allow_missing_digests=True, force=False):
"""
Fetch files to DISTDIR and also verify digests if they are available.
@@ -455,10 +455,23 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
@param allow_missing_digests: Enable fetch even if there are no digests
available for verification.
@type allow_missing_digests: bool
+ @param force: Force download, even when a file already exists in
+ DISTDIR. This is most useful when there are no digests available,
+ since otherwise download will be automatically forced if the
+ existing file does not match the available digests. Also, this
+ avoids the need to remove the existing file in advance, which
+ makes it possible to atomically replace the file and avoid
+ interference with concurrent processes.
+ @type force: bool
@rtype: int
@return: 1 if successful, 0 otherwise.
"""
+ if force and digests:
+ # Since the force parameter can trigger unnecessary fetch when the
+ # digests match, do not allow force=True when digests are provided.
+ raise PortageException(_('fetch: force=True is not allowed when digests are provided'))
+
if not myuris:
return 1
@@ -878,7 +891,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
match, mystat = _check_distfile(
myfile_path, pruned_digests, eout, hash_filter=hash_filter)
- if match:
+ if match and not force:
# Skip permission adjustment for symlinks, since we don't
# want to modify anything outside of the primary DISTDIR,
# and symlinks typically point to PORTAGE_RO_DISTDIRS.
@@ -1042,10 +1055,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
os.unlink(download_path)
except EnvironmentError:
pass
- elif myfile not in mydigests:
+ elif not orig_digests:
# We don't have a digest, but the file exists. We must
# assume that it is fully downloaded.
- continue
+ if not force:
+ continue
else:
if (mydigests[myfile].get("size") is not None
and mystat.st_size < mydigests[myfile]["size"]
diff --git a/lib/portage/tests/ebuild/test_fetch.py b/lib/portage/tests/ebuild/test_fetch.py
index f50fea0dd..538fb1754 100644
--- a/lib/portage/tests/ebuild/test_fetch.py
+++ b/lib/portage/tests/ebuild/test_fetch.py
@@ -119,10 +119,44 @@ class EbuildFetchTestCase(TestCase):
with open(foo_path, 'rb') as f:
self.assertNotEqual(f.read(), distfiles['foo'])
- # Remove the stale file in order to forcefully update it.
- os.unlink(foo_path)
+ # Use force=True to update the stale file.
+ self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True)))
- self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False)))
+ with open(foo_path, 'rb') as f:
+ self.assertEqual(f.read(), distfiles['foo'])
+
+ # Test force=True with FEATURES=skiprocheck, using read-only DISTDIR.
+ # FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that
+ # FETCHCOMMAND must perform atomic rename itself due to read-only
+ # DISTDIR.
+ with open(foo_path, 'wb') as f:
+ f.write(b'stale content\n')
+ orig_fetchcommand = settings['FETCHCOMMAND']
+ orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
+ temp_fetchcommand = os.path.join(eubin, 'fetchcommand')
+ with open(temp_fetchcommand, 'w') as f:
+ f.write("""
+ set -e
+ URI=$1
+ DISTDIR=$2
+ FILE=$3
+ trap 'chmod a-w "${DISTDIR}"' EXIT
+ chmod ug+w "${DISTDIR}"
+ %s
+ mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}"
+ """ % orig_fetchcommand.replace('${FILE}', '${FILE}.__download__'))
+ settings['FETCHCOMMAND'] = '"%s" "%s" "${URI}" "${DISTDIR}" "${FILE}"' % (BASH_BINARY, temp_fetchcommand)
+ settings.features.add('skiprocheck')
+ settings.features.remove('distlocks')
+ os.chmod(settings['DISTDIR'], 0o555)
+ try:
+ self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True)))
+ finally:
+ settings['FETCHCOMMAND'] = orig_fetchcommand
+ os.chmod(settings['DISTDIR'], orig_distdir_mode)
+ settings.features.remove('skiprocheck')
+ settings.features.add('distlocks')
+ os.unlink(temp_fetchcommand)
with open(foo_path, 'rb') as f:
self.assertEqual(f.read(), distfiles['foo'])
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
@ 2021-02-22 12:18 Zac Medico
0 siblings, 0 replies; 4+ messages in thread
From: Zac Medico @ 2021-02-22 12:18 UTC (permalink / raw
To: gentoo-commits
commit: a4f06ab3cf7339100b2af2146ae90cbba8bac371
Author: Daniel Robbins <drobbins <AT> funtoo <DOT> org>
AuthorDate: Sat Feb 20 23:11:46 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 11:48:41 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=a4f06ab3
Add content-hash distfiles layout (bug 756778)
The content-hash layout is identical to the filename-hash layout,
except for these three differences:
1) A content digest is used instead of a filename digest.
2) The final element of the path returned from the get_path method
corresponds to the complete content digest. The path is a function
of the content digest alone.
3) Because the path is a function of content digest alone, the
get_filenames implementation cannot derive distfiles names from
paths, so it instead yields DistfileName instances whose names are
equal to content digest values. The DistfileName documentation
discusses resulting implications.
Motivations to use the content-hash layout instead of the
filename-hash layout may include:
1) Since the file path is independent of the file name, file
name collisions cannot occur. This makes the content-hash
layout suitable for storage of multiple types of files (not
only gentoo distfiles). For example, it can be used to store
distfiles for multiple linux distros within the same tree,
with automatic deduplication based on content digest. This
layout can be used to store and distribute practically anything
(including binary packages for example).
2) Allows multiple revisions for the same distfiles name. An
existing distfile can be updated, and if a user still has an
older copy of an ebuild repository (or an overlay), then a user
can successfully fetch a desired revision of the distfile as
long as it has not been purged from the mirror.
3) File integrity data is integrated into the layout itself,
making it very simple to verify the integrity of any file that
it contains. The only tool required is an implementation of
the chosen hash algorithm.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/package/ebuild/fetch.py | 97 ++++++++++++++++++++++++++++++++++
lib/portage/tests/ebuild/test_fetch.py | 36 +++++++++++++
2 files changed, 133 insertions(+)
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
index af9edd91e..f0ae864ad 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -464,6 +464,97 @@ class FilenameHashLayout:
return False
+class ContentHashLayout(FilenameHashLayout):
+ """
+ The content-hash layout is identical to the filename-hash layout,
+ except for these three differences:
+
+ 1) A content digest is used instead of a filename digest.
+
+ 2) The final element of the path returned from the get_path method
+ corresponds to the complete content digest. The path is a function
+ of the content digest alone.
+
+ 3) Because the path is a function of content digest alone, the
+ get_filenames implementation cannot derive distfiles names from
+ paths, so it instead yields DistfileName instances whose names are
+ equal to content digest values. The DistfileName documentation
+ discusses resulting implications.
+
+ Motivations to use the content-hash layout instead of the
+ filename-hash layout may include:
+
+ 1) Since the file path is independent of the file name, file
+ name collisions cannot occur. This makes the content-hash
+ layout suitable for storage of multiple types of files (not
+ only gentoo distfiles). For example, it can be used to store
+ distfiles for multiple linux distros within the same tree,
+ with automatic deduplication based on content digest. This
+ layout can be used to store and distribute practically anything
+ (including binary packages for example).
+
+ 2) Allows multiple revisions for the same distfiles name. An
+ existing distfile can be updated, and if a user still has an
+ older copy of an ebuild repository (or an overlay), then a user
+ can successfully fetch a desired revision of the distfile as
+ long as it has not been purged from the mirror.
+
+ 3) File integrity data is integrated into the layout itself,
+ making it very simple to verify the integrity of any file that
+ it contains. The only tool required is an implementation of
+ the chosen hash algorithm.
+ """
+
+ def get_path(self, filename):
+ """
+ For content-hash, the path is a function of the content digest alone.
+ The final element of the path returned from the get_path method
+ corresponds to the complete content digest.
+ """
+ fnhash = remaining = filename.digests[self.algo]
+ ret = ""
+ for c in self.cutoffs:
+ assert c % 4 == 0
+ c = c // 4
+ ret += remaining[:c] + "/"
+ remaining = remaining[c:]
+ return ret + fnhash
+
+ def get_filenames(self, distdir):
+ """
+ Yields DistfileName instances each with filename corresponding
+ to a digest value for self.algo, and which can be compared to
+ other DistfileName instances with their digests_equal method.
+ """
+ for filename in super(ContentHashLayout, self).get_filenames(distdir):
+ yield DistfileName(
+ filename, digests=dict([(self.algo, filename)])
+ )
+
+ @staticmethod
+ def verify_args(args, filename=None):
+ """
+ If the filename argument is given, then supported hash
+ algorithms are constrained by digests available in the filename
+ digests attribute.
+
+ @param args: layout.conf entry args
+ @param filename: filename with digests attribute
+ @return: True if args are valid for available digest algorithms,
+ and False otherwise
+ """
+ if len(args) != 3:
+ return False
+ if filename is None:
+ supported_algos = get_valid_checksum_keys()
+ else:
+ supported_algos = filename.digests
+ algo = args[1].upper()
+ if algo not in supported_algos:
+ return False
+ return FilenameHashLayout.verify_args(args)
+
+
class MirrorLayoutConfig:
"""
Class to read layout.conf from a mirror.
@@ -505,6 +596,8 @@ class MirrorLayoutConfig:
return FlatLayout.verify_args(val)
elif val[0] == 'filename-hash':
return FilenameHashLayout.verify_args(val)
+ elif val[0] == 'content-hash':
+ return ContentHashLayout.verify_args(val, filename=filename)
return False
def get_best_supported_layout(self, filename=None):
@@ -521,6 +614,8 @@ class MirrorLayoutConfig:
return FlatLayout(*val[1:])
elif val[0] == 'filename-hash':
return FilenameHashLayout(*val[1:])
+ elif val[0] == 'content-hash':
+ return ContentHashLayout(*val[1:])
# fallback
return FlatLayout()
@@ -533,6 +628,8 @@ class MirrorLayoutConfig:
ret.append(FlatLayout(*val[1:]))
elif val[0] == 'filename-hash':
ret.append(FilenameHashLayout(*val[1:]))
+ elif val[0] == 'content-hash':
+ ret.append(ContentHashLayout(*val[1:]))
if not ret:
ret.append(FlatLayout())
return ret
diff --git a/lib/portage/tests/ebuild/test_fetch.py b/lib/portage/tests/ebuild/test_fetch.py
index b88ae3efb..c195888cc 100644
--- a/lib/portage/tests/ebuild/test_fetch.py
+++ b/lib/portage/tests/ebuild/test_fetch.py
@@ -20,6 +20,7 @@ from portage.util._eventloop.global_event_loop import global_event_loop
from portage.package.ebuild.config import config
from portage.package.ebuild.digestgen import digestgen
from portage.package.ebuild.fetch import (
+ ContentHashLayout,
DistfileName,
_download_suffix,
fetch,
@@ -109,6 +110,11 @@ class EbuildFetchTestCase(TestCase):
"1=filename-hash BLAKE2B 8",
"0=flat",
),
+ (
+ "[structure]",
+ "0=content-hash SHA512 8:8:8",
+ "1=flat",
+ ),
)
fetchcommand = portage.util.shlex_split(playground.settings["FETCHCOMMAND"])
@@ -444,6 +450,35 @@ class EbuildFetchTestCase(TestCase):
self.assertEqual(FilenameHashLayout('SHA1', '8:16:24').get_path('foo-1.tar.gz'),
'19/c3b6/37a94b/foo-1.tar.gz')
+ def test_content_hash_layout(self):
+ self.assertFalse(ContentHashLayout.verify_args(('content-hash',)))
+ self.assertTrue(ContentHashLayout.verify_args(('content-hash', 'SHA1', '8')))
+ self.assertFalse(ContentHashLayout.verify_args(('content-hash', 'INVALID-HASH', '8')))
+ self.assertTrue(ContentHashLayout.verify_args(('content-hash', 'SHA1', '4:8:12')))
+ self.assertFalse(ContentHashLayout.verify_args(('content-hash', 'SHA1', '3')))
+ self.assertFalse(ContentHashLayout.verify_args(('content-hash', 'SHA1', 'junk')))
+ self.assertFalse(ContentHashLayout.verify_args(('content-hash', 'SHA1', '4:8:junk')))
+
+ filename = DistfileName(
+ 'foo-1.tar.gz',
+ digests=dict((algo, checksum_str(b'', hashname=algo)) for algo in MANIFEST2_HASH_DEFAULTS),
+ )
+
+ # Raise KeyError for a hash algorithm SHA1 which is not in MANIFEST2_HASH_DEFAULTS.
+ self.assertRaises(KeyError, ContentHashLayout('SHA1', '4').get_path, filename)
+
+ # Raise AttributeError for a plain string argument.
+ self.assertRaises(AttributeError, ContentHashLayout('SHA512', '4').get_path, str(filename))
+
+ self.assertEqual(ContentHashLayout('SHA512', '4').get_path(filename),
+ 'c/cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e')
+ self.assertEqual(ContentHashLayout('SHA512', '8').get_path(filename),
+ 'cf/cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e')
+ self.assertEqual(ContentHashLayout('SHA512', '8:16').get_path(filename),
+ 'cf/83e1/cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e')
+ self.assertEqual(ContentHashLayout('SHA512', '8:16:24').get_path(filename),
+ 'cf/83e1/357eef/cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e')
+
def test_mirror_layout_config(self):
mlc = MirrorLayoutConfig()
self.assertEqual(mlc.serialize(), ())
@@ -521,6 +556,7 @@ class EbuildFetchTestCase(TestCase):
FilenameHashLayout('SHA1', '8'),
FilenameHashLayout('SHA1', '8:16'),
FilenameHashLayout('SHA1', '8:16:24'),
+ ContentHashLayout('SHA512', '8:8:8'),
)
for layout in layouts:
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
@ 2021-02-22 12:18 Zac Medico
0 siblings, 0 replies; 4+ messages in thread
From: Zac Medico @ 2021-02-22 12:18 UTC (permalink / raw
To: gentoo-commits
commit: b9ef191c74982b0e8d837aa7dd256dc3c52f7d2c
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 20 23:11:46 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 11:48:41 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9ef191c
MirrorLayoutConfig: content digest support (bug 756778)
In order to support mirror layouts that use content
digests, extend MirrorLayoutConfig validate_structure and
get_best_supported_layout methods to support an optional
filename parameter of type DistfileName which includes a digests
attribute. Use the new parameter to account for availablility
of specific distfile content digests when validating and selecting
mirror layouts which require those digests.
The DistfileName type represents a distfile name and associated
content digests, used by MirrorLayoutConfig and related layout
implementations.
The path of a distfile within a layout must be dependent on
nothing more than the distfile name and its associated content
digests. For filename-hash layout, path is dependent on distfile
name alone, and the get_filenames implementation yields strings
corresponding to distfile names. For content-hash layout, path is
dependent on content digest alone, and the get_filenames
implementation yields DistfileName instances whose names are equal
to content digest values. The content-hash layout simply lacks
the filename-hash layout's innate ability to translate a distfile
path to a distfile name, and instead caries an innate ability
to translate a distfile path to a content digest.
In order to prepare for a migration from filename-hash to
content-hash layout, all consumers of the layout get_filenames
method need to be updated to work with content digests as a
substitute for distfile names. For example, in order to prepare
emirrordist for content-hash, a key-value store needs to be
added as a means to associate distfile names with content
digest values yielded by the content-hash get_filenames
implementation.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/package/ebuild/fetch.py | 98 ++++++++++++++++++++++++++++++----
lib/portage/tests/ebuild/test_fetch.py | 33 +++++++++---
2 files changed, 114 insertions(+), 17 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
index e0fecaf23..af9edd91e 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ['fetch']
@@ -344,6 +344,57 @@ _size_suffix_map = {
}
+class DistfileName(str):
+ """
+ The DistfileName type represents a distfile name and associated
+ content digests, used by MirrorLayoutConfig and related layout
+ implementations.
+
+ The path of a distfile within a layout must be dependent on
+ nothing more than the distfile name and its associated content
+ digests. For filename-hash layout, path is dependent on distfile
+ name alone, and the get_filenames implementation yields strings
+ corresponding to distfile names. For content-hash layout, path is
+ dependent on content digest alone, and the get_filenames
+ implementation yields DistfileName instances whose names are equal
+ to content digest values. The content-hash layout simply lacks
+ the filename-hash layout's innate ability to translate a distfile
+ path to a distfile name, and instead caries an innate ability
+ to translate a distfile path to a content digest.
+
+ In order to prepare for a migration from filename-hash to
+ content-hash layout, all consumers of the layout get_filenames
+ method need to be updated to work with content digests as a
+ substitute for distfile names. For example, in order to prepare
+ emirrordist for content-hash, a key-value store needs to be
+ added as a means to associate distfile names with content
+ digest values yielded by the content-hash get_filenames
+ implementation.
+ """
+ def __new__(cls, s, digests=None):
+ return str.__new__(cls, s)
+
+ def __init__(self, s, digests=None):
+ super().__init__()
+ self.digests = {} if digests is None else digests
+
+ def digests_equal(self, other):
+ """
+ Test if digests compare equal to those of another instance.
+ """
+ if not isinstance(other, DistfileName):
+ return False
+ matches = []
+ for algo, digest in self.digests.items():
+ other_digest = other.digests.get(algo)
+ if other_digest is not None:
+ if other_digest == digest:
+ matches.append(algo)
+ else:
+ return False
+ return bool(matches)
+
+
class FlatLayout:
def get_path(self, filename):
return filename
@@ -439,19 +490,36 @@ class MirrorLayoutConfig:
self.structure = data
@staticmethod
- def validate_structure(val):
+ def validate_structure(val, filename=None):
+ """
+ If the filename argument is given, then supported hash
+ algorithms are constrained by digests available in the filename
+ digests attribute.
+
+ @param val: layout.conf entry args
+ @param filename: filename with digests attribute
+ @return: True if args are valid for available digest algorithms,
+ and False otherwise
+ """
if val[0] == 'flat':
return FlatLayout.verify_args(val)
- if val[0] == 'filename-hash':
+ elif val[0] == 'filename-hash':
return FilenameHashLayout.verify_args(val)
return False
- def get_best_supported_layout(self):
+ def get_best_supported_layout(self, filename=None):
+ """
+ If the filename argument is given, then acceptable hash
+ algorithms are constrained by digests available in the filename
+ digests attribute.
+
+ @param filename: filename with digests attribute
+ """
for val in self.structure:
- if self.validate_structure(val):
+ if self.validate_structure(val, filename=filename):
if val[0] == 'flat':
return FlatLayout(*val[1:])
- if val[0] == 'filename-hash':
+ elif val[0] == 'filename-hash':
return FilenameHashLayout(*val[1:])
# fallback
return FlatLayout()
@@ -515,7 +583,7 @@ def get_mirror_url(mirror_url, filename, mysettings, cache_path=None):
# For some protocols, urlquote is required for correct behavior,
# and it must not be used for other protocols like rsync and sftp.
- path = mirror_conf.get_best_supported_layout().get_path(filename)
+ path = mirror_conf.get_best_supported_layout(filename=filename).get_path(filename)
if urlparse(mirror_url).scheme in ('ftp', 'http', 'https'):
path = urlquote(path)
return mirror_url + "/distfiles/" + path
@@ -722,15 +790,23 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
if hasattr(myuris, 'items'):
for myfile, uri_set in myuris.items():
for myuri in uri_set:
- file_uri_tuples.append((myfile, myuri))
+ file_uri_tuples.append(
+ (DistfileName(myfile, digests=mydigests.get(myfile)), myuri)
+ )
if not uri_set:
- file_uri_tuples.append((myfile, None))
+ file_uri_tuples.append(
+ (DistfileName(myfile, digests=mydigests.get(myfile)), None)
+ )
else:
for myuri in myuris:
if urlparse(myuri).scheme:
- file_uri_tuples.append((os.path.basename(myuri), myuri))
+ file_uri_tuples.append(
+ (DistfileName(myfile, digests=mydigests.get(myfile)), myuri)
+ )
else:
- file_uri_tuples.append((os.path.basename(myuri), None))
+ file_uri_tuples.append(
+ (DistfileName(myfile, digests=mydigests.get(myfile)), None)
+ )
filedict = OrderedDict()
primaryuri_dict = {}
diff --git a/lib/portage/tests/ebuild/test_fetch.py b/lib/portage/tests/ebuild/test_fetch.py
index c5ea8253b..b88ae3efb 100644
--- a/lib/portage/tests/ebuild/test_fetch.py
+++ b/lib/portage/tests/ebuild/test_fetch.py
@@ -7,7 +7,8 @@ import tempfile
import portage
from portage import shutil, os
-from portage.const import BASH_BINARY, PORTAGE_PYM_PATH
+from portage.checksum import checksum_str
+from portage.const import BASH_BINARY, MANIFEST2_HASH_DEFAULTS, PORTAGE_PYM_PATH
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import ResolverPlayground
from portage.tests.util.test_socks5 import AsyncHTTPServer
@@ -18,8 +19,14 @@ from portage.util._async.SchedulerInterface import SchedulerInterface
from portage.util._eventloop.global_event_loop import global_event_loop
from portage.package.ebuild.config import config
from portage.package.ebuild.digestgen import digestgen
-from portage.package.ebuild.fetch import (_download_suffix, fetch, FlatLayout,
- FilenameHashLayout, MirrorLayoutConfig)
+from portage.package.ebuild.fetch import (
+ DistfileName,
+ _download_suffix,
+ fetch,
+ FilenameHashLayout,
+ FlatLayout,
+ MirrorLayoutConfig,
+)
from _emerge.EbuildFetcher import EbuildFetcher
from _emerge.Package import Package
@@ -142,9 +149,14 @@ class EbuildFetchTestCase(TestCase):
content["/distfiles/layout.conf"] = layout_data.encode("utf8")
for k, v in distfiles.items():
+ filename = DistfileName(
+ k,
+ digests=dict((algo, checksum_str(v, hashname=algo)) for algo in MANIFEST2_HASH_DEFAULTS),
+ )
+
# mirror path
for layout in layouts:
- content["/distfiles/" + layout.get_path(k)] = v
+ content["/distfiles/" + layout.get_path(filename)] = v
# upstream path
content["/distfiles/{}.txt".format(k)] = v
@@ -499,6 +511,10 @@ class EbuildFetchTestCase(TestCase):
io.StringIO(conf))
def test_filename_hash_layout_get_filenames(self):
+ filename = DistfileName(
+ 'foo-1.tar.gz',
+ digests=dict((algo, checksum_str(b'', hashname=algo)) for algo in MANIFEST2_HASH_DEFAULTS),
+ )
layouts = (
FlatLayout(),
FilenameHashLayout('SHA1', '4'),
@@ -506,7 +522,6 @@ class EbuildFetchTestCase(TestCase):
FilenameHashLayout('SHA1', '8:16'),
FilenameHashLayout('SHA1', '8:16:24'),
)
- filename = 'foo-1.tar.gz'
for layout in layouts:
distdir = tempfile.mkdtemp()
@@ -520,6 +535,12 @@ class EbuildFetchTestCase(TestCase):
with open(path, 'wb') as f:
pass
- self.assertEqual([filename], list(layout.get_filenames(distdir)))
+ file_list = list(layout.get_filenames(distdir))
+ self.assertTrue(len(file_list) > 0)
+ for filename_result in file_list:
+ if isinstance(filename_result, DistfileName):
+ self.assertTrue(filename_result.digests_equal(filename))
+ else:
+ self.assertEqual(filename_result, str(filename))
finally:
shutil.rmtree(distdir)
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
@ 2024-03-02 4:09 Zac Medico
0 siblings, 0 replies; 4+ messages in thread
From: Zac Medico @ 2024-03-02 4:09 UTC (permalink / raw
To: gentoo-commits
commit: fe510e099bc9a8055c3ee50fced47fc3dc7ba166
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 1 17:09:56 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 1 18:08:51 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fe510e09
doebuild: Call _setup_locale
Call _setup_locale in order to prevent an AssertionError from
config.environ() for the config phase (or any other phase for
that matter).
For returnproc or returnpid assume that the event loop is running
so we can't run the event loop to call _setup_locale in this case
and we have to assume the caller took care of it (otherwise
config.environ() will raise AssertionError).
Update DoebuildFdPipesTestCase to use EAPI 8 and test the
pkg_config function with an ebuild located in /var/db/pkg just
like emerge --config does. Set LC_ALL=C just before doebuild
calls in order to try and trigger the config.environ()
split_LC_ALL assertion.
Bug: https://bugs.gentoo.org/925863
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/package/ebuild/doebuild.py | 8 +++
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 77 ++++++++++++----------
2 files changed, 52 insertions(+), 33 deletions(-)
diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
index bc51fdff2d..942fa90101 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -43,6 +43,7 @@ portage.proxy.lazyimport.lazyimport(
"portage.util._async.SchedulerInterface:SchedulerInterface",
"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util.ExtractKernelVersion:ExtractKernelVersion",
+ "_emerge.EbuildPhase:_setup_locale",
)
from portage import (
@@ -1034,6 +1035,13 @@ def doebuild(
myebuild, mydo, myroot, mysettings, debug, use_cache, mydbapi
)
+ # For returnproc or returnpid assume that the event loop is running
+ # so we can't run the event loop to call _setup_locale in this case
+ # and we have to assume the caller took care of it (otherwise
+ # config.environ() will raise AssertionError).
+ if not (returnproc or returnpid):
+ asyncio.run(_setup_locale(mysettings))
+
if mydo in clean_phases:
builddir_lock = None
if not returnpid and "PORTAGE_BUILDDIR_LOCKED" not in mysettings:
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index 678486ed16..b38605bb90 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Gentoo Authors
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import multiprocessing
@@ -27,20 +27,22 @@ class DoebuildFdPipesTestCase(TestCase):
output_fd = self.output_fd
ebuild_body = ["S=${WORKDIR}"]
- for phase_func in (
- "pkg_info",
- "pkg_nofetch",
- "pkg_pretend",
- "pkg_setup",
- "src_unpack",
- "src_prepare",
- "src_configure",
- "src_compile",
- "src_test",
- "src_install",
+ for phase_func, default in (
+ ("pkg_info", False),
+ ("pkg_nofetch", False),
+ ("pkg_pretend", False),
+ ("pkg_setup", False),
+ ("pkg_config", False),
+ ("src_unpack", False),
+ ("src_prepare", True),
+ ("src_configure", False),
+ ("src_compile", False),
+ ("src_test", False),
+ ("src_install", False),
):
ebuild_body.append(
- ("%s() { echo ${EBUILD_PHASE}" " 1>&%s; }") % (phase_func, output_fd)
+ ("%s() { %secho ${EBUILD_PHASE}" " 1>&%s; }")
+ % (phase_func, "default; " if default else "", output_fd)
)
ebuild_body.append("")
@@ -48,7 +50,7 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
- "EAPI": "5",
+ "EAPI": "8",
"MISC_CONTENT": ebuild_body,
}
}
@@ -103,24 +105,33 @@ class DoebuildFdPipesTestCase(TestCase):
type_name="ebuild",
)
settings.setcpv(pkg)
- ebuildpath = portdb.findname(cpv)
- self.assertNotEqual(ebuildpath, None)
-
- for phase in (
- "info",
- "nofetch",
- "pretend",
- "setup",
- "unpack",
- "prepare",
- "configure",
- "compile",
- "test",
- "install",
- "qmerge",
- "clean",
- "merge",
+
+ # Try to trigger the config.environ() split_LC_ALL assertion for bug 925863.
+ settings["LC_ALL"] = "C"
+
+ source_ebuildpath = portdb.findname(cpv)
+ self.assertNotEqual(source_ebuildpath, None)
+
+ for phase, tree, ebuildpath in (
+ ("info", "porttree", source_ebuildpath),
+ ("nofetch", "porttree", source_ebuildpath),
+ ("pretend", "porttree", source_ebuildpath),
+ ("setup", "porttree", source_ebuildpath),
+ ("unpack", "porttree", source_ebuildpath),
+ ("prepare", "porttree", source_ebuildpath),
+ ("configure", "porttree", source_ebuildpath),
+ ("compile", "porttree", source_ebuildpath),
+ ("test", "porttree", source_ebuildpath),
+ ("install", "porttree", source_ebuildpath),
+ ("qmerge", "porttree", source_ebuildpath),
+ ("clean", "porttree", source_ebuildpath),
+ ("merge", "porttree", source_ebuildpath),
+ ("clean", "porttree", source_ebuildpath),
+ ("config", "vartree", root_config.trees["vartree"].dbapi.findname(cpv)),
):
+ if ebuildpath is not source_ebuildpath:
+ self.assertNotEqual(ebuildpath, None)
+
pr, pw = multiprocessing.Pipe(duplex=False)
producer = ForkProcess(
@@ -131,8 +142,8 @@ class DoebuildFdPipesTestCase(TestCase):
args=(QueryCommand._db, pw, ebuildpath, phase),
kwargs={
"settings": settings,
- "mydbapi": portdb,
- "tree": "porttree",
+ "mydbapi": root_config.trees[tree].dbapi,
+ "tree": tree,
"vartree": root_config.trees["vartree"],
"prev_mtimes": {},
},
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2024-03-02 4:09 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-02-22 12:18 [gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/ Zac Medico
-- strict thread matches above, loose matches on Subject: below --
2024-03-02 4:09 Zac Medico
2021-02-22 12:18 Zac Medico
2019-10-20 8:34 Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox