* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2022-12-26 17:28 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2022-12-26 17:28 UTC (permalink / raw
To: gentoo-commits
commit: 5a51817e29ee8a9879876be1576eaa38b357ffa3
Author: Brian Harring <ferringb <AT> gmail <DOT> com>
AuthorDate: Mon Dec 26 01:31:02 2022 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Mon Dec 26 17:27:24 2022 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=5a51817e
Add use flag validation for /etc/portage/package.use/* content.
Had this been in place, it would've detected pkgcore/pkgcore#384 long ago.
Signed-off-by: Brian Harring <ferringb <AT> gmail.com>
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/domain.py | 8 ++++++++
tests/ebuild/test_domain.py | 19 +++++++++++--------
2 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/src/pkgcore/ebuild/domain.py b/src/pkgcore/ebuild/domain.py
index 6f86f7d14..9fef5f843 100644
--- a/src/pkgcore/ebuild/domain.py
+++ b/src/pkgcore/ebuild/domain.py
@@ -44,6 +44,7 @@ from ..util.parserestrict import ParseError, parse_match
from . import const
from . import repository as ebuild_repo
from .atom import atom as _atom
+from .eapi import get_latest_PMS_eapi
from .misc import (
ChunkedDataDict,
chunked_data,
@@ -86,7 +87,10 @@ def package_use_splitter(iterable):
USE_EXPAND target and should be expanded into their normalized/long form.
"""
+ eapi_obj = get_latest_PMS_eapi()
+
def f(tokens: list[str]):
+
i = iter(tokens)
for idx, x in enumerate(i):
if x.endswith(":"):
@@ -99,8 +103,12 @@ def package_use_splitter(iterable):
flag = f"-{x}_{flag[1:]}"
else:
flag = f"{x}_{flag}"
+ if not eapi_obj.is_valid_use_flag(flag.lstrip("-")):
+ raise ParseError(f"token {flag} is not a valid use flag")
l.append(flag)
return l
+ elif not eapi_obj.is_valid_use_flag(x.lstrip("-")):
+ raise ParseError(f"token {flag} is not a valid use flag")
# if we made it here, there's no USE_EXPAND; thus just return the original sequence
return tokens
diff --git a/tests/ebuild/test_domain.py b/tests/ebuild/test_domain.py
index 87c3d489f..90594b9d6 100644
--- a/tests/ebuild/test_domain.py
+++ b/tests/ebuild/test_domain.py
@@ -20,6 +20,8 @@ class TestDomain:
self.profile_base = tmp_path_factory.mktemp("profiles")
self.profile1 = self.profile_base / "profile1"
self.pmixin.mk_profile(self.profile_base, str(self.profile1))
+ self.pusedir = self.confdir / "package.use"
+ self.pusedir.mkdir()
def mk_domain(self):
return domain_mod.domain(
@@ -32,14 +34,11 @@ class TestDomain:
def test_sorting(self):
"""assert that configuration files are read in alphanum ordering"""
- cdir = self.confdir / "package.use"
- cdir.mkdir()
-
# assert the base state; no files, no content.
assert () == self.mk_domain().pkg_use
- open(cdir / "00", "w").write("*/* X")
- open(cdir / "01", "w").write("*/* -X Y")
+ open(self.pusedir / "00", "w").write("*/* X")
+ open(self.pusedir / "01", "w").write("*/* -X Y")
# Force the returned ordering to be reversed; this is to assert that
# the domain forces a sort.
@@ -57,9 +56,7 @@ class TestDomain:
) == self.mk_domain().pkg_use
def test_use_expand_syntax(self):
- puse = self.confdir / "package.use"
- puse.mkdir()
- open(puse / "a", "w").write(
+ open(self.pusedir / "a", "w").write(
textwrap.dedent(
"""
*/* x_y1
@@ -82,3 +79,9 @@ class TestDomain:
),
),
) == self.mk_domain().pkg_use
+
+ def test_use_flag_parsing_enforcement(self):
+ open(self.pusedir / "a", "w").write("*/* X:")
+ # TODO: need to catch the warning here, but I'm not sure how.
+ # Meanwhile, ensure that the failed token is ignored.
+ assert ((packages.AlwaysTrue, ((), ())),) == self.mk_domain().pkg_use
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2022-12-26 17:28 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2022-12-26 17:28 UTC (permalink / raw
To: gentoo-commits
commit: 6bcdbd769bfa8ebf78e31887e4fd54eaf1c47032
Author: Brian Harring <ferringb <AT> gmail <DOT> com>
AuthorDate: Sun Dec 25 23:23:15 2022 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Mon Dec 26 17:27:24 2022 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=6bcdbd76
Add USE_EXPAND expansion awareness for /etc/portage/package.use/* files.
Specifically, if you have:
`*/* PYTHON_TARGETS: -python2_7 python3_9`
Pkgcore was treating `PYTHON_TARGETS:` as a use flag. That's obviously
wrong, and the parsing should be tightened there.
Closes: https://github.com/pkgcore/pkgcore/issues/384
Signed-off-by: Brian Harring <ferringb <AT> gmail.com>
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/domain.py | 44 ++++++++++++++++++++++++++++++++++++++------
tests/ebuild/test_domain.py | 28 ++++++++++++++++++++++++++++
2 files changed, 66 insertions(+), 6 deletions(-)
diff --git a/src/pkgcore/ebuild/domain.py b/src/pkgcore/ebuild/domain.py
index 7e76f1112..6f86f7d14 100644
--- a/src/pkgcore/ebuild/domain.py
+++ b/src/pkgcore/ebuild/domain.py
@@ -66,15 +66,47 @@ def package_masks(iterable):
logger.warning(f"{path!r}, line {lineno}: parsing error: {e}")
-def package_keywords_splitter(iterable):
+def restriction_payload_splitter(iterable, post_process=lambda x: x):
for line, lineno, path in iterable:
v = line.split()
try:
- yield parse_match(v[0]), tuple(v[1:]), line, lineno, path
+ # TODO: expand this invocation to allow threading token level validation down.
+ # things like "is this a valid use flag?"
+ yield parse_match(v[0]), tuple(post_process(v[1:])), line, lineno, path
except ParseError as e:
logger.warning(f"{path!r}, line {lineno}: parsing error: {e}")
+def package_use_splitter(iterable):
+ """Parse package.use user configuration files
+
+ Basic syntax is <query> (?:-?use_f )* (?:USE_EXPAND: (?:flags)*)
+ IE, a restriction to match, use flags to turn on. If a 'flag' ends in ':'
+ then it's considered a USE_EXPAND directive, and all that follow are values of that
+ USE_EXPAND target and should be expanded into their normalized/long form.
+ """
+
+ def f(tokens: list[str]):
+ i = iter(tokens)
+ for idx, x in enumerate(i):
+ if x.endswith(":"):
+ # we encountered `USE_EXPAND:` , thus all following tokens
+ # are values of that.
+ x = x.lower()[:-1]
+ l = tokens[0:idx]
+ for flag in i:
+ if flag.startswith("-"):
+ flag = f"-{x}_{flag[1:]}"
+ else:
+ flag = f"{x}_{flag}"
+ l.append(flag)
+ return l
+ # if we made it here, there's no USE_EXPAND; thus just return the original sequence
+ return tokens
+
+ return restriction_payload_splitter(iterable, post_process=f)
+
+
def package_env_splitter(basedir, iterable):
for line, lineno, path in iterable:
val = line.split()
@@ -396,25 +428,25 @@ class domain(config_domain):
return tuple(x[0] for x in data)
# TODO: deprecated, remove in 0.11
- @load_property("package.keywords", parse_func=package_keywords_splitter)
+ @load_property("package.keywords", parse_func=restriction_payload_splitter)
def pkg_keywords(self, data, debug=False):
if debug:
return tuple(data)
return tuple((x[0], stable_unique(x[1])) for x in data)
- @load_property("package.accept_keywords", parse_func=package_keywords_splitter)
+ @load_property("package.accept_keywords", parse_func=restriction_payload_splitter)
def pkg_accept_keywords(self, data, debug=False):
if debug:
return tuple(data)
return tuple((x[0], stable_unique(x[1])) for x in data)
- @load_property("package.license", parse_func=package_keywords_splitter)
+ @load_property("package.license", parse_func=restriction_payload_splitter)
def pkg_licenses(self, data, debug=False):
if debug:
return tuple(data)
return tuple((x[0], stable_unique(x[1])) for x in data)
- @load_property("package.use", parse_func=package_keywords_splitter)
+ @load_property("package.use", parse_func=package_use_splitter)
def pkg_use(self, data, debug=False):
if debug:
return tuple(data)
diff --git a/tests/ebuild/test_domain.py b/tests/ebuild/test_domain.py
index 6629cfb9a..87c3d489f 100644
--- a/tests/ebuild/test_domain.py
+++ b/tests/ebuild/test_domain.py
@@ -1,3 +1,4 @@
+import textwrap
from unittest import mock
import pytest
@@ -54,3 +55,30 @@ class TestDomain:
(packages.AlwaysTrue, ((), ("X",))),
(packages.AlwaysTrue, (("X",), ("Y",))),
) == self.mk_domain().pkg_use
+
+ def test_use_expand_syntax(self):
+ puse = self.confdir / "package.use"
+ puse.mkdir()
+ open(puse / "a", "w").write(
+ textwrap.dedent(
+ """
+ */* x_y1
+ # unrelated is there to verify that it's unaffected by the USE_EXPAND
+ */* unrelated X: -y1 y2
+ """
+ )
+ )
+
+ assert (
+ (packages.AlwaysTrue, ((), ("x_y1",))),
+ (
+ packages.AlwaysTrue,
+ (
+ ("x_y1",),
+ (
+ "unrelated",
+ "x_y2",
+ ),
+ ),
+ ),
+ ) == self.mk_domain().pkg_use
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2022-12-26 17:28 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2022-12-26 17:28 UTC (permalink / raw
To: gentoo-commits
commit: de599e61198012a5c7a6bda37e864609bc8d5754
Author: Brian Harring <ferringb <AT> gmail <DOT> com>
AuthorDate: Sun Dec 25 22:50:50 2022 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Mon Dec 26 17:27:24 2022 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=de599e61
Force stable sorting of /etc/portage/* files loading.
Specifically, alphanumeric sorting; a '00' should always be
parsed before an '01' or 'a'.
The ordering must be stable so that globals- in a 00 file- can
be overridden per package via other files. If there is no stable
ordering forced on how the order of files read, then this isn't supported.
Closes: https://github.com/pkgcore/pkgcore/issues/385
Signed-off-by: Brian Harring <ferringb <AT> gmail.com>
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/domain.py | 3 ++-
tests/ebuild/test_domain.py | 56 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/src/pkgcore/ebuild/domain.py b/src/pkgcore/ebuild/domain.py
index 4e33e0fc9..7e76f1112 100644
--- a/src/pkgcore/ebuild/domain.py
+++ b/src/pkgcore/ebuild/domain.py
@@ -132,7 +132,8 @@ def generate_filter(masks, unmasks, *extra):
def _read_config_file(path):
"""Read all the data files under a given path."""
try:
- for fs_obj in iter_scan(path, follow_symlinks=True):
+ # sort based on location by default; this is to ensure 00 is before 01, and before a
+ for fs_obj in sorted(iter_scan(path, follow_symlinks=True)):
if not fs_obj.is_reg or "/." in fs_obj.location:
continue
for lineno, line in iter_read_bash(
diff --git a/tests/ebuild/test_domain.py b/tests/ebuild/test_domain.py
new file mode 100644
index 000000000..6629cfb9a
--- /dev/null
+++ b/tests/ebuild/test_domain.py
@@ -0,0 +1,56 @@
+from unittest import mock
+
+import pytest
+
+from pkgcore.ebuild import domain as domain_mod
+from pkgcore.ebuild import profiles
+from pkgcore.fs.livefs import iter_scan
+from pkgcore.restrictions import packages
+
+from .test_profiles import profile_mixin
+
+
+class TestDomain:
+ @pytest.fixture(autouse=True, scope="function")
+ def _setup(self, tmp_path_factory):
+ self.confdir = tmp_path_factory.mktemp("conf")
+ self.rootdir = tmp_path_factory.mktemp("root")
+ self.pmixin = profile_mixin()
+ self.profile_base = tmp_path_factory.mktemp("profiles")
+ self.profile1 = self.profile_base / "profile1"
+ self.pmixin.mk_profile(self.profile_base, str(self.profile1))
+
+ def mk_domain(self):
+ return domain_mod.domain(
+ profiles.OnDiskProfile(str(self.profile_base), "profile1"),
+ [],
+ [],
+ ROOT=self.rootdir,
+ config_dir=self.confdir,
+ )
+
+ def test_sorting(self):
+ """assert that configuration files are read in alphanum ordering"""
+ cdir = self.confdir / "package.use"
+ cdir.mkdir()
+
+ # assert the base state; no files, no content.
+ assert () == self.mk_domain().pkg_use
+
+ open(cdir / "00", "w").write("*/* X")
+ open(cdir / "01", "w").write("*/* -X Y")
+
+ # Force the returned ordering to be reversed; this is to assert that
+ # the domain forces a sort.
+ orig_func = iter_scan
+
+ def rev_iter_scan(*args, **kwargs):
+ return iter(sorted(orig_func(*args, **kwargs), reverse=True))
+
+ with mock.patch(
+ "pkgcore.fs.livefs.iter_scan", side_effect=rev_iter_scan
+ ), mock.patch("pkgcore.ebuild.domain.iter_scan", side_effect=rev_iter_scan):
+ assert (
+ (packages.AlwaysTrue, ((), ("X",))),
+ (packages.AlwaysTrue, (("X",), ("Y",))),
+ ) == self.mk_domain().pkg_use
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2022-12-26 17:28 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2022-12-26 17:28 UTC (permalink / raw
To: gentoo-commits
commit: 4cce9a19dcbe670a676af89c2362f90411c5c796
Author: Brian Harring <ferringb <AT> gmail <DOT> com>
AuthorDate: Mon Dec 26 04:24:06 2022 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Mon Dec 26 17:27:25 2022 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=4cce9a19
Ignore both empty and non-existant repos.conf files.
Whilst this should probably matter, all reporting pathways I know of
involve "don't yell at the user" this it's probably not optimal
to have pathways that "yell at the user".
Closes: https://github.com/pkgcore/pkgcore/issues/365
Signed-off-by: Brian Harring <ferringb <AT> gmail.com>
Closes: https://github.com/pkgcore/pkgcore/pull/387
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/portage_conf.py | 8 ++++++--
tests/ebuild/test_portage_conf.py | 7 +------
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/src/pkgcore/ebuild/portage_conf.py b/src/pkgcore/ebuild/portage_conf.py
index d2865c0bb..907a1a2dd 100644
--- a/src/pkgcore/ebuild/portage_conf.py
+++ b/src/pkgcore/ebuild/portage_conf.py
@@ -357,8 +357,10 @@ class PortageConfig(DictMixin):
hidden=False,
backup=False,
):
+ had_repo_conf = False
try:
with open(fp) as f:
+ had_repo_conf = True
defaults, repo_confs = parser.parse_file(f)
except PermissionError as e:
raise base_errors.PermissionDenied(fp, write=False) from e
@@ -375,8 +377,10 @@ class PortageConfig(DictMixin):
)
main_defaults.update(defaults)
- if not repo_confs:
- logger.warning(f"repos.conf: parsing {fp!r}: file is empty")
+ if not had_repo_conf and not repo_confs:
+ logger.warning(
+ "repos.conf: not found, but should exist for modern support"
+ )
for name, repo_conf in repo_confs.items():
if name in repos:
diff --git a/tests/ebuild/test_portage_conf.py b/tests/ebuild/test_portage_conf.py
index 395373b85..edb3d89b6 100644
--- a/tests/ebuild/test_portage_conf.py
+++ b/tests/ebuild/test_portage_conf.py
@@ -5,12 +5,12 @@ import stat
import textwrap
import pytest
+from snakeoil.osutils import pjoin
from pkgcore import const
from pkgcore import exceptions as base_errors
from pkgcore.config import errors as config_errors
from pkgcore.ebuild.portage_conf import PortageConfig
-from snakeoil.osutils import pjoin
load_make_conf = PortageConfig.load_make_conf
load_repos_conf = PortageConfig.load_repos_conf
@@ -78,11 +78,6 @@ class TestReposConf:
with pytest.raises(base_errors.PermissionDenied):
load_repos_conf(path)
- def test_blank_file(self, tmp_path, caplog):
- (path := tmp_path / "file").touch()
- load_repos_conf(path)
- assert "file is empty" in caplog.text
-
def test_garbage_file(self, tmp_path):
(path := tmp_path / "file").write_bytes(binascii.b2a_hex(os.urandom(10)))
with pytest.raises(config_errors.ConfigurationError):
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2023-01-02 20:03 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2023-01-02 20:03 UTC (permalink / raw
To: gentoo-commits
commit: e174fa3c960fef3ed6225fa9a43e14e0e11332b0
Author: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 2 19:57:07 2023 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Mon Jan 2 19:57:07 2023 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=e174fa3c
domain: fix parsing of multiple USE_EXAPNDs
- add support for parsing of multiple USE_EXAPNDs
- fix error reporting when not USE_EXPAND has parsing error
- updates tests to catch warning thrown by parsing
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/domain.py | 11 +++++++----
tests/ebuild/test_domain.py | 29 ++++++++++++++++++++++++++---
2 files changed, 33 insertions(+), 7 deletions(-)
diff --git a/src/pkgcore/ebuild/domain.py b/src/pkgcore/ebuild/domain.py
index 9fef5f843..0dba49d5d 100644
--- a/src/pkgcore/ebuild/domain.py
+++ b/src/pkgcore/ebuild/domain.py
@@ -92,13 +92,16 @@ def package_use_splitter(iterable):
def f(tokens: list[str]):
i = iter(tokens)
- for idx, x in enumerate(i):
- if x.endswith(":"):
+ for idx, flag in enumerate(i):
+ if flag.endswith(":"):
# we encountered `USE_EXPAND:` , thus all following tokens
# are values of that.
- x = x.lower()[:-1]
+ x = flag.lower()[:-1]
l = tokens[0:idx]
for flag in i:
+ if flag.endswith(":"):
+ x = flag.lower()[:-1]
+ continue
if flag.startswith("-"):
flag = f"-{x}_{flag[1:]}"
else:
@@ -107,7 +110,7 @@ def package_use_splitter(iterable):
raise ParseError(f"token {flag} is not a valid use flag")
l.append(flag)
return l
- elif not eapi_obj.is_valid_use_flag(x.lstrip("-")):
+ elif not eapi_obj.is_valid_use_flag(flag.lstrip("-")):
raise ParseError(f"token {flag} is not a valid use flag")
# if we made it here, there's no USE_EXPAND; thus just return the original sequence
return tokens
diff --git a/tests/ebuild/test_domain.py b/tests/ebuild/test_domain.py
index 28706e254..e7b93b11b 100644
--- a/tests/ebuild/test_domain.py
+++ b/tests/ebuild/test_domain.py
@@ -62,6 +62,8 @@ class TestDomain:
*/* x_y1
# unrelated is there to verify that it's unaffected by the USE_EXPAND
*/* unrelated X: -y1 y2
+ # multiple USE_EXPANDs
+ */* unrelated X: -y1 y2 Z: -z3 z4
"""
)
)
@@ -78,10 +80,31 @@ class TestDomain:
),
),
),
+ (
+ packages.AlwaysTrue,
+ (
+ ("x_y1", "z_z3"),
+ (
+ "unrelated",
+ "x_y2",
+ "z_z4",
+ ),
+ ),
+ ),
) == self.mk_domain().pkg_use
- def test_use_flag_parsing_enforcement(self):
+ def test_use_flag_parsing_enforcement(self, caplog):
(self.pusedir / "a").write_text("*/* X:")
- # TODO: need to catch the warning here, but I'm not sure how.
- # Meanwhile, ensure that the failed token is ignored.
assert ((packages.AlwaysTrue, ((), ())),) == self.mk_domain().pkg_use
+ assert caplog.text == "" # no problems with nothing after USE_EXPAND:
+ caplog.clear()
+
+ (self.pusedir / "a").write_text("*/* y $x")
+ assert () == self.mk_domain().pkg_use
+ assert "token $x is not a valid use flag" in caplog.text
+ caplog.clear()
+
+ (self.pusedir / "a").write_text("*/* y X: $z")
+ assert () == self.mk_domain().pkg_use
+ assert "token x_$z is not a valid use flag" in caplog.text
+ caplog.clear()
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2023-01-17 20:50 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2023-01-17 20:50 UTC (permalink / raw
To: gentoo-commits
commit: f1c58b74df5eab2dac1e8e5810d9a042785f60cc
Author: Brian Harring <ferringb <AT> gmail <DOT> com>
AuthorDate: Wed Jan 11 05:52:01 2023 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Tue Jan 17 20:49:46 2023 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=f1c58b74
refactor(portage_conf): Push fallback (and path awareness) logic of repos.conf into methods.
The previous code had the exception + fallback handling in the init; this is already
a long init, and there are changes needed for repos conf, thus introduce a method to
do the fallback logic.
Said method will be extended to essentially bury the complexity of repos.conf- that problem-
into an encapsulated method to try and reduce the cognitive burden of dealing with it.
Why this matters: this work is prep for extension of repos.conf to add in various syncing
features portage grew over the last decade+.
Signed-off-by: Brian Harring <ferringb <AT> gmail.com>
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/portage_conf.py | 56 ++++++++++++++++++++------------------
tests/ebuild/test_portage_conf.py | 2 +-
2 files changed, 31 insertions(+), 27 deletions(-)
diff --git a/src/pkgcore/ebuild/portage_conf.py b/src/pkgcore/ebuild/portage_conf.py
index 815d563d4..9e5fffac7 100644
--- a/src/pkgcore/ebuild/portage_conf.py
+++ b/src/pkgcore/ebuild/portage_conf.py
@@ -172,24 +172,7 @@ class PortageConfig(DictMixin):
}
)
- try:
- repos_conf_defaults, repos_conf = self.load_repos_conf(
- pjoin(self.dir, "repos.conf")
- )
- except config_errors.ParsingError as e:
- if not getattr(getattr(e, "exc", None), "errno", None) == errno.ENOENT:
- raise
- try:
- # fallback to defaults provided by pkgcore
- repos_conf_defaults, repos_conf = self.load_repos_conf(
- pjoin(const.CONFIG_PATH, "repos.conf")
- )
- except IGNORED_EXCEPTIONS:
- raise
- except Exception as e:
- raise config_errors.ParsingError(
- "failed to find a usable repos.conf"
- ) from e
+ repos_conf_defaults, repos_conf = self.load_repos_conf()
self["ebuild-repo-common"] = basics.AutoConfigSection(
{
@@ -332,9 +315,31 @@ class PortageConfig(DictMixin):
# quirk of read_bash_dict; it returns only what was mutated.
vars_dict.update(new_vars)
+ def load_repos_conf(self) -> tuple[dict, dict]:
+ """parse and return repos.conf content, tracing the default and the fallback location"""
+ try:
+ repos_conf_defaults, repos_conf = self.parse_repos_conf_path(
+ pjoin(self.dir, "repos.conf")
+ )
+ except config_errors.ParsingError as e:
+ if not getattr(getattr(e, "exc", None), "errno", None) == errno.ENOENT:
+ raise
+ try:
+ # fallback to defaults provided by pkgcore
+ repos_conf_defaults, repos_conf = self.parse_repos_conf_path(
+ pjoin(const.CONFIG_PATH, "repos.conf")
+ )
+ except IGNORED_EXCEPTIONS:
+ raise
+ except Exception as e:
+ raise config_errors.ParsingError(
+ "failed to find a usable repos.conf"
+ ) from e
+ return repos_conf_defaults, repos_conf
+
@classmethod
- def load_repos_conf(cls, path):
- """parse repos.conf files
+ def parse_repos_conf_path(cls, path: str):
+ """parse repos.conf files from a given entrypoint
Args:
path (str): path to the repos.conf which can be a regular file or
@@ -372,9 +377,7 @@ class PortageConfig(DictMixin):
) from e
if defaults and main_defaults:
- logger.warning(
- f"repos.conf: parsing {fp!r}: overriding DEFAULT section"
- )
+ logger.warning("repos.conf: parsing %r: overriding DEFAULT section", fp)
main_defaults.update(defaults)
if not had_repo_conf and not repo_confs:
@@ -385,15 +388,16 @@ class PortageConfig(DictMixin):
for name, repo_conf in repo_confs.items():
if name in repos:
logger.warning(
- f"repos.conf: parsing {fp!r}: overriding {name!r} repo"
+ "repos.conf: parsing %r: overriding %r repo", fp, name
)
# ignore repo if location is unset
location = repo_conf.get("location", None)
if location is None:
logger.warning(
- f"repos.conf: parsing {fp!r}: "
- f"{name!r} repo missing location setting, ignoring repo"
+ "repos.conf: parsing %r: %r repo missing location setting, ignoring repo",
+ fp,
+ name,
)
continue
location = os.path.expanduser(location)
diff --git a/tests/ebuild/test_portage_conf.py b/tests/ebuild/test_portage_conf.py
index edb3d89b6..b939ee09e 100644
--- a/tests/ebuild/test_portage_conf.py
+++ b/tests/ebuild/test_portage_conf.py
@@ -13,7 +13,7 @@ from pkgcore.config import errors as config_errors
from pkgcore.ebuild.portage_conf import PortageConfig
load_make_conf = PortageConfig.load_make_conf
-load_repos_conf = PortageConfig.load_repos_conf
+load_repos_conf = PortageConfig.parse_repos_conf_path
class TestMakeConf:
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2023-02-02 19:58 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2023-02-02 19:58 UTC (permalink / raw
To: gentoo-commits
commit: 7c9f1da09e6660ba6f5eb4c677412c5c7c601c34
Author: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 20 08:59:38 2023 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Fri Jan 20 08:59:38 2023 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=7c9f1da0
domain: support `-*` in use as previous removal
Add support for `-*` and `USE_EXPAND: -*` syntax. Allow this syntax, and
update incremental computation of resulting use combination.
Resolves: https://github.com/pkgcore/pkgcore/issues/393
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/domain.py | 40 ++++++++++++++++++++++++----------------
src/pkgcore/ebuild/misc.py | 8 +++++++-
tests/ebuild/test_domain.py | 41 ++++++++++++++++++++++++++++++++---------
3 files changed, 63 insertions(+), 26 deletions(-)
diff --git a/src/pkgcore/ebuild/domain.py b/src/pkgcore/ebuild/domain.py
index 0dba49d5d..fd5b39fc3 100644
--- a/src/pkgcore/ebuild/domain.py
+++ b/src/pkgcore/ebuild/domain.py
@@ -69,11 +69,11 @@ def package_masks(iterable):
def restriction_payload_splitter(iterable, post_process=lambda x: x):
for line, lineno, path in iterable:
- v = line.split()
+ pkg, *flags = line.split()
try:
# TODO: expand this invocation to allow threading token level validation down.
# things like "is this a valid use flag?"
- yield parse_match(v[0]), tuple(post_process(v[1:])), line, lineno, path
+ yield parse_match(pkg), tuple(post_process(flags)), line, lineno, path
except ParseError as e:
logger.warning(f"{path!r}, line {lineno}: parsing error: {e}")
@@ -90,30 +90,40 @@ def package_use_splitter(iterable):
eapi_obj = get_latest_PMS_eapi()
def f(tokens: list[str]):
-
+ start_idx = 0
i = iter(tokens)
for idx, flag in enumerate(i):
- if flag.endswith(":"):
+ if flag == "-*":
+ start_idx = idx
+ elif flag.endswith(":"):
# we encountered `USE_EXPAND:` , thus all following tokens
# are values of that.
- x = flag.lower()[:-1]
- l = tokens[0:idx]
+ use_expand = flag.lower()[:-1]
+ yield from tokens[start_idx:idx]
+ buffer: list[str] = []
for flag in i:
if flag.endswith(":"):
- x = flag.lower()[:-1]
+ use_expand = flag.lower()[:-1]
+ yield from buffer
+ buffer.clear()
+ continue
+ if flag == "-*":
+ buffer.clear()
+ yield f"-{use_expand}_*"
continue
if flag.startswith("-"):
- flag = f"-{x}_{flag[1:]}"
+ flag = f"-{use_expand}_{flag[1:]}"
else:
- flag = f"{x}_{flag}"
+ flag = f"{use_expand}_{flag}"
if not eapi_obj.is_valid_use_flag(flag.lstrip("-")):
raise ParseError(f"token {flag} is not a valid use flag")
- l.append(flag)
- return l
+ buffer.append(flag)
+ yield from buffer
+ return
elif not eapi_obj.is_valid_use_flag(flag.lstrip("-")):
raise ParseError(f"token {flag} is not a valid use flag")
# if we made it here, there's no USE_EXPAND; thus just return the original sequence
- return tokens
+ yield from tokens[start_idx:]
return restriction_payload_splitter(iterable, post_process=f)
@@ -600,10 +610,8 @@ class domain(config_domain):
@klass.jit_attr_none
def use_expand_re(self):
- return re.compile(
- "^(?:[+-])?(%s)_(.*)$"
- % "|".join(x.lower() for x in self.profile.use_expand)
- )
+ expands = "|".join(x.lower() for x in self.profile.use_expand)
+ return re.compile(rf"^(?:[+-])?({expands})_(.*)$")
def _split_use_expand_flags(self, use_stream):
stream = ((self.use_expand_re.match(x), x) for x in use_stream)
diff --git a/src/pkgcore/ebuild/misc.py b/src/pkgcore/ebuild/misc.py
index 8c22b277f..eafcf6f60 100644
--- a/src/pkgcore/ebuild/misc.py
+++ b/src/pkgcore/ebuild/misc.py
@@ -66,8 +66,14 @@ def optimize_incrementals(sequence):
finalized.add(item)
-def incremental_chunked(orig, iterables):
+def incremental_chunked(orig: set[str], iterables):
for cinst in iterables:
+ if "*" in cinst.neg: # remove all previous set flags
+ orig.clear()
+ for flag in cinst.neg:
+ if flag.endswith("_*"): # remove previous USE_EXPAND
+ drop = [f for f in orig if f.startswith(flag[:-2])]
+ orig.difference_update(drop)
orig.difference_update(cinst.neg)
orig.update(cinst.pos)
diff --git a/tests/ebuild/test_domain.py b/tests/ebuild/test_domain.py
index e7b93b11b..54aa8f36b 100644
--- a/tests/ebuild/test_domain.py
+++ b/tests/ebuild/test_domain.py
@@ -57,15 +57,17 @@ class TestDomain:
def test_use_expand_syntax(self):
(self.pusedir / "a").write_text(
- textwrap.dedent(
- """
- */* x_y1
- # unrelated is there to verify that it's unaffected by the USE_EXPAND
- */* unrelated X: -y1 y2
- # multiple USE_EXPANDs
- */* unrelated X: -y1 y2 Z: -z3 z4
- """
- )
+ """
+ */* x_y1
+ # unrelated is there to verify that it's unaffected by the USE_EXPAND
+ */* unrelated X: -y1 y2
+ # multiple USE_EXPANDs
+ */* unrelated X: -y1 y2 Z: -z3 z4
+ # cleanup previous
+ */* x y -* z
+ # cleanup previous USE_EXPAND
+ */* unrelated Y: y1 -* y2 Z: z1 -* -z2
+ """
)
assert (
@@ -91,6 +93,27 @@ class TestDomain:
),
),
),
+ (
+ packages.AlwaysTrue,
+ (
+ ("*",),
+ ("z",),
+ ),
+ ),
+ (
+ packages.AlwaysTrue,
+ (
+ (
+ "y_*",
+ "z_*",
+ "z_z2",
+ ),
+ (
+ "unrelated",
+ "y_y2",
+ ),
+ ),
+ ),
) == self.mk_domain().pkg_use
def test_use_flag_parsing_enforcement(self, caplog):
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2023-08-29 17:37 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2023-08-29 17:37 UTC (permalink / raw
To: gentoo-commits
commit: 73b2cc0ecf48fec9242823009e0de953981913b3
Author: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
AuthorDate: Tue Aug 29 17:35:08 2023 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Tue Aug 29 17:35:08 2023 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=73b2cc0e
ebuild.repository: require .group extension for stabilization groups
After further consideration, it seems that the requiring .group extension
will make our life easier, with exact format, and save us from backup
files, readme or similar.
https://github.com/pkgcore/pkgcore/pull/412#discussion_r1307738865
Suggested-by: Michał Górny <mgorny <AT> gentoo.org>
Follows: d00711f2d6cbae14a57088ef78caa3daf72069aa
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/repository.py | 35 ++++++++++++++++++++---------------
tests/ebuild/test_repository.py | 21 +++++++++++----------
2 files changed, 31 insertions(+), 25 deletions(-)
diff --git a/src/pkgcore/ebuild/repository.py b/src/pkgcore/ebuild/repository.py
index 50f115cc1..49bd72e19 100644
--- a/src/pkgcore/ebuild/repository.py
+++ b/src/pkgcore/ebuild/repository.py
@@ -676,22 +676,27 @@ class UnconfiguredTree(prototype.tree):
@klass.jit_attr
def stabilization_groups(self):
"""Return a mapping of stabilization groups to packages."""
- stabilization_groups = {}
base_dir = pjoin(self.location, "metadata", "stabilization-groups")
- for dirname, _dirs, files in os.walk(base_dir):
- dirbase = dirname.removeprefix(base_dir)
- for file in files:
- pkgs = set()
- for lineno, line in enumerate(readlines_utf8(pjoin(dirname, file)), 1):
- try:
- if line := line.split("#", maxsplit=1)[0].strip():
- pkgs.add(atom(line))
- except ebuild_errors.MalformedAtom as exc:
- logger.error(
- f"{dirname.removeprefix(self.location)}/{file}, line {lineno}: parsing error: {exc}"
- )
- group = f"{dirbase}/{file}".removeprefix("/")
- stabilization_groups[group] = frozenset(pkgs)
+ group_files = {
+ pjoin(dirname, file)
+ .removeprefix(base_dir + "/")
+ .removesuffix(".group"): pjoin(dirname, file)
+ for dirname, _dirs, files in os.walk(base_dir)
+ for file in files
+ if file.endswith(".group")
+ }
+ stabilization_groups = {}
+ for group_name, group_file in group_files.items():
+ pkgs = set()
+ for lineno, line in enumerate(readlines_utf8(group_file), 1):
+ try:
+ if line := line.split("#", maxsplit=1)[0].strip():
+ pkgs.add(atom(line))
+ except ebuild_errors.MalformedAtom as exc:
+ logger.error(
+ f"{group_file.removeprefix(self.location)}, line {lineno}: parsing error: {exc}"
+ )
+ stabilization_groups[group_name] = frozenset(pkgs)
return ImmutableDict(stabilization_groups)
def _regen_operation_helper(self, **kwds):
diff --git a/tests/ebuild/test_repository.py b/tests/ebuild/test_repository.py
index fceb70b37..2ba4eda57 100644
--- a/tests/ebuild/test_repository.py
+++ b/tests/ebuild/test_repository.py
@@ -202,17 +202,18 @@ class TestUnconfiguredTree:
def test_stabilization_groups(self, tmp_path, caplog):
base = tmp_path / "metadata/stabilization-groups"
(base / "pkgcore").mkdir(parents=True)
- (base / "gentoo").write_text(
- textwrap.dedent(
- """\
- # some text here
- dev-python/pkgcore
- dev-python/pytest # comment
- @bad_atom@
- """
- )
+ (base / "gentoo.group").write_text(
+ """\
+ # some text here
+ dev-python/pkgcore
+ dev-python/pytest # comment
+ @bad_atom@
+ """
+ )
+ (base / "pkgcore" / "snakeoil.group").write_text(
+ """dev-python/snakeoil # comment"""
)
- (base / "pkgcore" / "snakeoil").write_text("""dev-python/snakeoil # comment""")
+ (base / "pkgcore" / "pkgdev").write_text("""dev-python/pkgdev # comment""")
mirrors = self.mk_tree(tmp_path).stabilization_groups
assert set(mirrors.keys()) == {"gentoo", "pkgcore/snakeoil"}
assert set(mirrors["gentoo"]) == {
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2023-10-23 17:35 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2023-10-23 17:35 UTC (permalink / raw
To: gentoo-commits
commit: 2f939c3e9366a4c8858892c5252b23e2f08d2447
Author: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 23 17:03:27 2023 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Mon Oct 23 17:03:27 2023 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=2f939c3e
eclassdoc: remove @BUGREPORTS tag
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/eclass.py | 9 ++++-----
tests/ebuild/test_eclass.py | 3 ---
2 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/src/pkgcore/ebuild/eclass.py b/src/pkgcore/ebuild/eclass.py
index 66e07766a..4612d0b33 100644
--- a/src/pkgcore/ebuild/eclass.py
+++ b/src/pkgcore/ebuild/eclass.py
@@ -218,7 +218,6 @@ class EclassBlock(ParseEclassDoc):
"@PROVIDES:": ("raw_provides", False, self._tag_inline_list, ()),
"@MAINTAINER:": ("maintainers", True, self._tag_multiline_args, None),
"@AUTHOR:": ("authors", False, self._tag_multiline_args, None),
- "@BUGREPORTS:": ("bugreports", False, self._tag_multiline_str, None),
"@DESCRIPTION:": ("description", False, self._tag_multiline_str, None),
"@EXAMPLE:": ("example", False, self._tag_multiline_str, None),
"@SUPPORTED_EAPIS:": ("supported_eapis", False, self._supported_eapis, ()),
@@ -642,10 +641,10 @@ class EclassDoc(AttrDict):
rst.extend(_rst_header("-", "Maintainers"))
rst.append("\n".join(f"| {x}" for x in self.maintainers))
rst.append("")
- if self.bugreports:
- rst.extend(_rst_header("-", "Bug Reports"))
- rst.append(self.bugreports)
- rst.append("")
+
+ rst.extend(_rst_header("-", "Reporting Bugs"))
+ rst.append("Please report bugs via https://bugs.gentoo.org/")
+ rst.append("")
return "\n".join(rst)
diff --git a/tests/ebuild/test_eclass.py b/tests/ebuild/test_eclass.py
index 56bf2ec39..045261f39 100644
--- a/tests/ebuild/test_eclass.py
+++ b/tests/ebuild/test_eclass.py
@@ -38,8 +38,6 @@ FOO_ECLASS = """
# @AUTHOR:
# Another Person <another@random.email>
# Random Person <maintainer@random.email>
-# @BUGREPORTS:
-# Report bugs somewhere.
# @VCSURL: https://example.com/foo.eclass
# @SUPPORTED_EAPIS: 0 1 2 3 4 5 6 7
# @PROVIDES: bar
@@ -124,7 +122,6 @@ class TestEclassDoc:
"Another Person <another@random.email>",
"Random Person <maintainer@random.email>",
)
- assert doc.bugreports == "::\n\n Report bugs somewhere."
assert doc.description == (
"::\n\n"
" Yadda yadda yadda.\n"
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/
@ 2023-12-28 5:27 Arthur Zamarin
0 siblings, 0 replies; 10+ messages in thread
From: Arthur Zamarin @ 2023-12-28 5:27 UTC (permalink / raw
To: gentoo-commits
commit: c82a051e9031149eb51a947c951c145b3c60eab7
Author: Brian Harring <ferringb <AT> gmail <DOT> com>
AuthorDate: Wed Dec 27 18:35:17 2023 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Thu Dec 28 05:27:13 2023 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=c82a051e
fix: tweak CPV parsing rules to match PMS.
Bug #421 captures this; pkgcore was allowing version components
as the package name if it was suffixed by a revision; the fix
for that incorrectly limited a package name that has a trailing
revision, but *no version syntax* in the name.
This commit just refactors the revision check to also force a check
for a leading version if revision-like is found.
Resolves: https://github.com/pkgcore/pkgcore/issues/421
Signed-off-by: Brian Harring <ferringb <AT> gmail.com>
Closes: https://github.com/pkgcore/pkgcore/pull/422
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
src/pkgcore/ebuild/cpv.py | 7 ++++++-
tests/ebuild/test_cpv.py | 1 +
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py
index b7149fc1e..39a0339ed 100644
--- a/src/pkgcore/ebuild/cpv.py
+++ b/src/pkgcore/ebuild/cpv.py
@@ -42,7 +42,12 @@ def isvalid_pkg_name(chunks):
# chunk, i.e. at least one hyphen
if len(chunks) == 1:
return True
- return not (isvalid_version_re.match(chunks[-1]) or isvalid_rev(chunks[-1]))
+ if isvalid_version_re.match(chunks[-1]):
+ return False
+ if len(chunks) >= 3 and isvalid_rev(chunks[-1]):
+ # if the last chunk is a revision, the proceeding *must not* be version like.
+ return not isvalid_version_re.match(chunks[-2])
+ return True
def isvalid_rev(s: str):
diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py
index 1a778a936..ae7d80260 100644
--- a/tests/ebuild/test_cpv.py
+++ b/tests/ebuild/test_cpv.py
@@ -64,6 +64,7 @@ class TestCPV:
"bah/f-100dpi",
"dev-util/diffball-blah-monkeys",
"virtual/7z",
+ "x11-drivers/xf86-video-r128",
)
good_vers = ("1", "2.3.4", "2.3.4a", "02.3", "2.03", "3d", "3D")
^ permalink raw reply related [flat|nested] 10+ messages in thread
end of thread, other threads:[~2023-12-28 5:27 UTC | newest]
Thread overview: 10+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-12-26 17:28 [gentoo-commits] proj/pkgcore/pkgcore:master commit in: tests/ebuild/, src/pkgcore/ebuild/ Arthur Zamarin
-- strict thread matches above, loose matches on Subject: below --
2023-12-28 5:27 Arthur Zamarin
2023-10-23 17:35 Arthur Zamarin
2023-08-29 17:37 Arthur Zamarin
2023-02-02 19:58 Arthur Zamarin
2023-01-17 20:50 Arthur Zamarin
2023-01-02 20:03 Arthur Zamarin
2022-12-26 17:28 Arthur Zamarin
2022-12-26 17:28 Arthur Zamarin
2022-12-26 17:28 Arthur Zamarin
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox