* [gentoo-commits] proj/portage:master commit in: lib/portage/tests/sync/, lib/portage/dbapi/
@ 2024-05-25 23:14 Zac Medico
0 siblings, 0 replies; only message in thread
From: Zac Medico @ 2024-05-25 23:14 UTC (permalink / raw
To: gentoo-commits
commit: 5aed7289d516fab5b63557da46348125eabab368
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 14 04:09:34 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 25 22:08:15 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=5aed7289
bintree: Add REPO_REVISIONS to package index header
As a means for binhost clients to select source repo
revisions which are consistent with binhosts, inject
REPO_REVISIONS from a package into the index header,
using a history of synced revisions to guarantee
forward progress. This queries the relevant repos to
check if any new revisions have appeared in the
absence of a proper sync operation.
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 67 ++++++++++++++++++++++++++++-
lib/portage/tests/sync/test_sync_local.py | 71 +++++++++++++++++++++++++------
2 files changed, 124 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 221afbd154..64dfee4faa 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -48,6 +48,7 @@ from portage.exception import (
from portage.localization import _
from portage.output import colorize
from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.sync.revision_history import get_repo_revision_history
from portage.util import ensure_dirs
from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
@@ -62,6 +63,7 @@ from portage import _unicode_encode
import codecs
import errno
import io
+import json
import re
import shlex
import stat
@@ -135,13 +137,19 @@ class bindbapi(fakedbapi):
"USE",
"_mtime_",
}
+ # Keys required only when initially adding a package.
+ self._init_aux_keys = {
+ "REPO_REVISIONS",
+ }
self._aux_cache = {}
self._aux_cache_slot_dict_cache = None
@property
def _aux_cache_slot_dict(self):
if self._aux_cache_slot_dict_cache is None:
- self._aux_cache_slot_dict_cache = slot_dict_class(self._aux_cache_keys)
+ self._aux_cache_slot_dict_cache = slot_dict_class(
+ chain(self._aux_cache_keys, self._init_aux_keys)
+ )
return self._aux_cache_slot_dict_cache
def __getstate__(self):
@@ -1791,6 +1799,11 @@ class binarytree:
pkgindex = self._new_pkgindex()
d = self._inject_file(pkgindex, cpv, full_path)
+ repo_revisions = (
+ json.loads(d["REPO_REVISIONS"]) if d.get("REPO_REVISIONS") else None
+ )
+ if repo_revisions:
+ self._inject_repo_revisions(pkgindex.header, repo_revisions)
self._update_pkgindex_header(pkgindex.header)
self._pkgindex_write(pkgindex)
@@ -1872,7 +1885,7 @@ class binarytree:
@return: package metadata
"""
if keys is None:
- keys = self.dbapi._aux_cache_keys
+ keys = chain(self.dbapi._aux_cache_keys, self.dbapi._init_aux_keys)
metadata = self.dbapi._aux_cache_slot_dict()
else:
metadata = {}
@@ -1916,6 +1929,56 @@ class binarytree:
return metadata
+ def _inject_repo_revisions(self, header, repo_revisions):
+ """
+ Inject REPO_REVISIONS from a package into the index header,
+ using a history of synced revisions to guarantee forward
+ progress. This queries the relevant repos to check if any
+ new revisions have appeared in the absence of a proper sync
+ operation.
+
+ This does not expose REPO_REVISIONS that do not appear in
+ the sync history, since such revisions suggest that the
+ package was not built locally, and in this case its
+ REPO_REVISIONS are not intended to be exposed.
+ """
+ synced_repo_revisions = get_repo_revision_history(
+ self.settings["EROOT"],
+ [self.settings.repositories[repo_name] for repo_name in repo_revisions],
+ )
+ header_repo_revisions = (
+ json.loads(header["REPO_REVISIONS"]) if header.get("REPO_REVISIONS") else {}
+ )
+ for repo_name, repo_revision in repo_revisions.items():
+ rev_list = synced_repo_revisions.get(repo_name, [])
+ header_rev = header_repo_revisions.get(repo_name)
+ if not rev_list or header_rev in (repo_revision, rev_list[0]):
+ continue
+ try:
+ header_rev_index = (
+ None if header_rev is None else rev_list.index(header_rev)
+ )
+ except ValueError:
+ header_rev_index = None
+ try:
+ repo_revision_index = rev_list.index(repo_revision)
+ except ValueError:
+ repo_revision_index = None
+ if repo_revision_index is not None and (
+ header_rev_index is None or repo_revision_index < header_rev_index
+ ):
+ # There is forward progress when repo_revision is more recent
+ # than header_rev or header_rev was not found in the history.
+ # Do not expose repo_revision here if it does not appear in
+ # the history, since this suggests that the package was not
+ # built locally and in this case its REPO_REVISIONS are not
+ # intended to be exposed here.
+ header_repo_revisions[repo_name] = repo_revision
+ if header_repo_revisions:
+ header["REPO_REVISIONS"] = json.dumps(
+ header_repo_revisions, ensure_ascii=False, sort_keys=True
+ )
+
def _inject_file(self, pkgindex, cpv, filename):
"""
Add a package to internal data structures, and add an
diff --git a/lib/portage/tests/sync/test_sync_local.py b/lib/portage/tests/sync/test_sync_local.py
index 91649398de..7e6158ee45 100644
--- a/lib/portage/tests/sync/test_sync_local.py
+++ b/lib/portage/tests/sync/test_sync_local.py
@@ -380,6 +380,45 @@ class SyncLocalTestCase(TestCase):
(homedir, lambda: self.assertTrue(bool(get_revision_history()))),
)
+ def assert_latest_rev_in_packages_index(positive):
+ """
+ If we build a binary package then its REPO_REVISIONS should
+ propagate into $PKGDIR/Packages as long as it results in
+ forward progress according to the repo revision history.
+ """
+ revision_history = get_revision_history()
+ prefix = "REPO_REVISIONS:"
+ header_repo_revisions = None
+ try:
+ with open(
+ os.path.join(settings["PKGDIR"], "Packages"), encoding="utf8"
+ ) as f:
+ for line in f:
+ if line.startswith(prefix):
+ header_repo_revisions = line[len(prefix) :].strip()
+ break
+ except FileNotFoundError:
+ pass
+
+ if positive:
+ self.assertFalse(header_repo_revisions is None)
+
+ if header_repo_revisions is None:
+ header_repo_revisions = {}
+ else:
+ header_repo_revisions = json.loads(header_repo_revisions)
+
+ (self.assertEqual if positive else self.assertNotEqual)(
+ revision_history.get(repo.name, [False])[0],
+ header_repo_revisions.get(repo.name, None),
+ )
+
+ pkgindex_revisions_cmds = (
+ (homedir, lambda: assert_latest_rev_in_packages_index(False)),
+ (homedir, cmds["emerge"] + ("-B", "dev-libs/A")),
+ (homedir, lambda: assert_latest_rev_in_packages_index(True)),
+ )
+
def hg_init_global_config():
with open(os.path.join(homedir, ".hgrc"), "w") as f:
f.write(f"[ui]\nusername = {committer_name} <{committer_email}>\n")
@@ -447,18 +486,25 @@ class SyncLocalTestCase(TestCase):
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
- env = {
- "PORTAGE_OVERRIDE_EPREFIX": eprefix,
- "DISTDIR": distdir,
- "GENTOO_COMMITTER_NAME": committer_name,
- "GENTOO_COMMITTER_EMAIL": committer_email,
- "HOME": homedir,
- "PATH": settings["PATH"],
- "PORTAGE_GRPNAME": os.environ["PORTAGE_GRPNAME"],
- "PORTAGE_USERNAME": os.environ["PORTAGE_USERNAME"],
- "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
- "PYTHONPATH": pythonpath,
- }
+ env = settings.environ()
+ env.update(
+ {
+ "PORTAGE_OVERRIDE_EPREFIX": eprefix,
+ "DISTDIR": distdir,
+ "GENTOO_COMMITTER_NAME": committer_name,
+ "GENTOO_COMMITTER_EMAIL": committer_email,
+ "HOME": homedir,
+ "PORTAGE_INST_GID": str(os.getgid()),
+ "PORTAGE_INST_UID": str(os.getuid()),
+ "PORTAGE_GRPNAME": os.environ["PORTAGE_GRPNAME"],
+ "PORTAGE_USERNAME": os.environ["PORTAGE_USERNAME"],
+ "PYTHONDONTWRITEBYTECODE": os.environ.get(
+ "PYTHONDONTWRITEBYTECODE", ""
+ ),
+ "PYTHONPATH": pythonpath,
+ }
+ )
+
repos_set_conf("rsync")
if os.environ.get("SANDBOX_ON") == "1":
@@ -518,6 +564,7 @@ class SyncLocalTestCase(TestCase):
+ upstream_git_commit
+ sync_cmds
+ repo_revisions_cmds
+ + pkgindex_revisions_cmds
+ mercurial_tests
):
if hasattr(cmd, "__call__"):
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2024-05-25 23:14 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-05-25 23:14 [gentoo-commits] proj/portage:master commit in: lib/portage/tests/sync/, lib/portage/dbapi/ Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox