[gentoo-portage-dev] [PATCH 2/2] bintree: Add REPO_REVISIONS to package index header

[Zac Medico <zmedico@gentoo.org>]

As a means for binhost clients to select source repo
revisions which are consistent with binhosts, inject
REPO_REVISIONS from a package into the index header,
using a history of synced revisions to guarantee
forward progress. This queries the relevant repos to
check if any new revisions have appeared in the
absence of a proper sync operation.

Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico <zmedico@gentoo.org>
---
 lib/portage/dbapi/bintree.py              | 66 ++++++++++++++++++++-
 lib/portage/tests/sync/test_sync_local.py | 71 +++++++++++++++++++----
 2 files changed, 123 insertions(+), 14 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7bc1f60f6d..fbf60e74eb 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -48,6 +48,7 @@ from portage.exception import (
 from portage.localization import _
 from portage.output import colorize
 from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.sync.revision_history import get_repo_revision_history
 from portage.util import ensure_dirs
 from portage.util.file_copy import copyfile
 from portage.util.futures import asyncio
@@ -62,6 +63,7 @@ from portage import _unicode_encode
 import codecs
 import errno
 import io
+import json
 import re
 import stat
 import subprocess
@@ -134,13 +136,19 @@ class bindbapi(fakedbapi):
             "USE",
             "_mtime_",
         }
+        # Keys required only when initially adding a package.
+        self._init_aux_keys = {
+            "REPO_REVISIONS",
+        }
         self._aux_cache = {}
         self._aux_cache_slot_dict_cache = None
 
     @property
     def _aux_cache_slot_dict(self):
         if self._aux_cache_slot_dict_cache is None:
-            self._aux_cache_slot_dict_cache = slot_dict_class(self._aux_cache_keys)
+            self._aux_cache_slot_dict_cache = slot_dict_class(
+                chain(self._aux_cache_keys, self._init_aux_keys)
+            )
         return self._aux_cache_slot_dict_cache
 
     def __getstate__(self):
@@ -1791,6 +1799,10 @@ class binarytree:
                 pkgindex = self._new_pkgindex()
 
             d = self._inject_file(pkgindex, cpv, full_path)
+            repo_revisions = d.get("REPO_REVISIONS")
+            if repo_revisions:
+                repo_revisions = json.loads(repo_revisions)
+                self._inject_repo_revisions(pkgindex.header, repo_revisions)
             self._update_pkgindex_header(pkgindex.header)
             self._pkgindex_write(pkgindex)
 
@@ -1872,7 +1884,7 @@ class binarytree:
         @return: package metadata
         """
         if keys is None:
-            keys = self.dbapi._aux_cache_keys
+            keys = chain(self.dbapi._aux_cache_keys, self.dbapi._init_aux_keys)
             metadata = self.dbapi._aux_cache_slot_dict()
         else:
             metadata = {}
@@ -1916,6 +1928,56 @@ class binarytree:
 
         return metadata
 
+    def _inject_repo_revisions(self, header, repo_revisions):
+        """
+        Inject REPO_REVISIONS from a package into the index header,
+        using a history of synced revisions to guarantee forward
+        progress. This queries the relevant repos to check if any
+        new revisions have appeared in the absence of a proper sync
+        operation.
+
+        This does not expose REPO_REVISIONS that do not appear in
+        the sync history, since such revisions suggest that the
+        package was not built locally, and in this case its
+        REPO_REVISIONS are not intended to be exposed.
+        """
+        synced_repo_revisions = get_repo_revision_history(
+            self.settings["EROOT"],
+            [self.settings.repositories[repo_name] for repo_name in repo_revisions],
+        )
+        header_repo_revisions = (
+            json.loads(header["REPO_REVISIONS"]) if header.get("REPO_REVISIONS") else {}
+        )
+        for repo_name, repo_revision in repo_revisions.items():
+            rev_list = synced_repo_revisions.get(repo_name, [])
+            header_rev = header_repo_revisions.get(repo_name)
+            if not rev_list or header_rev in (repo_revision, rev_list[0]):
+                continue
+            try:
+                header_rev_index = (
+                    None if header_rev is None else rev_list.index(header_rev)
+                )
+            except ValueError:
+                header_rev_index = None
+            try:
+                repo_revision_index = rev_list.index(repo_revision)
+            except ValueError:
+                repo_revision_index = None
+            if repo_revision_index is not None and (
+                header_rev_index is None or repo_revision_index < header_rev_index
+            ):
+                # There is forward progress when repo_revision is more recent
+                # than header_rev or header_rev was not found in the history.
+                # Do not expose repo_revision here if it does not appear in
+                # the history, since this suggests that the package was not
+                # built locally and in this case its REPO_REVISIONS are not
+                # intended to be exposed here.
+                header_repo_revisions[repo_name] = repo_revision
+        if header_repo_revisions:
+            header["REPO_REVISIONS"] = json.dumps(
+                header_repo_revisions, ensure_ascii=False, sort_keys=True
+            )
+
     def _inject_file(self, pkgindex, cpv, filename):
         """
         Add a package to internal data structures, and add an
diff --git a/lib/portage/tests/sync/test_sync_local.py b/lib/portage/tests/sync/test_sync_local.py
index 91649398de..7e6158ee45 100644
--- a/lib/portage/tests/sync/test_sync_local.py
+++ b/lib/portage/tests/sync/test_sync_local.py
@@ -380,6 +380,45 @@ class SyncLocalTestCase(TestCase):
             (homedir, lambda: self.assertTrue(bool(get_revision_history()))),
         )
 
+        def assert_latest_rev_in_packages_index(positive):
+            """
+            If we build a binary package then its REPO_REVISIONS should
+            propagate into $PKGDIR/Packages as long as it results in
+            forward progress according to the repo revision history.
+            """
+            revision_history = get_revision_history()
+            prefix = "REPO_REVISIONS:"
+            header_repo_revisions = None
+            try:
+                with open(
+                    os.path.join(settings["PKGDIR"], "Packages"), encoding="utf8"
+                ) as f:
+                    for line in f:
+                        if line.startswith(prefix):
+                            header_repo_revisions = line[len(prefix) :].strip()
+                            break
+            except FileNotFoundError:
+                pass
+
+            if positive:
+                self.assertFalse(header_repo_revisions is None)
+
+            if header_repo_revisions is None:
+                header_repo_revisions = {}
+            else:
+                header_repo_revisions = json.loads(header_repo_revisions)
+
+            (self.assertEqual if positive else self.assertNotEqual)(
+                revision_history.get(repo.name, [False])[0],
+                header_repo_revisions.get(repo.name, None),
+            )
+
+        pkgindex_revisions_cmds = (
+            (homedir, lambda: assert_latest_rev_in_packages_index(False)),
+            (homedir, cmds["emerge"] + ("-B", "dev-libs/A")),
+            (homedir, lambda: assert_latest_rev_in_packages_index(True)),
+        )
+
         def hg_init_global_config():
             with open(os.path.join(homedir, ".hgrc"), "w") as f:
                 f.write(f"[ui]\nusername = {committer_name} <{committer_email}>\n")
@@ -447,18 +486,25 @@ class SyncLocalTestCase(TestCase):
                 pythonpath = ":" + pythonpath
             pythonpath = PORTAGE_PYM_PATH + pythonpath
 
-        env = {
-            "PORTAGE_OVERRIDE_EPREFIX": eprefix,
-            "DISTDIR": distdir,
-            "GENTOO_COMMITTER_NAME": committer_name,
-            "GENTOO_COMMITTER_EMAIL": committer_email,
-            "HOME": homedir,
-            "PATH": settings["PATH"],
-            "PORTAGE_GRPNAME": os.environ["PORTAGE_GRPNAME"],
-            "PORTAGE_USERNAME": os.environ["PORTAGE_USERNAME"],
-            "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
-            "PYTHONPATH": pythonpath,
-        }
+        env = settings.environ()
+        env.update(
+            {
+                "PORTAGE_OVERRIDE_EPREFIX": eprefix,
+                "DISTDIR": distdir,
+                "GENTOO_COMMITTER_NAME": committer_name,
+                "GENTOO_COMMITTER_EMAIL": committer_email,
+                "HOME": homedir,
+                "PORTAGE_INST_GID": str(os.getgid()),
+                "PORTAGE_INST_UID": str(os.getuid()),
+                "PORTAGE_GRPNAME": os.environ["PORTAGE_GRPNAME"],
+                "PORTAGE_USERNAME": os.environ["PORTAGE_USERNAME"],
+                "PYTHONDONTWRITEBYTECODE": os.environ.get(
+                    "PYTHONDONTWRITEBYTECODE", ""
+                ),
+                "PYTHONPATH": pythonpath,
+            }
+        )
+
         repos_set_conf("rsync")
 
         if os.environ.get("SANDBOX_ON") == "1":
@@ -518,6 +564,7 @@ class SyncLocalTestCase(TestCase):
                 + upstream_git_commit
                 + sync_cmds
                 + repo_revisions_cmds
+                + pkgindex_revisions_cmds
                 + mercurial_tests
             ):
                 if hasattr(cmd, "__call__"):
-- 
2.41.0