[gentoo-portage-dev] [PATCH] egencache: add --external-cache-only option (bug 741474)

[gentoo-portage-dev] [PATCH] egencache: add --external-cache-only option (bug 741474)

[Zac Medico]

The --external-cache-only option is useful for client-side
use cases where writing cache files inside the repository itself
may interfere with repository verification. This option is
currently supported for --update and --update-pkg-desc-index
actions, for which consumers of the corresponding cache or
index files are already capable of consuming files from the
external cache directory (/var/cache/edb/dep).

Bug: https://bugs.gentoo.org/737470
Signed-off-by: Zac Medico <zmedico@gentoo.org>
---
 bin/egencache               | 44 ++++++++++++++++++++++---------------
 cnf/repo.postsync.d/example | 19 ++++++++--------
 man/egencache.1             |  7 ++++++
 3 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/bin/egencache b/bin/egencache
index 532e37f20..4ee63edad 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -107,6 +107,9 @@ def parse_args(args):
 	common.add_argument("--config-root",
 		help="location of portage config files",
 		dest="portage_configroot")
+	common.add_argument("--external-cache-only",
+		action="store_true",
+		help="Output only to the external cache (not the repository itself)")
 	common.add_argument("--gpg-dir",
 		help="override the PORTAGE_GPG_DIR variable",
 		dest="gpg_dir")
@@ -246,7 +249,7 @@ def parse_args(args):
 
 class GenCache:
 	def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None,
-		rsync=False):
+		rsync=False, external_cache_only=False):
 		# The caller must set portdb.porttrees in order to constrain
 		# findname, cp_list, and cpv_list to the desired tree.
 		tree = portdb.porttrees[0]
@@ -263,18 +266,21 @@ class GenCache:
 		else:
 			self._cp_set = None
 			self._cp_missing = set()
-		write_auxdb = "metadata-transfer" in portdb.settings.features
+		write_auxdb = external_cache_only or "metadata-transfer" in portdb.settings.features
 		self._regen = MetadataRegen(portdb, cp_iter=cp_iter,
 			consumer=self._metadata_callback,
 			max_jobs=max_jobs, max_load=max_load,
 			write_auxdb=write_auxdb, main=True)
 		self.returncode = os.EX_OK
 		conf = portdb.repositories.get_repo_for_location(tree)
-		self._trg_caches = tuple(conf.iter_pregenerated_caches(
-			self._auxdbkeys, force=True, readonly=False))
-		if not self._trg_caches:
-			raise Exception("cache formats '%s' aren't supported" %
-				(" ".join(conf.cache_formats),))
+		if external_cache_only:
+			self._trg_caches = ()
+		else:
+			self._trg_caches = tuple(conf.iter_pregenerated_caches(
+				self._auxdbkeys, force=True, readonly=False))
+			if not self._trg_caches:
+				raise Exception("cache formats '%s' aren't supported" %
+					(" ".join(conf.cache_formats),))
 
 		if rsync:
 			for trg_cache in self._trg_caches:
@@ -1092,7 +1098,8 @@ def egencache_main(args):
 		gen_cache = GenCache(portdb, cp_iter=cp_iter,
 			max_jobs=options.jobs,
 			max_load=options.load_average,
-			rsync=options.rsync)
+			rsync=options.rsync,
+			external_cache_only=options.external_cache_only)
 		gen_cache.run()
 		if options.tolerant:
 			ret.append(os.EX_OK)
@@ -1100,20 +1107,21 @@ def egencache_main(args):
 			ret.append(gen_cache.returncode)
 
 	if options.update_pkg_desc_index:
-		if repo_config.writable:
+		if not options.external_cache_only and repo_config.writable:
 			writable_location = repo_config.location
 		else:
 			writable_location = os.path.join(portdb.depcachedir,
 				repo_config.location.lstrip(os.sep))
-			msg = [
-				"WARNING: Repository is not writable: %s" % (
-				repo_config.location,),
-				"         Using cache directory instead: %s" % (
-				writable_location,)
-			]
-			msg = "".join(line + '\n' for line in msg)
-			writemsg_level(msg,
-				level=logging.WARNING, noiselevel=-1)
+			if not options.external_cache_only:
+				msg = [
+					"WARNING: Repository is not writable: %s" % (
+					repo_config.location,),
+					"         Using cache directory instead: %s" % (
+					writable_location,)
+				]
+				msg = "".join(line + '\n' for line in msg)
+				writemsg_level(msg,
+					level=logging.WARNING, noiselevel=-1)
 
 		gen_index = GenPkgDescIndex(repo_config, portdb, os.path.join(
 			writable_location, "metadata", "pkg_desc_index"),
diff --git a/cnf/repo.postsync.d/example b/cnf/repo.postsync.d/example
index d1e385c11..f7c6f5092 100644
--- a/cnf/repo.postsync.d/example
+++ b/cnf/repo.postsync.d/example
@@ -46,17 +46,16 @@ if [ -n "${repository_name}" ]; then
 		fi
 	fi
 
-	# Regenerate the metadata/pkg_desc_index file if needed. It's not
+	# Regenerate the metadata/pkg_desc_index file. This is not
 	# needed for https://gitweb.gentoo.org/repo/sync/gentoo.git which
-	# provides a freshly generated copy.
-	if [[ ! -e ${repository_path}/metadata/pkg_desc_index || (
-		-d ${repository_path}/metadata/md5-cache &&
-		-n $(find "${repository_path}/metadata/md5-cache" -type f -newer "${repository_path}/metadata/pkg_desc_index" -print -quit) ) ]]; then
-		if ! egencache --update-pkg-desc-index --repo="${repository_name}" ${PORTAGE_VERBOSE+--verbose}
-		then
-			echo "!!! egencache failed!"
-			ret=1
-		fi
+	# provides a freshly generated copy. The --external-cache-only
+	# option causes the metadata/pkg_desc_index file to be written under
+	# /var/cache/edb/dep instead of the repository itself, so that it
+	# does not interfere with repository verification.
+	if ! egencache --update-pkg-desc-index --external-cache-only --repo="${repository_name}" ${PORTAGE_VERBOSE+--verbose}
+	then
+		echo "!!! egencache failed!"
+		ret=1
 	fi
 fi
 
diff --git a/man/egencache.1 b/man/egencache.1
index ae7370e21..9826942d2 100644
--- a/man/egencache.1
+++ b/man/egencache.1
@@ -55,6 +55,13 @@ Location of portage config files.
 .br
 Defaults to /.
 .TP
+.BR "\-\-external\-cache\-only"
+Output to the external cache rather than to the repository itself.
+This is useful for client\-side usage, where it may not be possible to
+output to the repository itself since that would interfere with
+repository verification. This option is supported by the \-\-update and
+\-\-update\-pkg\-desc\-index actions.
+.TP
 .BR "\-\-gpg\-dir"
 Override the PORTAGE_GPG_DIR variable.
 .TP
-- 
2.25.3