[gentoo-portage-dev] [PATCH] _select_pkg_highest_available: selective cache invalidation (530010)

[gentoo-portage-dev] [PATCH] _select_pkg_highest_available: selective cache invalidation (530010)

[Zac Medico]

Implement selective invalidation of cache for the
depgraph._select_pkg_highest_available method, so that the entire cache
is not discarded whenever a package is added to the graph. On one of my
computers with this patch, 'emerge -puvDN @world' takes 15% less time,
and results in 58% fewer _select_pkg_highest_available_imp calls.

X-Gentoo-Bug: 530010
X-Gentoo-Url: https://bugs.gentoo.org/show_bug.cgi?id=530010
---
 pym/_emerge/Package.py  | 18 ++++++++++++++++--
 pym/_emerge/depgraph.py | 14 +++++++++++++-
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py
index a09f73c..8612e8b 100644
--- a/pym/_emerge/Package.py
+++ b/pym/_emerge/Package.py
@@ -35,8 +35,8 @@ class Package(Task):
 		"category", "counter", "cp", "cpv_split",
 		"inherited", "iuse", "mtime",
 		"pf", "root", "slot", "sub_slot", "slot_atom", "version") + \
-		("_invalid", "_masks", "_metadata", "_raw_metadata", "_use",
-		"_validated_atoms", "_visible")
+		("_invalid", "_masks", "_metadata", "_provided_cps",
+		"_raw_metadata", "_use", "_validated_atoms", "_visible")
 
 	metadata_keys = [
 		"BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "EAPI",
@@ -128,6 +128,20 @@ class Package(Task):
 		return self._metadata.properties
 
 	@property
+	def provided_cps(self):
+
+		if self._provided_cps is None:
+			provided_cps = [self.cp]
+			for atom in self._metadata["PROVIDE"].split():
+				try:
+					provided_cps.append(Atom(atom).cp)
+				except InvalidAtom:
+					pass
+			self._provided_cps = tuple(provided_cps)
+
+		return self._provided_cps
+
+	@property
 	def restrict(self):
 		return self._metadata.restrict
 
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 3f4a097..ab79aef 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -399,6 +399,7 @@ class _dynamic_depgraph_config(object):
 		self._initially_unsatisfied_deps = []
 		self._ignored_deps = []
 		self._highest_pkg_cache = {}
+		self._highest_pkg_cache_cp_map = {}
 		self._flatten_atoms_cache = {}
 
 		# Binary packages that have been rejected because their USE
@@ -2539,8 +2540,8 @@ class depgraph(object):
 			if not previously_added:
 				self._dynamic_config._package_tracker.add_pkg(pkg)
 				self._dynamic_config._filtered_trees[pkg.root]["porttree"].dbapi._clear_cache()
-				self._dynamic_config._highest_pkg_cache.clear()
 				self._check_masks(pkg)
+				self._prune_highest_pkg_cache(pkg)
 
 			if not pkg.installed:
 				# Allow this package to satisfy old-style virtuals in case it
@@ -2685,6 +2686,7 @@ class depgraph(object):
 		# Clear caches.
 		self._dynamic_config._filtered_trees[pkg.root]["porttree"].dbapi._clear_cache()
 		self._dynamic_config._highest_pkg_cache.clear()
+		self._dynamic_config._highest_pkg_cache_cp_map.clear()
 
 
 	def _check_masks(self, pkg):
@@ -3967,6 +3969,7 @@ class depgraph(object):
 		# Invalidate the package selection cache, since
 		# arguments influence package selections.
 		self._dynamic_config._highest_pkg_cache.clear()
+		self._dynamic_config._highest_pkg_cache_cp_map.clear()
 		for trees in self._dynamic_config._filtered_trees.values():
 			trees["porttree"].dbapi._clear_cache()
 
@@ -4987,6 +4990,8 @@ class depgraph(object):
 			return ret
 		ret = self._select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps, parent=parent)
 		self._dynamic_config._highest_pkg_cache[cache_key] = ret
+		self._dynamic_config._highest_pkg_cache_cp_map. \
+			setdefault(atom.cp, []).append(cache_key)
 		pkg, existing = ret
 		if pkg is not None:
 			if self._pkg_visibility_check(pkg) and \
@@ -4994,6 +4999,13 @@ class depgraph(object):
 				self._dynamic_config._visible_pkgs[pkg.root].cpv_inject(pkg)
 		return ret
 
+	def _prune_highest_pkg_cache(self, pkg):
+		for cp in pkg.provided_cps:
+			for cache_key in self._dynamic_config. \
+				_highest_pkg_cache_cp_map.pop(cp, []):
+				self._dynamic_config._highest_pkg_cache.pop(
+					cache_key, None)
+
 	def _want_installed_pkg(self, pkg):
 		"""
 		Given an installed package returned from select_pkg, return
-- 
2.0.4

Re: [gentoo-portage-dev] [PATCH] _select_pkg_highest_available: selective cache invalidation (530010)

[Alexander Berntsen]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

ACK.

We should really have a benchmark suite/some servers to run
benchmarking on... But for now it's nice that you record your results
in the commit message.
- -- 
Alexander
bernalex@gentoo.org
https://secure.plaimi.net/~alexander
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/

iF4EAREIAAYFAlRy2XoACgkQRtClrXBQc7V8WAEAnVhmsLJmXlck/+ImpnUFJ675
Qf8jjsGkVFP7lqK7pTEBAJy7TGvgskbXs7pGW1L3cQ+/yqV4BoTZI/6QnwXQqmTe
=GKet
-----END PGP SIGNATURE-----