[gentoo-commits] proj/portage:master commit in: pym/_emerge/

["Zac Medico" <zmedico@gentoo.org>]

commit:     99ec2a8f810ae7ea2c76d928665ed1d02c2d9cc7
Author:     David James <davidjames <AT> google <DOT> com>
AuthorDate: Fri Mar 25 01:14:45 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 01:14:45 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=99ec2a8f

Update Portage to sync BlockerDB at init.

Currently, Portage syncs the BlockerDB before every merge. This slows
down merges considerably because it requires reading the entire vardb
from disk. Since Portage doesn't merge conflicting packages anyway, we
can optimize this by just reading the vardb at initialization and
caching that.

Change-Id: I6701926f022ef3aa2da10482fc8b09573ae24610

Review URL: http://codereview.chromium.org/6688037

---
 pym/_emerge/BlockerDB.py   |   13 ++++++-------
 pym/_emerge/FakeVartree.py |   18 ++++++++++++------
 pym/_emerge/Scheduler.py   |   24 +++++++-----------------
 3 files changed, 25 insertions(+), 30 deletions(-)

diff --git a/pym/_emerge/BlockerDB.py b/pym/_emerge/BlockerDB.py
index fd4bf5e..f5adc4d 100644
--- a/pym/_emerge/BlockerDB.py
+++ b/pym/_emerge/BlockerDB.py
@@ -30,12 +30,7 @@ class BlockerDB(object):
 				"vartree"     :  fake_vartree,
 		}}
 
-	def _get_fake_vartree(self, acquire_lock=0):
-		fake_vartree = self._fake_vartree
-		fake_vartree.sync(acquire_lock=acquire_lock)
-		return fake_vartree
-
-	def findInstalledBlockers(self, new_pkg, acquire_lock=0):
+	def findInstalledBlockers(self, new_pkg):
 		"""
 		Search for installed run-time blockers in the root where
 		new_pkg is planned to be installed. This ignores build-time
@@ -45,7 +40,7 @@ class BlockerDB(object):
 		dep_keys = ["RDEPEND", "PDEPEND"]
 		settings = self._vartree.settings
 		stale_cache = set(blocker_cache)
-		fake_vartree = self._get_fake_vartree(acquire_lock=acquire_lock)
+		fake_vartree = self._fake_vartree
 		dep_check_trees = self._dep_check_trees
 		vardb = fake_vartree.dbapi
 		installed_pkgs = list(vardb)
@@ -118,3 +113,7 @@ class BlockerDB(object):
 
 		return blocking_pkgs
 
+	def discardBlocker(self, pkg):
+		"""Discard a package from the list of potential blockers."""
+		self._fake_vartree.cpv_discard(pkg)
+

diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py
index f7f292b..fa490c2 100644
--- a/pym/_emerge/FakeVartree.py
+++ b/pym/_emerge/FakeVartree.py
@@ -103,6 +103,16 @@ class FakeVartree(vartree):
 				pkg, self.dbapi, self._global_updates)
 		return self._aux_get(pkg, wants)
 
+	def cpv_discard(self, pkg):
+		"""
+		Discard a package from the fake vardb if it exists.
+		"""
+		old_pkg = self.dbapi.get(pkg)
+		if old_pkg is not None:
+			self.dbapi.cpv_remove(old_pkg)
+			self._pkg_cache.pop(old_pkg, None)
+			self._aux_get_history.discard(old_pkg.cpv)
+
 	def sync(self, acquire_lock=1):
 		"""
 		Call this method to synchronize state with the real vardb
@@ -144,9 +154,7 @@ class FakeVartree(vartree):
 		# Remove any packages that have been uninstalled.
 		for pkg in list(pkg_vardb):
 			if pkg.cpv not in current_cpv_set:
-				pkg_vardb.cpv_remove(pkg)
-				pkg_cache.pop(pkg, None)
-				aux_get_history.discard(pkg.cpv)
+				self.cpv_discard(pkg)
 
 		# Validate counters and timestamps.
 		slot_counters = {}
@@ -165,9 +173,7 @@ class FakeVartree(vartree):
 
 				if counter != pkg.counter or \
 					mtime != pkg.mtime:
-					pkg_vardb.cpv_remove(pkg)
-					pkg_cache.pop(pkg, None)
-					aux_get_history.discard(pkg.cpv)
+					self.cpv_discard(pkg)
 					pkg = None
 
 			if pkg is None:

diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index 9215678..7d34d70 100644
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -317,10 +317,6 @@ class Scheduler(PollScheduler):
 		Initialization structures used for dependency calculations
 		involving currently installed packages.
 		"""
-		# TODO: Replace the BlockerDB with a depgraph of installed packages
-		# that's updated incrementally with each upgrade/uninstall operation
-		# This will be useful for making quick and safe decisions with respect
-		# to aggressive parallelization discussed in bug #279623.
 		self._set_graph_config(graph_config)
 		self._blocker_db = {}
 		for root in self.trees:
@@ -329,6 +325,7 @@ class Scheduler(PollScheduler):
 					pkg_cache=self._pkg_cache)
 			else:
 				fake_vartree = graph_config.trees[root]['vartree']
+			fake_vartree.sync()
 			self._blocker_db[root] = BlockerDB(fake_vartree)
 
 	def _destroy_graph(self):
@@ -643,27 +640,20 @@ class Scheduler(PollScheduler):
 
 	def _find_blockers(self, new_pkg):
 		"""
-		Returns a callable which should be called only when
-		the vdb lock has been acquired.
+		Returns a callable.
 		"""
 		def get_blockers():
-			return self._find_blockers_with_lock(new_pkg, acquire_lock=0)
+			return self._find_blockers_impl(new_pkg)
 		return get_blockers
 
-	def _find_blockers_with_lock(self, new_pkg, acquire_lock=0):
+	def _find_blockers_impl(self, new_pkg):
 		if self._opts_ignore_blockers.intersection(self.myopts):
 			return None
 
-		# Call gc.collect() here to avoid heap overflow that
-		# triggers 'Cannot allocate memory' errors (reported
-		# with python-2.5).
-		gc.collect()
-
 		blocker_db = self._blocker_db[new_pkg.root]
 
 		blocker_dblinks = []
-		for blocking_pkg in blocker_db.findInstalledBlockers(
-			new_pkg, acquire_lock=acquire_lock):
+		for blocking_pkg in blocker_db.findInstalledBlockers(new_pkg):
 			if new_pkg.slot_atom == blocking_pkg.slot_atom:
 				continue
 			if new_pkg.cpv == blocking_pkg.cpv:
@@ -673,8 +663,6 @@ class Scheduler(PollScheduler):
 				self.pkgsettings[blocking_pkg.root], treetype="vartree",
 				vartree=self.trees[blocking_pkg.root]["vartree"]))
 
-		gc.collect()
-
 		return blocker_dblinks
 
 	def _dblink_pkg(self, pkg_dblink):
@@ -1529,6 +1517,8 @@ class Scheduler(PollScheduler):
 		self._completed_tasks.add(pkg)
 		self._unsatisfied_system_deps.discard(pkg)
 		self._choose_pkg_return_early = False
+		blocker_db = self._blocker_db[pkg.root]
+		blocker_db.discardBlocker(pkg)
 
 	def _merge(self):