[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/cache/, pym/portage/, ...

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/cache/, pym/portage/, ...

[Zac Medico]

commit:     386dc3d36b086d8d16fe396bff00b3f13f4c18f9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun  8 01:32:01 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun  8 01:32:44 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=386dc3d3

cache/template.py: call self.sync() from __del__

This allows portdbapi.portdbapi_instances to be eliminated, which is
nice because we no longer has to be so careful to avoid memory leaks
involving this variable. It was not just annoying for portage
internals, but also for any API consumers that needed to create/destroy
many portdbapi instances.

---
 pym/portage/__init__.py                          | 36 +-----------------------
 pym/portage/cache/template.py                    | 13 +++++++--
 pym/portage/dbapi/porttree.py                    | 27 ++++++++++++------
 pym/portage/tests/resolver/ResolverPlayground.py |  6 +---
 4 files changed, 31 insertions(+), 51 deletions(-)

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index f326ecb..2510f86 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -549,7 +549,7 @@ auxdbkeys = (
 auxdbkeylen=len(auxdbkeys)
 
 def portageexit():
-	close_portdbapi_caches()
+	pass
 
 class _trees_dict(dict):
 	__slots__ = ('_running_eroot', '_target_eroot',)
@@ -560,13 +560,6 @@ class _trees_dict(dict):
 
 def create_trees(config_root=None, target_root=None, trees=None, env=None,
 	eprefix=None):
-	if trees is not None:
-		# clean up any existing portdbapi instances
-		for myroot in trees:
-			portdb = trees[myroot]["porttree"].dbapi
-			portdb.close_caches()
-			portdbapi.portdbapi_instances.remove(portdb)
-			del trees[myroot]["porttree"], myroot, portdb
 
 	if trees is None:
 		trees = _trees_dict()
@@ -673,33 +666,6 @@ _legacy_global_var_names = ("archlist", "db", "features",
 def _reset_legacy_globals():
 
 	global _legacy_globals_constructed
-
-	if "_legacy_globals_constructed" in globals() and \
-		"db" in _legacy_globals_constructed:
-		try:
-			db
-		except NameError:
-			pass
-		else:
-			if isinstance(db, dict) and db:
-				for _x in db.values():
-					try:
-						if "porttree" in _x.lazy_items:
-							continue
-					except (AttributeError, TypeError):
-						continue
-					try:
-						_x = _x["porttree"].dbapi
-					except (AttributeError, KeyError):
-						continue
-					if not isinstance(_x, portdbapi):
-						continue
-					_x.close_caches()
-					try:
-						portdbapi.portdbapi_instances.remove(_x)
-					except ValueError:
-						pass
-
 	_legacy_globals_constructed = set()
 	for k in _legacy_global_var_names:
 		globals()[k] = _LegacyGlobalProxy(k)

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index cf1e8ae..9b8a4d3 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -1,6 +1,6 @@
-# Copyright: 2005-2012 Gentoo Foundation
+# Copyright 2005-2013 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
 # Author(s): Brian Harring (ferringb@gentoo.org)
-# License: GPL2
 
 from portage.cache import cache_errors
 from portage.cache.cache_errors import InvalidRestriction
@@ -164,7 +164,14 @@ class database(object):
 
 	def commit(self):
 		if not self.autocommits:
-			raise NotImplementedError
+			raise NotImplementedError(self)
+
+	def __del__(self):
+		# This used to be handled by an atexit hook that called
+		# close_portdbapi_caches() for all portdbapi instances, but that was
+		# prone to memory leaks for API consumers that needed to create/destroy
+		# many portdbapi instances. So, instead we rely on __del__.
+		self.sync()
 
 	def __contains__(self, cpv):
 		"""This method should always be overridden.  It is provided only for

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index a2082a3..1a6ffdd 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -53,9 +53,27 @@ if sys.hexversion >= 0x3000000:
 	basestring = str
 	long = int
 
+# It used to be necessary for API consumers to remove portdbapi instances
+# from portdbapi_instances, in order to avoid having accumulated instances
+# consume memory. Now, portdbapi_instances is just an empty dummy list, so
+# for backward compatibility, ignore ValueError for removal on non-existent
+# items.
+class _dummy_list(list):
+	def remove(self, item):
+		# TODO: Trigger a DeprecationWarning here, after stable portage
+		# has dummy portdbapi_instances.
+		try:
+			self.remove(item)
+		except ValueError:
+			pass
+
+def close_portdbapi_caches():
+	# Since portdbapi_instances is a dummy list, there's nothing to do here.
+	pass
+
 class portdbapi(dbapi):
 	"""this tree will scan a portage directory located at root (passed to init)"""
-	portdbapi_instances = []
+	portdbapi_instances = _dummy_list()
 	_use_mutable = True
 
 	@property
@@ -80,7 +98,6 @@ class portdbapi(dbapi):
 		@param mysettings: an immutable config instance
 		@type mysettings: portage.config
 		"""
-		portdbapi.portdbapi_instances.append(self)
 
 		from portage import config
 		if mysettings:
@@ -1000,12 +1017,6 @@ class portdbapi(dbapi):
 
 		return True
 
-def close_portdbapi_caches():
-	for i in portdbapi.portdbapi_instances:
-		i.close_caches()
-
-portage.process.atexit_register(portage.portageexit)
-
 class portagetree(object):
 	def __init__(self, root=DeprecationWarning, virtual=DeprecationWarning,
 		settings=None):

diff --git a/pym/portage/tests/resolver/ResolverPlayground.py b/pym/portage/tests/resolver/ResolverPlayground.py
index dd5e17c..15ef95d 100644
--- a/pym/portage/tests/resolver/ResolverPlayground.py
+++ b/pym/portage/tests/resolver/ResolverPlayground.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2012 Gentoo Foundation
+# Copyright 2010-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from itertools import permutations
@@ -544,10 +544,6 @@ class ResolverPlayground(object):
 				return
 
 	def cleanup(self):
-		for eroot in self.trees:
-			portdb = self.trees[eroot]["porttree"].dbapi
-			portdb.close_caches()
-			portage.dbapi.porttree.portdbapi.portdbapi_instances.remove(portdb)
 		if self.debug:
 			print("\nEROOT=%s" % self.eroot)
 		else: