[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/, pym/_emerge/

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/, pym/_emerge/

[Zac Medico]

commit:     42c96587b86ba8f551f605086e47a5cd96843a1a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Aug 26 19:53:55 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Aug 26 19:53:55 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=42c96587

python3.2 fixes: ResourceWarning: unclosed file

---
 pym/_emerge/BinpkgEnvExtractor.py |    2 +-
 pym/portage/dbapi/vartree.py      |   15 +++++++++++----
 pym/portage/xpak.py               |    3 ++-
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/pym/_emerge/BinpkgEnvExtractor.py b/pym/_emerge/BinpkgEnvExtractor.py
index f68971b..1c94718 100644
--- a/pym/_emerge/BinpkgEnvExtractor.py
+++ b/pym/_emerge/BinpkgEnvExtractor.py
@@ -59,7 +59,7 @@ class BinpkgEnvExtractor(CompositeTask):
 		# This is a signal to ebuild.sh, so that it knows to filter
 		# out things like SANDBOX_{DENY,PREDICT,READ,WRITE} that
 		# would be preserved between normal phases.
-		open(_unicode_encode(self._get_dest_env_path() + '.raw'), 'w')
+		open(_unicode_encode(self._get_dest_env_path() + '.raw'), 'wb').close()
 
 		self._current_task = None
 		self.returncode = os.EX_OK

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 7f7873b..6e49f38 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3192,17 +3192,22 @@ class dblink(object):
 					pass
 				continue
 
+			f = None
 			try:
-				val = io.open(_unicode_encode(
+				f = io.open(_unicode_encode(
 					os.path.join(inforoot, var_name),
 					encoding=_encodings['fs'], errors='strict'),
 					mode='r', encoding=_encodings['repo.content'],
-					errors='replace').readline().strip()
+					errors='replace')
+				val = f.readline().strip()
 			except EnvironmentError as e:
 				if e.errno != errno.ENOENT:
 					raise
 				del e
 				val = ''
+			finally:
+				if f is not None:
+					f.close()
 
 			if var_name == 'SLOT':
 				slot = val
@@ -3558,10 +3563,12 @@ class dblink(object):
 		# write local package counter for recording
 		if counter is None:
 			counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
-		io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
+		f = io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
 			encoding=_encodings['fs'], errors='strict'),
 			mode='w', encoding=_encodings['repo.content'],
-			errors='backslashreplace').write(_unicode_decode(str(counter)))
+			errors='backslashreplace')
+		f.write(_unicode_decode(str(counter)))
+		f.close()
 
 		self.updateprotect()
 

diff --git a/pym/portage/xpak.py b/pym/portage/xpak.py
index 01cfe7b..a05a860 100644
--- a/pym/portage/xpak.py
+++ b/pym/portage/xpak.py
@@ -99,7 +99,8 @@ def xpak(rootdir,outfile=None):
 			# CONTENTS is generated during the merge process.
 			continue
 		x = _unicode_encode(x, encoding=_encodings['fs'], errors='strict')
-		mydata[x] = open(os.path.join(rootdir, x), 'rb').read()
+		with open(os.path.join(rootdir, x), 'rb') as f:
+			mydata[x] = f.read()
 
 	xpak_segment = xpak_mem(mydata)
 	if outfile:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/, pym/_emerge/

[Zac Medico]

commit:     e45a3f15379d8de4a772e1c70eee51c3d0e358a5
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 24 17:54:48 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Oct 24 17:54:48 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e45a3f15

deprecate "root" attributes

In preparation for prefix support, were EROOT will be used in place of
ROOT in many places, deprecate unnecessary "root" attributes in order
to eliminate them as a possible source of confusion.

---
 pym/_emerge/BlockerDB.py     |    5 +++--
 pym/_emerge/FakeVartree.py   |   11 ++++++++++-
 pym/portage/dbapi/bintree.py |   28 +++++++++++++++++++++++++---
 pym/portage/dbapi/vartree.py |   40 ++++++++++++++++++++++++++++------------
 pym/portage/news.py          |    2 +-
 5 files changed, 67 insertions(+), 19 deletions(-)

diff --git a/pym/_emerge/BlockerDB.py b/pym/_emerge/BlockerDB.py
index 4819749..0033665 100644
--- a/pym/_emerge/BlockerDB.py
+++ b/pym/_emerge/BlockerDB.py
@@ -25,7 +25,7 @@ class BlockerDB(object):
 		self._dep_check_trees = None
 		self._fake_vartree = fake_vartree
 		self._dep_check_trees = {
-			self._vartree.root : {
+			self._vartree.settings["ROOT"] : {
 				"porttree"    :  fake_vartree,
 				"vartree"     :  fake_vartree,
 		}}
@@ -36,7 +36,8 @@ class BlockerDB(object):
 		new_pkg is planned to be installed. This ignores build-time
 		blockers, since new_pkg is assumed to be built already.
 		"""
-		blocker_cache = BlockerCache(self._vartree.root, self._vartree.dbapi)
+		blocker_cache = BlockerCache(self._vartree.settings["ROOT"],
+			self._vartree.dbapi)
 		dep_keys = ["RDEPEND", "PDEPEND"]
 		settings = self._vartree.settings
 		stale_cache = set(blocker_cache)

diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py
index ddec7c5..a2b9232 100644
--- a/pym/_emerge/FakeVartree.py
+++ b/pym/_emerge/FakeVartree.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import sys
+import warnings
 
 import portage
 from portage import os
@@ -49,7 +50,6 @@ class FakeVartree(vartree):
 		real_vartree = root_config.trees["vartree"]
 		self._real_vardb = real_vartree.dbapi
 		portdb = root_config.trees["porttree"].dbapi
-		self.root = real_vartree.root
 		self.settings = real_vartree.settings
 		mykeys = list(real_vartree.dbapi._aux_cache_keys)
 		if "_mtime_" not in mykeys:
@@ -71,6 +71,15 @@ class FakeVartree(vartree):
 		self._portdb = portdb
 		self._global_updates = None
 
+	@property
+	def root(self):
+		warnings.warn("The root attribute of "
+			"_emerge.FakeVartree.FakeVartree"
+			" is deprecated. Use "
+			"settings['ROOT'] instead.",
+			DeprecationWarning, stacklevel=2)
+		return self.settings['ROOT']
+
 	def _match_wrapper(self, cpv, use_cache=1):
 		"""
 		Make sure the metadata in Package instances gets updated for any

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 5c091bd..39bb030 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -41,6 +41,7 @@ import subprocess
 import sys
 import tempfile
 import textwrap
+import warnings
 from itertools import chain
 try:
 	from urllib.parse import urlparse
@@ -242,10 +243,22 @@ def _pkgindex_cpv_map_latest_build(pkgindex):
 
 class binarytree(object):
 	"this tree scans for a list of all packages available in PKGDIR"
-	def __init__(self, root, pkgdir, virtual=None, settings=None):
+	def __init__(self, _unused=None, pkgdir=None, virtual=None, settings=None):
+
+		if pkgdir is None:
+			raise TypeError("pkgdir parameter is required")
+
+		if settings is None:
+			raise TypeError("settings parameter is required")
+
+		if _unused is not None and _unused != settings['ROOT']:
+			warnings.warn("The root parameter of the "
+				"portage.dbapi.bintree.binarytree"
+				" constructor is now unused. Use "
+				"settings['ROOT'] instead.",
+				DeprecationWarning, stacklevel=2)
+
 		if True:
-			self.root = root
-			#self.pkgdir=settings["PKGDIR"]
 			self.pkgdir = normalize_path(pkgdir)
 			self.dbapi = bindbapi(self, settings=settings)
 			self.update_ents = self.dbapi.update_ents
@@ -330,6 +343,15 @@ class binarytree(object):
 				chain(*self._pkgindex_translated_keys)
 			))
 
+	@property
+	def root(self):
+		warnings.warn("The root attribute of "
+			"portage.dbapi.bintree.binarytree"
+			" is deprecated. Use "
+			"settings['ROOT'] instead.",
+			DeprecationWarning, stacklevel=2)
+		return self.settings['ROOT']
+
 	def move_ent(self, mylist, repo_match=None):
 		if not self.populated:
 			self.populate()

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 1fc95a3..b69b363 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -129,12 +129,11 @@ class vardbapi(dbapi):
 		if settings is None:
 			settings = portage.settings
 		self.settings = settings
-		self.root = settings['ROOT']
 
-		if _unused_param is not None and _unused_param != self.root:
-			warnings.warn("The first parameter of the " + \
-				"portage.dbapi.vartree.vardbapi" + \
-				" constructor is now unused. Use " + \
+		if _unused_param is not None and _unused_param != settings['ROOT']:
+			warnings.warn("The first parameter of the "
+				"portage.dbapi.vartree.vardbapi"
+				" constructor is now unused. Use "
 				"settings['ROOT'] instead.",
 				DeprecationWarning, stacklevel=2)
 
@@ -148,7 +147,7 @@ class vardbapi(dbapi):
 		self._fs_lock_count = 0
 
 		if vartree is None:
-			vartree = portage.db[self.root]["vartree"]
+			vartree = portage.db[settings["ROOT"]]["vartree"]
 		self.vartree = vartree
 		self._aux_cache_keys = set(
 			["BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "DESCRIPTION",
@@ -164,7 +163,7 @@ class vardbapi(dbapi):
 
 		self._plib_registry = None
 		if _ENABLE_PRESERVE_LIBS:
-			self._plib_registry = PreservedLibsRegistry(self.root,
+			self._plib_registry = PreservedLibsRegistry(settings["ROOT"],
 				os.path.join(self._eroot, PRIVATE_PATH,
 				"preserved_libs_registry"))
 
@@ -175,6 +174,15 @@ class vardbapi(dbapi):
 
 		self._cached_counter = None
 
+	@property
+	def root(self):
+		warnings.warn("The root attribute of "
+			"portage.dbapi.vartree.vardbapi"
+			" is deprecated. Use "
+			"settings['ROOT'] instead.",
+			DeprecationWarning, stacklevel=2)
+		return self.settings['ROOT']
+
 	def getpath(self, mykey, filename=None):
 		# This is an optimized hotspot, so don't use unicode-wrapped
 		# os module and don't use os.path.join().
@@ -1157,12 +1165,11 @@ class vartree(object):
 
 		if settings is None:
 			settings = portage.settings
-		self.root = settings['ROOT']
 
-		if root is not None and root != self.root:
-			warnings.warn("The 'root' parameter of the " + \
-				"portage.dbapi.vartree.vartree" + \
-				" constructor is now unused. Use " + \
+		if root is not None and root != settings['ROOT']:
+			warnings.warn("The 'root' parameter of the "
+				"portage.dbapi.vartree.vartree"
+				" constructor is now unused. Use "
 				"settings['ROOT'] instead.",
 				DeprecationWarning, stacklevel=2)
 
@@ -1170,6 +1177,15 @@ class vartree(object):
 		self.dbapi = vardbapi(settings=settings, vartree=self)
 		self.populated = 1
 
+	@property
+	def root(self):
+		warnings.warn("The root attribute of "
+			"portage.dbapi.vartree.vartree"
+			" is deprecated. Use "
+			"settings['ROOT'] instead.",
+			DeprecationWarning, stacklevel=2)
+		return self.settings['ROOT']
+
 	def getpath(self, mykey, filename=None):
 		return self.dbapi.getpath(mykey, filename=filename)
 

diff --git a/pym/portage/news.py b/pym/portage/news.py
index 031e98c..1bf0522 100644
--- a/pym/portage/news.py
+++ b/pym/portage/news.py
@@ -39,7 +39,7 @@ class NewsManager(object):
 	def __init__(self, portdb, vardb, news_path, unread_path, language_id='en'):
 		self.news_path = news_path
 		self.unread_path = unread_path
-		self.target_root = vardb.root
+		self.target_root = vardb.settings["ROOT"]
 		self.language_id = language_id
 		self.config = vardb.settings
 		self.vdb = vardb

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/, pym/_emerge/

[Zac Medico]

commit:     a75341bf3a66f75edd68d1a8bc5efdb51c0c0740
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 14 12:29:02 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 14 12:29:02 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a75341bf

After python fork, don't close fds for PyPy 1.8.

If we close all open file descriptors after a fork, with PyPy 1.8 it
triggers "[Errno 9] Bad file descriptor" later in the subprocess.
Apparently it is holding references to file descriptors and closing
them after they've already been closed and re-opened for other
purposes. As a workaround, we don't close the file descriptors, so
that they won't be re-used and therefore we won't be vulnerable to this
kind of interference.

The obvious caveat of not closing the fds is that the subprocess can
hold locks that belonged to the parent process, even after the parent
process has released the locks. Hopefully this won't be a major problem
though, since the subprocess has to exit at release the lock
eventually, when the EbuildFetcher or _MergeProcess task is complete.

---
 pym/_emerge/EbuildFetcher.py       |    6 +++++-
 pym/portage/dbapi/_MergeProcess.py |    7 ++++++-
 pym/portage/process.py             |   20 +++++++++++---------
 3 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index 61c7848..6ad4341 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -6,6 +6,7 @@ import traceback
 from _emerge.SpawnProcess import SpawnProcess
 import copy
 import io
+import platform
 import signal
 import sys
 import portage
@@ -166,7 +167,10 @@ class EbuildFetcher(SpawnProcess):
 			portage.process.spawned_pids.append(pid)
 			return [pid]
 
-		portage.process._setup_pipes(fd_pipes)
+		# TODO: Find out why PyPy 1.8 with close_fds=True triggers
+		# "[Errno 9] Bad file descriptor" in subprocesses.
+		close_fds = platform.python_implementation() != 'PyPy'
+		portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
 
 		# Use default signal handlers in order to avoid problems
 		# killing subprocesses as reported in bug #353239.

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 9bb67c9..cf59265 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import io
+import platform
 import signal
 import traceback
 
@@ -143,7 +144,11 @@ class MergeProcess(SpawnProcess):
 			return [pid]
 
 		os.close(elog_reader_fd)
-		portage.process._setup_pipes(fd_pipes)
+
+		# TODO: Find out why PyPy 1.8 with close_fds=True triggers
+		# "[Errno 9] Bad file descriptor" in subprocesses.
+		close_fds = platform.python_implementation() != 'PyPy'
+		portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
 
 		# Use default signal handlers since the ones inherited
 		# from the parent process are irrelevant here.

diff --git a/pym/portage/process.py b/pym/portage/process.py
index 47b0a21..e7313ab 100644
--- a/pym/portage/process.py
+++ b/pym/portage/process.py
@@ -386,7 +386,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
 	# And switch to the new process.
 	os.execve(binary, myargs, env)
 
-def _setup_pipes(fd_pipes):
+def _setup_pipes(fd_pipes, close_fds=True):
 	"""Setup pipes for a forked process."""
 	my_fds = {}
 	# To protect from cases where direct assignment could
@@ -397,14 +397,16 @@ def _setup_pipes(fd_pipes):
 	# Then assign them to what they should be.
 	for fd in my_fds:
 		os.dup2(my_fds[fd], fd)
-	# Then close _all_ fds that haven't been explicitly
-	# requested to be kept open.
-	for fd in get_open_fds():
-		if fd not in my_fds:
-			try:
-				os.close(fd)
-			except OSError:
-				pass
+
+	if close_fds:
+		# Then close _all_ fds that haven't been explicitly
+		# requested to be kept open.
+		for fd in get_open_fds():
+			if fd not in my_fds:
+				try:
+					os.close(fd)
+				except OSError:
+					pass
 
 def find_binary(binary):
 	"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/, pym/_emerge/

[Zac Medico]

commit:     144c23efbb4e9565debad03c13c5bcab833a8336
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Mar 27 18:28:12 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Mar 27 18:28:12 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=144c23ef

Close fewer file descriptors for fork / no exec.

This will fix bug #374335.

---
 pym/_emerge/EbuildFetcher.py       |   14 ++++----------
 pym/portage/dbapi/_MergeProcess.py |   13 +++----------
 pym/portage/locks.py               |   23 ++++++++++++++++++++++-
 pym/portage/process.py             |   14 +++++++++++++-
 4 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index f6dab54..c0a7fdd 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import traceback
@@ -6,7 +6,6 @@ import traceback
 from _emerge.SpawnProcess import SpawnProcess
 import copy
 import io
-import platform
 import signal
 import sys
 import portage
@@ -170,14 +169,9 @@ class EbuildFetcher(SpawnProcess):
 			portage.process.spawned_pids.append(pid)
 			return [pid]
 
-		# TODO: Find out why PyPy 1.8 with close_fds=True triggers
-		# "[Errno 9] Bad file descriptor" in subprocesses. It could
-		# be due to garbage collection of file objects that were not
-		# closed before going out of scope, since PyPy's garbage
-		# collector does not support the refcounting semantics that
-		# CPython does.
-		close_fds = platform.python_implementation() != 'PyPy'
-		portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
+		portage.locks._close_fds()
+		# Disable close_fds since we don't exec (see _setup_pipes docstring).
+		portage.process._setup_pipes(fd_pipes, close_fds=False)
 
 		# Use default signal handlers in order to avoid problems
 		# killing subprocesses as reported in bug #353239.

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 301625c..b5f6a0b 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -2,7 +2,6 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import io
-import platform
 import signal
 import sys
 import traceback
@@ -155,15 +154,9 @@ class MergeProcess(SpawnProcess):
 			return [pid]
 
 		os.close(elog_reader_fd)
-
-		# TODO: Find out why PyPy 1.8 with close_fds=True triggers
-		# "[Errno 9] Bad file descriptor" in subprocesses. It could
-		# be due to garbage collection of file objects that were not
-		# closed before going out of scope, since PyPy's garbage
-		# collector does not support the refcounting semantics that
-		# CPython does.
-		close_fds = platform.python_implementation() != 'PyPy'
-		portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
+		portage.locks._close_fds()
+		# Disable close_fds since we don't exec (see _setup_pipes docstring).
+		portage.process._setup_pipes(fd_pipes, close_fds=False)
 
 		# Use default signal handlers since the ones inherited
 		# from the parent process are irrelevant here.

diff --git a/pym/portage/locks.py b/pym/portage/locks.py
index 9fee5ae..87fbe07 100644
--- a/pym/portage/locks.py
+++ b/pym/portage/locks.py
@@ -1,5 +1,5 @@
 # portage: Lock management code
-# Copyright 2004-2011 Gentoo Foundation
+# Copyright 2004-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
@@ -36,6 +36,20 @@ if platform.python_implementation() == 'PyPy':
 # so that it doesn't interfere with the status display.
 _quiet = False
 
+
+_open_fds = set()
+
+def _close_fds():
+	"""
+	This is intended to be called after a fork, in order to close file
+	descriptors for locks held by the parent process. This can be called
+	safely after a fork without exec, unlike the _setup_pipes close_fds
+	behavior.
+	.
+	"""
+	while _open_fds:
+		os.close(_open_fds.pop())
+
 def lockdir(mydir, flags=0):
 	return lockfile(mydir, wantnewlockfile=1, flags=flags)
 def unlockdir(mylock):
@@ -193,6 +207,9 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
 			mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 			waiting_msg=waiting_msg, flags=flags)
 
+	if myfd != HARDLINK_FD:
+		_open_fds.add(myfd)
+
 	writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 	return (lockfilename,myfd,unlinkfile,locking_method)
 
@@ -233,6 +250,7 @@ def unlockfile(mytuple):
 		writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1)
 		if myfd is not None:
 			os.close(myfd)
+			_open_fds.remove(myfd)
 		return False
 
 	try:
@@ -243,6 +261,7 @@ def unlockfile(mytuple):
 	except OSError:
 		if isinstance(lockfilename, basestring):
 			os.close(myfd)
+			_open_fds.remove(myfd)
 		raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 
 	try:
@@ -264,6 +283,7 @@ def unlockfile(mytuple):
 			else:
 				writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 				os.close(myfd)
+				_open_fds.remove(myfd)
 				return False
 	except SystemExit:
 		raise
@@ -276,6 +296,7 @@ def unlockfile(mytuple):
 	# open fd closed automatically on them.
 	if isinstance(lockfilename, basestring):
 		os.close(myfd)
+		_open_fds.remove(myfd)
 
 	return True
 

diff --git a/pym/portage/process.py b/pym/portage/process.py
index 94687ea..f3cec88 100644
--- a/pym/portage/process.py
+++ b/pym/portage/process.py
@@ -401,7 +401,19 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
 	os.execve(binary, myargs, env)
 
 def _setup_pipes(fd_pipes, close_fds=True):
-	"""Setup pipes for a forked process."""
+	"""Setup pipes for a forked process.
+
+	WARNING: When not followed by exec, the close_fds behavior
+	can trigger interference from destructors that close file
+	descriptors. This interference happens when the garbage
+	collector intermittently executes such destructors after their
+	corresponding file descriptors have been re-used, leading
+	to intermittent "[Errno 9] Bad file descriptor" exceptions in
+	forked processes. This problem has been observed with PyPy 1.8,
+	and also with CPython under some circumstances (as triggered
+	by xmpppy in bug #374335). In order to close a safe subset of
+	file descriptors, see portage.locks._close_fds().
+	"""
 	my_fds = {}
 	# To protect from cases where direct assignment could
 	# clobber needed fds ({1:2, 2:1}) we first dupe the fds

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/portage/, pym/_emerge/

[Zac Medico]

commit:     2377357f4e0164d338c1249c79f479229aa7c0e4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 29 02:36:17 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 29 02:36:17 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2377357f

Handle KeyError when loading pickles.

See http://forums.gentoo.org/viewtopic-t-938022.html for example.

---
 pym/_emerge/BlockerCache.py  |    4 +++-
 pym/portage/dbapi/vartree.py |    4 +++-
 pym/portage/getbinpkg.py     |    4 +++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/pym/_emerge/BlockerCache.py b/pym/_emerge/BlockerCache.py
index fce81f8..9195524 100644
--- a/pym/_emerge/BlockerCache.py
+++ b/pym/_emerge/BlockerCache.py
@@ -62,7 +62,9 @@ class BlockerCache(portage.cache.mappings.MutableMapping):
 			self._cache_data = mypickle.load()
 			f.close()
 			del f
-		except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e:
+		except (SystemExit, KeyboardInterrupt):
+			raise
+		except Exception as e:
 			if isinstance(e, EnvironmentError) and \
 				getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES):
 				pass

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index f8980f7..5299b29 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -585,7 +585,9 @@ class vardbapi(dbapi):
 			aux_cache = mypickle.load()
 			f.close()
 			del f
-		except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e:
+		except (SystemExit, KeyboardInterrupt):
+			raise
+		except Exception as e:
 			if isinstance(e, EnvironmentError) and \
 				getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES):
 				pass

diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py
index 34a5e0e..538f8c2 100644
--- a/pym/portage/getbinpkg.py
+++ b/pym/portage/getbinpkg.py
@@ -559,7 +559,9 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
 		out.write(_("Loaded metadata pickle.\n"))
 		out.flush()
 		metadatafile.close()
-	except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError):
+	except (SystemExit, KeyboardInterrupt):
+		raise
+	except Exception:
 		metadata = {}
 	if baseurl not in metadata:
 		metadata[baseurl]={}