[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     adbefe1e3c84f4015e7dbcc9aee18b0f10833e4d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May  8 20:46:40 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May  8 20:46:40 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=adbefe1e

Add back cpv_exists check from previous commit.

---
 pym/portage/dbapi/vartree.py |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index b4e231e..a34f1ec 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1485,6 +1485,8 @@ class dblink(object):
 				if cpv_lib_map:
 					self._remove_preserved_libs(cpv_lib_map)
 					for cpv, removed in cpv_lib_map.items():
+						if not self.vartree.dbapi.cpv_exists(cpv):
+							continue
 						self.vartree.dbapi.removeFromContents(cpv, removed)
 
 				if unmerge:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     84431fe188ed1b3e2e7b52918329bd696d8304b1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 16 05:07:10 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jul 16 05:18:51 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=84431fe1

portdbapi.cp_list: honor porttrees modifications (bug 650814)

Consumers, such as repoman and emirrordist, may modify the porttrees
attribute in order to modify the effective set of repositories for all
portdbapi operations.

Fixes: 27eeeb6e4fc8 ("portdbapi.cp_list: cache repo associations (bug 650814)")
Bug: https://bugs.gentoo.org/650814

 pym/portage/dbapi/porttree.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 2e271ea76..677452273 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -322,6 +322,26 @@ class portdbapi(dbapi):
 		self._better_cache = None
 		self._broken_ebuilds = set()
 
+	def _set_porttrees(self, porttrees):
+		"""
+		Consumers, such as repoman and emirrordist, may modify the porttrees
+		attribute in order to modify the effective set of repositories for
+		all portdbapi operations.
+
+		@param porttrees: list of repo locations, in ascending order by
+			repo priority
+		@type porttrees: list
+		"""
+		self._porttrees_repos = portage.OrderedDict((repo.name, repo)
+			for repo in (self.repositories.get_repo_for_location(location)
+			for location in porttrees))
+		self._porttrees = tuple(porttrees)
+
+	def _get_porttrees(self):
+		return self._porttrees
+
+	porttrees = property(_get_porttrees, _set_porttrees)
+
 	@property
 	def _event_loop(self):
 		if portage._internal_caller:
@@ -972,9 +992,10 @@ class portdbapi(dbapi):
 				repos = [self.repositories.get_repo_for_location(location)
 					for location in mytree]
 		elif self._better_cache is None:
-			repos = list(self.repositories)
+			repos = self._porttrees_repos.values()
 		else:
-			repos = reversed(self._better_cache[mycp])
+			repos = [repo for repo in reversed(self._better_cache[mycp])
+				if repo.name in self._porttrees_repos]
 		mylist = []
 		for repo in repos:
 			oroot = repo.location

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     31edfe62541d8b55c64584b056ce64ad759f6a8e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 15 21:38:30 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul 15 21:38:30 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=31edfe62

portdbapi.xmatch: return separate items for each repo

Since xmatch match-all shares cache with the cp_list method, it needs
to return separate items when the same package version is found in
multiple repos.

Fixes: 27eeeb6e4fc8 ("portdbapi.cp_list: cache repo associations (bug 650814)")
Bug: https://bugs.gentoo.org/650814

 pym/portage/dbapi/porttree.py | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index f6076ee2b..2e271ea76 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1108,20 +1108,10 @@ class portdbapi(dbapi):
 			else:
 				iterfunc = iter
 
-			if mydep.repo is not None:
-				repos = [mydep.repo]
-			else:
-				# We iterate over self.porttrees, since it's common to
-				# tweak this attribute in order to adjust match behavior.
-				repos = []
-				for tree in reversed(self.porttrees):
-					repos.append(self.repositories.get_name_for_location(tree))
-
 			for cpv in iterfunc(mylist):
-				for repo in repos:
 					try:
 						metadata = dict(zip(aux_keys,
-							self.aux_get(cpv, aux_keys, myrepo=repo)))
+							self.aux_get(cpv, aux_keys, myrepo=cpv.repo)))
 					except KeyError:
 						# ebuild not in this repo, or masked by corruption
 						continue
@@ -1145,11 +1135,8 @@ class portdbapi(dbapi):
 						continue
 
 					myval.append(pkg_str)
-					# only yield a given cpv once
-					break
-
-				if myval and single_match:
-					break
+					if single_match:
+						break
 
 			if single_match:
 				if myval:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3a25c3fa13d7c62ba8c00d6c7a75eb907d34b568
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 13 03:58:04 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jul 14 23:54:29 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a25c3fa

_unmerge_dirs: revisit parents of removed symlinks (bug 640058)

When removal of a symlink is triggered by removal of the directory
that it points to, revisit the parent directories of the symlink.

Bug: https://bugs.gentoo.org/640058
Reviewed-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 1a86940f1..a104306eb 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2753,9 +2753,13 @@ class dblink(object):
 		real_root = self.settings['ROOT']
 
 		dirs = sorted(dirs)
-		dirs.reverse()
+		revisit = {}
 
-		for obj, inode_key in dirs:
+		while True:
+			try:
+				obj, inode_key = dirs.pop()
+			except IndexError:
+				break
 			# Treat any directory named "info" as a candidate here,
 			# since it might have been in INFOPATH previously even
 			# though it may not be there now.
@@ -2818,6 +2822,7 @@ class dblink(object):
 					raise
 				if e.errno != errno.ENOENT:
 					show_unmerge("---", unmerge_desc["!empty"], "dir", obj)
+					revisit[obj] = inode_key
 
 				# Since we didn't remove this directory, record the directory
 				# itself for use in syncfs calls, if we have removed another
@@ -2838,6 +2843,7 @@ class dblink(object):
 				# no need to protect symlinks that point to it.
 				unmerge_syms = protected_symlinks.pop(inode_key, None)
 				if unmerge_syms is not None:
+					parents = []
 					for relative_path in unmerge_syms:
 						obj = os.path.join(real_root,
 							relative_path.lstrip(os.sep))
@@ -2849,6 +2855,21 @@ class dblink(object):
 								raise
 							del e
 							show_unmerge("!!!", "", "sym", obj)
+						else:
+							parents.append(os.path.dirname(obj))
+
+					if parents:
+						# Revisit parents recursively (bug 640058).
+						recursive_parents = []
+						for parent in set(parents):
+							while parent in revisit:
+								recursive_parents.append(parent)
+								parent = os.path.dirname(parent)
+								if parent == '/':
+									break
+
+						for parent in sorted(set(recursive_parents)):
+							dirs.append((parent, revisit.pop(parent)))
 
 	def isowner(self, filename, destroot=None):
 		"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     2710f9ab525e7c726c2ffb027e242dbdf31cfe76
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jun 26 20:14:09 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jun 26 20:16:25 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2710f9ab

binarytree._merge_pkgindex_header: deduplicate values (bug 657422)

Deduplicate values of implicit IUSE variables, in order to
prevent them from growing without bound, triggering "[Errno 7]
Argument list too long" errors as reported in bug 657422.

Fixes: cab78dba98c4 ("emerge --usepkgonly: propagate implicit IUSE and USE_EXPAND (bug 640318)")
Bug: https://bugs.gentoo.org/657422

 pym/portage/dbapi/bintree.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 269a7b226..9c2d877e7 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1320,7 +1320,9 @@ class binarytree(object):
 		for k, v in iter_iuse_vars(src):
 			v_before = dest.get(k)
 			if v_before is not None:
-				v = v_before + ' ' + v
+				merged_values = set(v_before.split())
+				merged_values.update(v.split())
+				v = ' '.join(sorted(merged_values))
 			dest[k] = v
 
 		if 'ARCH' not in dest and 'ARCH' in src:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     41af82685d688cb03da743cdd03295271a3ef09c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 30 05:45:13 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 30 06:20:01 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=41af8268

_MergeProcess: add_reader asyncio compat (bug 654382)

Use add_reader for asyncio compatibility.

Bug: https://bugs.gentoo.org/654382

 pym/portage/dbapi/_MergeProcess.py | 27 +++++++--------------------
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index bfbe387e4..42f2d84e5 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -7,7 +7,6 @@ import signal
 import sys
 import traceback
 
-import errno
 import fcntl
 import portage
 from portage import os, _unicode_decode
@@ -24,7 +23,7 @@ class MergeProcess(ForkProcess):
 	__slots__ = ('mycat', 'mypkg', 'settings', 'treetype',
 		'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild',
 		'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge',
-		'_elog_reader_fd', '_elog_reg_id',
+		'_elog_reader_fd',
 		'_buf', '_elog_keys', '_locked_vdb')
 
 	def _start(self):
@@ -79,14 +78,8 @@ class MergeProcess(ForkProcess):
 			self.vartree.dbapi.unlock()
 			self._locked_vdb = False
 
-	def _elog_output_handler(self, fd, event):
-		output = None
-		if event & self.scheduler.IO_IN:
-			try:
-				output = os.read(fd, self._bufsize)
-			except OSError as e:
-				if e.errno not in (errno.EAGAIN, errno.EINTR):
-					raise
+	def _elog_output_handler(self):
+		output = self._read_buf(self._elog_reader_fd, None)
 		if output:
 			lines = _unicode_decode(output).split('\n')
 			if len(lines) == 1:
@@ -101,15 +94,12 @@ class MergeProcess(ForkProcess):
 					reporter = getattr(portage.elog.messages, funcname)
 					reporter(msg, phase=phase, key=key, out=out)
 
-		if event & self.scheduler.IO_HUP:
-			self.scheduler.source_remove(self._elog_reg_id)
-			self._elog_reg_id = None
+		elif output is not None: # EIO/POLLHUP
+			self.scheduler.remove_reader(self._elog_reader_fd)
 			os.close(self._elog_reader_fd)
 			self._elog_reader_fd = None
 			return False
 
-		return True
-
 	def _spawn(self, args, fd_pipes, **kwargs):
 		"""
 		Fork a subprocess, apply local settings, and call
@@ -142,8 +132,7 @@ class MergeProcess(ForkProcess):
 			treetype=self.treetype, vartree=self.vartree,
 			blockers=blockers, pipe=elog_writer_fd)
 		fd_pipes[elog_writer_fd] = elog_writer_fd
-		self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd,
-			self._registered_events, self._elog_output_handler)
+		self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler)
 
 		# If a concurrent emerge process tries to install a package
 		# in the same SLOT as this one at the same time, there is an
@@ -275,10 +264,8 @@ class MergeProcess(ForkProcess):
 				pass
 
 		self._unlock_vdb()
-		if self._elog_reg_id is not None:
-			self.scheduler.source_remove(self._elog_reg_id)
-			self._elog_reg_id = None
 		if self._elog_reader_fd is not None:
+			self.scheduler.remove_reader(self._elog_reader_fd)
 			os.close(self._elog_reader_fd)
 			self._elog_reader_fd = None
 		if self._elog_keys is not None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     542c6e6c20a1d93a3a2af47c8de50eac3c891d5d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 29 20:48:23 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 29 21:26:37 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=542c6e6c

MergeProcess: fix deprecated _set_returncode (bug 654276)

Move cleanup code from _set_returncode to _async_waitpid_cb,
since _set_returncode expects an os.waitpid return value which
is inconveniently different from the returncode that is passed to
asyncio.AbstractChildWatcher.add_child_handler callbacks.

Bug: https://bugs.gentoo.org/654276

 pym/portage/dbapi/_MergeProcess.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 35591af16..bfbe387e4 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2013 Gentoo Foundation
+# Copyright 2010-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import io
@@ -251,8 +251,12 @@ class MergeProcess(ForkProcess):
 				# in order to avoid a race condition.
 				os._exit(1)
 
-	def _set_returncode(self, wait_retval):
-		ForkProcess._set_returncode(self, wait_retval)
+	def _async_waitpid_cb(self, *args, **kwargs):
+		"""
+		Override _async_waitpid_cb to perform cleanup that is
+		not necessarily idempotent.
+		"""
+		ForkProcess._async_waitpid_cb(self, *args, **kwargs)
 		if self.returncode == portage.const.RETURNCODE_POSTINST_FAILURE:
 			self.postinst_failure = True
 			self.returncode = os.EX_OK

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     698bbcc494e06719cdbb0522aff6a5fb0d249b36
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 24 06:31:32 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 24 06:55:20 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=698bbcc4

portdbapi.async_fetch_map: fix future.exception() reference

Fixes: 8002d9343c38 ("portdbapi: add async_fetch_map method (bug 653810)")

 pym/portage/dbapi/porttree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 8a5fc690c..975f03d5e 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -759,7 +759,7 @@ class portdbapi(dbapi):
 			if result.cancelled():
 				return
 			if aux_get_future.exception() is not None:
-				if isinstance(future.exception(), PortageKeyError):
+				if isinstance(aux_get_future.exception(), PortageKeyError):
 					# Convert this to an InvalidDependString exception since
 					# callers already handle it.
 					result.set_exception(portage.exception.InvalidDependString(

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     508c7667bb0755c9f72b443879dcf7f631434fda
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 23 19:51:09 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 23 19:53:47 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=508c7667

portdbapi.async_fetch_map: pass loop to async_aux_get

Fixes: 8002d9343c38 ("portdbapi: add async_fetch_map method (bug 653810)")

 pym/portage/dbapi/porttree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 3cd929963..8a5fc690c 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -783,7 +783,7 @@ class portdbapi(dbapi):
 				{'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
 
 		aux_get_future = self.async_aux_get(
-			mypkg, ["EAPI", "SRC_URI"], mytree=mytree)
+			mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop)
 		result.add_done_callback(lambda result:
 			aux_get_future.cancel() if result.cancelled() else None)
 		aux_get_future.add_done_callback(aux_get_done)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8002d9343c385075ece19a131b0f584ec255a5b5
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 22 19:57:09 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 23 18:39:16 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8002d934

portdbapi: add async_fetch_map method (bug 653810)

Add a portdbapi.async_fetch_map method which is identical to the
existing portdbapi.getFetchMap method, but returns a future. This
will be used by EbuildFetcher in order to avoid event loop recursion.

Bug: https://bugs.gentoo.org/653810
Reviewed-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/porttree.py | 75 +++++++++++++++++++++++++++++++++----------
 1 file changed, 58 insertions(+), 17 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 951e5760a..3cd929963 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -728,25 +728,66 @@ class portdbapi(dbapi):
 			URIs.
 		@rtype: dict
 		"""
+		loop = self._event_loop
+		return loop.run_until_complete(
+			self.async_fetch_map(mypkg, useflags=useflags,
+				mytree=mytree, loop=loop))
 
-		try:
-			eapi, myuris = self.aux_get(mypkg,
-				["EAPI", "SRC_URI"], mytree=mytree)
-		except KeyError:
-			# Convert this to an InvalidDependString exception since callers
-			# already handle it.
-			raise portage.exception.InvalidDependString(
-				"getFetchMap(): aux_get() error reading "+mypkg+"; aborting.")
+	def async_fetch_map(self, mypkg, useflags=None, mytree=None, loop=None):
+		"""
+		Asynchronous form of getFetchMap.
 
-		if not eapi_is_supported(eapi):
-			# Convert this to an InvalidDependString exception
-			# since callers already handle it.
-			raise portage.exception.InvalidDependString(
-				"getFetchMap(): '%s' has unsupported EAPI: '%s'" % \
-				(mypkg, eapi))
-
-		return _parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris},
-			use=useflags)
+		@param mypkg: cpv for an ebuild
+		@type mypkg: String
+		@param useflags: a collection of enabled USE flags, for evaluation of
+			conditionals
+		@type useflags: set, or None to enable all conditionals
+		@param mytree: The canonical path of the tree in which the ebuild
+			is located, or None for automatic lookup
+		@type mypkg: String
+		@param loop: event loop (defaults to global event loop)
+		@type loop: EventLoop
+		@return: A future that results in a dict which maps each file name to
+			a set of alternative URIs.
+		@rtype: asyncio.Future (or compatible)
+		"""
+		loop = loop or global_event_loop()
+		loop = getattr(loop, '_asyncio_wrapper', loop)
+		result = loop.create_future()
+
+		def aux_get_done(aux_get_future):
+			if result.cancelled():
+				return
+			if aux_get_future.exception() is not None:
+				if isinstance(future.exception(), PortageKeyError):
+					# Convert this to an InvalidDependString exception since
+					# callers already handle it.
+					result.set_exception(portage.exception.InvalidDependString(
+						"getFetchMap(): aux_get() error reading "
+						+ mypkg + "; aborting."))
+				else:
+					result.set_exception(future.exception())
+				return
+
+			eapi, myuris = aux_get_future.result()
+
+			if not eapi_is_supported(eapi):
+				# Convert this to an InvalidDependString exception
+				# since callers already handle it.
+				result.set_exception(portage.exception.InvalidDependString(
+					"getFetchMap(): '%s' has unsupported EAPI: '%s'" % \
+					(mypkg, eapi)))
+				return
+
+			result.set_result(_parse_uri_map(mypkg,
+				{'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
+
+		aux_get_future = self.async_aux_get(
+			mypkg, ["EAPI", "SRC_URI"], mytree=mytree)
+		result.add_done_callback(lambda result:
+			aux_get_future.cancel() if result.cancelled() else None)
+		aux_get_future.add_done_callback(aux_get_done)
+		return result
 
 	def getfetchsizes(self, mypkg, useflags=None, debug=0, myrepo=None):
 		# returns a filename:size dictionnary of remaining downloads

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     6b50f39c3d45456b40f21ca85c6d57296c64c377
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 17 17:12:44 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 17 17:15:44 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6b50f39c

IndexedVardb: pass db to _pkg_str (bug 653372)

Fix an AttributeError when emerge --search contructs a Package instance
in order to check visibility of an installed package.

Fixes: bfe7892202b8 ("_pkg_str: add _db attribute (bug 640318)")
Bug: https://bugs.gentoo.org/653372

 pym/portage/dbapi/IndexedVardb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/IndexedVardb.py b/pym/portage/dbapi/IndexedVardb.py
index b9466707d..e2910b27f 100644
--- a/pym/portage/dbapi/IndexedVardb.py
+++ b/pym/portage/dbapi/IndexedVardb.py
@@ -52,7 +52,7 @@ class IndexedVardb(object):
 		self._cp_map = cp_map = {}
 		for cpv in self._vardb._aux_cache["packages"]:
 			try:
-				cpv = _pkg_str(cpv)
+				cpv = _pkg_str(cpv, db=self._vardb)
 			except InvalidData:
 				continue
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e767fd990b89bce010b9e9a29c2892eecbf15fa9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 13 01:18:10 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Apr 13 01:22:23 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e767fd99

binarytree._update_pkgindex_header: include USE_EXPAND_IMPLICIT vars (bug 644990)

It's useful to have values of USE_EXPAND_IMPLICIT variables in the
Packages header.

Bug: https://bugs.gentoo.org/644990

 pym/portage/dbapi/bintree.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 324305aef..a963c578e 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1322,6 +1322,7 @@ class binarytree(object):
 		else:
 			header.pop("URI", None)
 		for k in list(self._pkgindex_header_keys) + \
+			self.settings.get("USE_EXPAND_IMPLICIT", "").split() + \
 			self.settings.get("USE_EXPAND_UNPREFIXED", "").split():
 			v = self.settings.get(k, None)
 			if v:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     82759203b9b39389de9ab27aeae58319b604f1ae
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 10 15:16:21 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Apr 11 01:44:34 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=82759203

async_aux_get: support asyncio via _PortageEventLoopPolicy (bug 649588)

Support portage's internal EventLoop as well as the _PortageEventLoop
asyncio compatibility wrapper, by using the respective _loop and
_asyncio_wrapper attributes where appropriate.

Example usage for coroutine with PEP 492 async and await syntax:

  import asyncio
  import portage
  from portage.exception import PortageKeyError
  from portage.util.futures.unix_events import DefaultEventLoopPolicy

  asyncio.set_event_loop_policy(DefaultEventLoopPolicy())

  async def aux_get_demo():
    portdb = portage.portdb
    for cpv in portdb.cp_list('sys-apps/portage'):
      try:
        result = await portdb.async_aux_get(cpv, portage.auxdbkeys)
      except PortageKeyError as e:
        # aux_get failed
        print('error:', cpv, e)
      else:
        print(cpv, result)

  asyncio.get_event_loop().run_until_complete(aux_get_demo())

Bug: https://bugs.gentoo.org/649588

 pym/portage/dbapi/porttree.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 910e90e6f..951e5760a 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -611,6 +611,7 @@ class portdbapi(dbapi):
 		# Callers of this method certainly want the same event loop to
 		# be used for all calls.
 		loop = loop or global_event_loop()
+		loop = getattr(loop, '_asyncio_wrapper', loop)
 		future = loop.create_future()
 		cache_me = False
 		if myrepo is not None:
@@ -665,7 +666,7 @@ class portdbapi(dbapi):
 
 		proc = EbuildMetadataPhase(cpv=mycpv,
 			ebuild_hash=ebuild_hash, portdb=self,
-			repo_path=mylocation, scheduler=loop,
+			repo_path=mylocation, scheduler=loop._loop,
 			settings=self.doebuild_settings)
 
 		proc.addExitListener(functools.partial(self._aux_get_return,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1689911cf9aeb363b23e0bbd4d357cab5ce78538
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 10 22:16:07 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 10 22:16:07 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1689911c

binarytree._update_pkgindex_header: include USE_EXPAND_UNPREFIXED vars (bug 640318)

Values of USE_EXPAND_UNPREFIXED variables are required for implicit
IUSE calculations.

Bug: https://bugs.gentoo.org/640318

 pym/portage/dbapi/bintree.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index ab4f4760d..324305aef 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1321,7 +1321,8 @@ class binarytree(object):
 			header["URI"] = base_uri
 		else:
 			header.pop("URI", None)
-		for k in self._pkgindex_header_keys:
+		for k in list(self._pkgindex_header_keys) + \
+			self.settings.get("USE_EXPAND_UNPREFIXED", "").split():
 			v = self.settings.get(k, None)
 			if v:
 				header[k] = v

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     232aae7f1fd79573426c0da9c1cbd79324994099
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 10 01:32:04 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Apr 10 01:32:04 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=232aae7f

binarytree._update_pkgindex_header: skip update if profile invalid (bug 640318)

Return silently if the current profile is invalid or does not have
an IUSE_IMPLICIT variable, since it's useful to maintain a cache of
implicit IUSE settings for use with binary packages.

Bug: https://bugs.gentoo.org/640318

 pym/portage/dbapi/bintree.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 42d334d24..ab4f4760d 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1293,6 +1293,20 @@ class binarytree(object):
 			translated_keys=self._pkgindex_translated_keys)
 
 	def _update_pkgindex_header(self, header):
+		"""
+		Add useful settings to the Packages file header, for use by
+		binhost clients.
+
+		This will return silently if the current profile is invalid or
+		does not have an IUSE_IMPLICIT variable, since it's useful to
+		maintain a cache of implicit IUSE settings for use with binary
+		packages.
+		"""
+		if not (self.settings.profile_path and
+			"IUSE_IMPLICIT" in self.settings):
+			header.setdefault("VERSION", _unicode(self._pkgindex_version))
+			return
+
 		portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"]))
 		profiles_base = os.path.join(portdir, "profiles") + os.path.sep
 		if self.settings.profile_path:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Michał Górny]

commit:     e12f6c3626da36ba558782ff52278be1566c5125
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 16 10:04:58 2018 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Fri Mar 16 21:10:37 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e12f6c36

portage.dbapi.vartree: Remove one more unfounded virtual case

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 8b1b77f7d..6406682d6 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3747,13 +3747,6 @@ class dblink(object):
 		is_binpkg = self.settings.get("EMERGE_FROM") == "binary"
 		slot = ''
 		for var_name in ('CHOST', 'SLOT'):
-			if var_name == 'CHOST' and self.cat == 'virtual':
-				try:
-					os.unlink(os.path.join(inforoot, var_name))
-				except OSError:
-					pass
-				continue
-
 			try:
 				with io.open(_unicode_encode(
 					os.path.join(inforoot, var_name),

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e3960e27f1b75120c2c0511b6207bd2ebdcc26a3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 25 22:27:13 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 28 17:22:19 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e3960e27

portdbapi: add async_aux_get method (bug 648790)

Add async_aux_get method that returns a Future and otherwise
behaves identically to aux_get. Use async_aux_get to implement
the synchronous aux_get method.

Bug: https://bugs.gentoo.org/648790
Reviewed-by: Alec Warner <antarus <AT> gentoo.org>

 pym/portage/dbapi/porttree.py | 99 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 78 insertions(+), 21 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index f5979d2d0..f8ff4833b 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -45,6 +45,7 @@ import traceback
 import warnings
 import errno
 import collections
+import functools
 
 try:
 	from urllib.parse import urlparse
@@ -577,11 +578,46 @@ class portdbapi(dbapi):
 		"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
 		'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
 		'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise PortageKeyError if error'
+		# For external API consumers, self._event_loop returns a new event
+		# loop on each access, so a local reference is needed in order
+		# to avoid instantiating more than one.
+		loop = self._event_loop
+		return loop.run_until_complete(
+			self.async_aux_get(mycpv, mylist, mytree=mytree,
+			myrepo=myrepo, loop=loop))
+
+	def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None):
+		"""
+		Asynchronous form form of aux_get.
+
+		@param mycpv: cpv for an ebuild
+		@type mycpv: str
+		@param mylist: list of metadata keys
+		@type mylist: list
+		@param mytree: The canonical path of the tree in which the ebuild
+			is located, or None for automatic lookup
+		@type mytree: str
+		@param myrepo: name of the repo in which the ebuild is located,
+			or None for automatic lookup
+		@type myrepo: str
+		@param loop: event loop (defaults to global event loop)
+		@type loop: EventLoop
+		@return: list of metadata values
+		@rtype: asyncio.Future (or compatible)
+		"""
+		# Don't default to self._event_loop here, since that creates a
+		# local event loop for thread safety, and that could easily lead
+		# to simultaneous instantiation of multiple event loops here.
+		# Callers of this method certainly want the same event loop to
+		# be used for all calls.
+		loop = loop or global_event_loop()
+		future = loop.create_future()
 		cache_me = False
 		if myrepo is not None:
 			mytree = self.treemap.get(myrepo)
 			if mytree is None:
-				raise PortageKeyError(myrepo)
+				future.set_exception(PortageKeyError(myrepo))
+				return future
 
 		if mytree is not None and len(self.porttrees) == 1 \
 			and mytree == self.porttrees[0]:
@@ -596,43 +632,64 @@ class portdbapi(dbapi):
 			mylist).difference(self._aux_cache_keys):
 			aux_cache = self._aux_cache.get(mycpv)
 			if aux_cache is not None:
-				return [aux_cache.get(x, "") for x in mylist]
+				future.set_result([aux_cache.get(x, "") for x in mylist])
+				return future
 			cache_me = True
 
 		try:
 			cat, pkg = mycpv.split("/", 1)
 		except ValueError:
 			# Missing slash. Can't find ebuild so raise PortageKeyError.
-			raise PortageKeyError(mycpv)
+			future.set_exception(PortageKeyError(mycpv))
+			return future
 
 		myebuild, mylocation = self.findname2(mycpv, mytree)
 
 		if not myebuild:
 			writemsg("!!! aux_get(): %s\n" % \
 				_("ebuild not found for '%s'") % mycpv, noiselevel=1)
-			raise PortageKeyError(mycpv)
+			future.set_exception(PortageKeyError(mycpv))
+			return future
 
 		mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
-		doregen = mydata is None
-
-		if doregen:
-			if myebuild in self._broken_ebuilds:
-				raise PortageKeyError(mycpv)
-
-			proc = EbuildMetadataPhase(cpv=mycpv,
-				ebuild_hash=ebuild_hash, portdb=self,
-				repo_path=mylocation, scheduler=self._event_loop,
-				settings=self.doebuild_settings)
-
-			proc.start()
-			proc.wait()
 
+		if mydata is not None:
+			self._aux_get_return(
+				future, mycpv, mylist, myebuild, ebuild_hash,
+				mydata, mylocation, cache_me, None)
+			return future
+
+		if myebuild in self._broken_ebuilds:
+			future.set_exception(PortageKeyError(mycpv))
+			return future
+
+		proc = EbuildMetadataPhase(cpv=mycpv,
+			ebuild_hash=ebuild_hash, portdb=self,
+			repo_path=mylocation, scheduler=loop,
+			settings=self.doebuild_settings)
+
+		proc.addExitListener(functools.partial(self._aux_get_return,
+			future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation,
+			cache_me))
+		future.add_done_callback(functools.partial(self._aux_get_cancel, proc))
+		proc.start()
+		return future
+
+	@staticmethod
+	def _aux_get_cancel(proc, future):
+		if future.cancelled() and proc.returncode is None:
+			proc.cancel()
+
+	def _aux_get_return(self, future, mycpv, mylist, myebuild, ebuild_hash,
+		mydata, mylocation, cache_me, proc):
+		if future.cancelled():
+			return
+		if proc is not None:
 			if proc.returncode != os.EX_OK:
 				self._broken_ebuilds.add(myebuild)
-				raise PortageKeyError(mycpv)
-
+				future.set_exception(PortageKeyError(mycpv))
+				return
 			mydata = proc.metadata
-
 		mydata["repository"] = self.repositories.get_name_for_location(mylocation)
 		mydata["_mtime_"] = ebuild_hash.mtime
 		eapi = mydata.get("EAPI")
@@ -651,7 +708,7 @@ class portdbapi(dbapi):
 				aux_cache[x] = mydata.get(x, "")
 			self._aux_cache[mycpv] = aux_cache
 
-		return returnme
+		future.set_result(returnme)
 
 	def getFetchMap(self, mypkg, useflags=None, mytree=None):
 		"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     ca300d4fccffb8cd3d0d12f60c7c8c368f049258
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Nov 21 19:22:21 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Nov 21 19:27:17 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ca300d4f

binarytree.populate: avoid unnecessary index re-write (bug 638292)

Exclude hashes from the minimum_keys, so that the Packages index
will not be needlessly re-written due to missing digests.

Bug: https://bugs.gentoo.org/638292

 pym/portage/dbapi/bintree.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index f4e8a1c66..201666c41 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -569,6 +569,11 @@ class binarytree(object):
 	def _populate_local(self):
 		self.dbapi.clear()
 		_instance_key = self.dbapi._instance_key
+		# In order to minimize disk I/O, we never compute digests here.
+		# Therefore we exclude hashes from the minimum_keys, so that
+		# the Packages file will not be needlessly re-written due to
+		# missing digests.
+		minimum_keys = self._pkgindex_keys.difference(self._pkgindex_hashes)
 		if True:
 			pkg_paths = {}
 			self._pkg_paths = pkg_paths
@@ -631,7 +636,7 @@ class binarytree(object):
 									continue
 							except (KeyError, ValueError):
 								continue
-							if not self._pkgindex_keys.difference(d):
+							if not minimum_keys.difference(d):
 								match = d
 								break
 						if match:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8267445cf2f8697f12f1424ecdb4f495dc19f27f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Nov 21 00:45:18 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Nov 21 01:31:21 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8267445c

binarytree.populate: avoid lock when possible (bug 607872)

In order to avoid unecessary lock contention, do not lock
the Packages file unless it needs to be updated. This is
useful when PKGDIR is shared via NFS.

Bug: https://bugs.gentoo.org/607872

 pym/portage/dbapi/bintree.py | 80 +++++++++++++++++++++++++-------------------
 1 file changed, 46 insertions(+), 34 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 95bd5dbf8..ffac8d216 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -523,21 +523,9 @@ class binarytree(object):
 		if self._populating:
 			return
 
-		pkgindex_lock = None
-		try:
-			if os.access(self.pkgdir, os.W_OK):
-				pkgindex_lock = lockfile(self._pkgindex_file,
-					wantnewlockfile=1)
-			self._populating = True
-			self._populate(getbinpkgs, getbinpkg_refresh=getbinpkg_refresh)
-		finally:
-			if pkgindex_lock:
-				unlockfile(pkgindex_lock)
-			self._populating = False
-
-	def _populate(self, getbinpkgs=False, getbinpkg_refresh=True):
-		if (not os.path.isdir(self.pkgdir) and not getbinpkgs):
-			return 0
+		if not os.path.isdir(self.pkgdir) and not getbinpkgs:
+			self.populated = True
+			return
 
 		# Clear all caches in case populate is called multiple times
 		# as may be the case when _global_updates calls populate()
@@ -545,6 +533,41 @@ class binarytree(object):
 		# operate on local packages (getbinpkgs=0).
 		self._remotepkgs = None
 		self.dbapi.clear()
+
+		self._populating = True
+		try:
+			update_pkgindex = self._populate_local()
+
+			if update_pkgindex and self.dbapi.writable:
+				# If the Packages file needs to be updated, then _populate_local
+				# needs to be called once again while the file is locked, so
+				# that changes made by a concurrent process cannot be lost. This
+				# case is avoided when possible, in order to minimize lock
+				# contention.
+				pkgindex_lock = None
+				try:
+					pkgindex_lock = lockfile(self._pkgindex_file,
+						wantnewlockfile=True)
+					update_pkgindex = self._populate_local()
+					if update_pkgindex:
+						self._pkgindex_write(update_pkgindex)
+				finally:
+					if pkgindex_lock:
+						unlockfile(pkgindex_lock)
+
+			if getbinpkgs:
+				if not self.settings.get("PORTAGE_BINHOST"):
+					writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
+						noiselevel=-1)
+				else:
+					self._populate_remote(getbinpkg_refresh=getbinpkg_refresh)
+
+		finally:
+			self._populating = False
+
+		self.populated = True
+
+	def _populate_local(self):
 		_instance_key = self.dbapi._instance_key
 		if True:
 			pkg_paths = {}
@@ -557,7 +580,6 @@ class binarytree(object):
 			pkgindex = self._load_pkgindex()
 			if not self._pkgindex_version_supported(pkgindex):
 				pkgindex = self._new_pkgindex()
-			header = pkgindex.header
 			metadata = {}
 			basename_index = {}
 			for d in pkgindex.packages:
@@ -773,22 +795,16 @@ class binarytree(object):
 				if instance_key not in pkg_paths:
 					del metadata[instance_key]
 
-			# Do not bother to write the Packages index if $PKGDIR/All/ exists
-			# since it will provide no benefit due to the need to read CATEGORY
-			# from xpak.
-			if update_pkgindex and os.access(self.pkgdir, os.W_OK):
+			if update_pkgindex:
 				del pkgindex.packages[:]
 				pkgindex.packages.extend(iter(metadata.values()))
 				self._update_pkgindex_header(pkgindex.header)
-				self._pkgindex_write(pkgindex)
 
-		if getbinpkgs and not self.settings.get("PORTAGE_BINHOST"):
-			writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
-				noiselevel=-1)
+		return pkgindex if update_pkgindex else None
 
-		if not getbinpkgs or 'PORTAGE_BINHOST' not in self.settings:
-			self.populated=1
-			return
+	def _populate_remote(self, getbinpkg_refresh=True):
+
+		self._remote_has_index = False
 		self._remotepkgs = {}
 		for base_url in self.settings["PORTAGE_BINHOST"].split():
 			parsed_url = urlparse(base_url)
@@ -802,7 +818,7 @@ class binarytree(object):
 				user_passwd = user + "@"
 				if ":" in user:
 					user, passwd = user.split(":", 1)
-			port_args = []
+
 			if port is not None:
 				port_str = ":%s" % (port,)
 				if host.endswith(port_str):
@@ -1008,23 +1024,19 @@ class binarytree(object):
 				for d in pkgindex.packages:
 					cpv = _pkg_str(d["CPV"], metadata=d,
 						settings=self.settings)
-					instance_key = _instance_key(cpv)
 					# Local package instances override remote instances
 					# with the same instance_key.
-					if instance_key in metadata:
+					if self.dbapi.cpv_exists(cpv):
 						continue
 
 					d["CPV"] = cpv
 					d["BASE_URI"] = remote_base_uri
 					d["PKGINDEX_URI"] = url
-					self._remotepkgs[instance_key] = d
-					metadata[instance_key] = d
+					self._remotepkgs[self.dbapi._instance_key(cpv)] = d
 					self.dbapi.cpv_inject(cpv)
 
 				self._remote_has_index = True
 
-		self.populated=1
-
 	def inject(self, cpv, filename=None):
 		"""Add a freshly built package to the database.  This updates
 		$PKGDIR/Packages with the new package metadata (including MD5).

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0936d873aedccfb46ca4353c0b52f460287d6682
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 13 02:11:07 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Nov 14 03:14:19 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0936d873

vardbapi.removeFromContents: update NEEDED (bug 637284)

When removing files from CONTENTS, also remove the corresponding
lines from NEEDED, so that they do not corrupt LinkageMap data
for preserve-libs.

Bug: https://bugs.gentoo.org/637284
Reviewed-by: Manuel Rüger <mrueg <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 42 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 04a40b732..b28b1c56c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -39,6 +39,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util._xattr:xattr',
 	'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry',
 	'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap',
+	'portage.util._dyn_libs.NeededEntry:NeededEntry',
 	'portage.util._async.SchedulerInterface:SchedulerInterface',
 	'portage.util._eventloop.EventLoop:EventLoop',
 	'portage.util._eventloop.global_event_loop:global_event_loop',
@@ -1094,18 +1095,55 @@ class vardbapi(dbapi):
 				removed += 1
 
 		if removed:
-			self.writeContentsToContentsFile(pkg, new_contents)
+			# Also remove corresponding NEEDED lines, so that they do
+			# no corrupt LinkageMap data for preserve-libs.
+			needed_filename = os.path.join(pkg.dbdir, LinkageMap._needed_aux_key)
+			new_needed = None
+			try:
+				with io.open(_unicode_encode(needed_filename,
+					encoding=_encodings['fs'], errors='strict'),
+					mode='r', encoding=_encodings['repo.content'],
+					errors='replace') as f:
+					needed_lines = f.readlines()
+			except IOError as e:
+				if e.errno not in (errno.ENOENT, errno.ESTALE):
+					raise
+			else:
+				new_needed = []
+				for l in needed_lines:
+					l = l.rstrip("\n")
+					if not l:
+						continue
+					try:
+						entry = NeededEntry.parse(needed_filename, l)
+					except InvalidData as e:
+						writemsg_level("\n%s\n\n" % (e,),
+							level=logging.ERROR, noiselevel=-1)
+						continue
 
-	def writeContentsToContentsFile(self, pkg, new_contents):
+					filename = os.path.join(root, entry.filename.lstrip(os.sep))
+					if filename in new_contents:
+						new_needed.append(entry)
+
+			self.writeContentsToContentsFile(pkg, new_contents, new_needed=new_needed)
+
+	def writeContentsToContentsFile(self, pkg, new_contents, new_needed=None):
 		"""
 		@param pkg: package to write contents file for
 		@type pkg: dblink
 		@param new_contents: contents to write to CONTENTS file
 		@type new_contents: contents dictionary of the form
 					{u'/path/to/file' : (contents_attribute 1, ...), ...}
+		@param new_needed: new NEEDED entries
+		@type new_needed: list of NeededEntry
 		"""
 		root = self.settings['ROOT']
 		self._bump_mtime(pkg.mycpv)
+		if new_needed is not None:
+			f = atomic_ofstream(os.path.join(pkg.dbdir, LinkageMap._needed_aux_key))
+			for entry in new_needed:
+				f.write(_unicode(entry))
+			f.close()
 		f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS"))
 		write_contents(new_contents, root, f)
 		f.close()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     c5a2a0edc4f4b01b16a274268431fa21f7f678b2
Author:     Daniel Robbins <drobbins <AT> funtoo <DOT> org>
AuthorDate: Sat Oct 14 23:38:05 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 15 00:53:40 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c5a2a0ed

portdbapi: factor out _better_cache class

Better_cache -- now even better :) This version only scans individual
categories on-demand. I have addressed concerns about PMS-compliance by
enhancing the documentation so that developers are aware of what
assumptions to make (and not make) when using better_cache.

Closes: https://github.com/gentoo/portage/pull/219

 pym/portage/dbapi/porttree.py | 124 ++++++++++++++++++++++++------------------
 1 file changed, 71 insertions(+), 53 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 53edcd18f..f5979d2d0 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -16,7 +16,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.package.ebuild.doebuild:doebuild',
 	'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
 	'portage.util.listdir:listdir',
-	'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str',
+	'portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str',
 )
 
 from portage.cache import volatile
@@ -103,6 +103,68 @@ class _dummy_list(list):
 		except ValueError:
 			pass
 
+
+class _better_cache(object):
+
+	"""
+	The purpose of better_cache is to locate catpkgs in repositories using ``os.listdir()`` as much as possible, which
+	is less expensive IO-wise than exhaustively doing a stat on each repo for a particular catpkg. better_cache stores a
+	list of repos in which particular catpkgs appear. Various dbapi methods use better_cache to locate repositories of
+	interest related to particular catpkg rather than performing an exhaustive scan of all repos/overlays.
+
+	Better_cache.items data may look like this::
+
+	  { "sys-apps/portage" : [ repo1, repo2 ] }
+
+	Without better_cache, Portage will get slower and slower (due to excessive IO) as more overlays are added.
+
+	Also note that it is OK if this cache has some 'false positive' catpkgs in it. We use it to search for specific
+	catpkgs listed in ebuilds. The likelihood of a false positive catpkg in our cache causing a problem is extremely
+	low, because the user of our cache is passing us a catpkg that came from somewhere and has already undergone some
+	validation, and even then will further interrogate the short-list of repos we return to gather more information
+	on the catpkg.
+
+	Thus, the code below is optimized for speed rather than painstaking correctness. I have added a note to
+	``dbapi.getRepositories()`` to ensure that developers are aware of this just in case.
+
+	The better_cache has been redesigned to perform on-demand scans -- it will only scan a category at a time, as
+	needed. This should further optimize IO performance by not scanning category directories that are not needed by
+	Portage.
+	"""
+
+	def __init__(self, repositories):
+		self._items = collections.defaultdict(list)
+		self._scanned_cats = set()
+
+		# ordered list of all portree locations we'll scan:
+		self._repo_list = [repo for repo in reversed(list(repositories))
+			if repo.location is not None]
+
+	def __getitem__(self, catpkg):
+		result = self._items.get(catpkg)
+		if result is not None:
+			return result
+
+		cat, pkg = catsplit(catpkg)
+		if cat not in self._scanned_cats:
+			self._scan_cat(cat)
+		return self._items[catpkg]
+
+	def _scan_cat(self, cat):
+		for repo in self._repo_list:
+			cat_dir = repo.location + "/" + cat
+			try:
+				pkg_list = os.listdir(cat_dir)
+			except OSError as e:
+				if e.errno not in (errno.ENOTDIR, errno.ENOENT, errno.ESTALE):
+					raise
+				continue
+			for p in pkg_list:
+				if os.path.isdir(cat_dir + "/" + p):
+					self._items[cat + "/" + p].append(repo)
+		self._scanned_cats.add(cat)
+
+
 class portdbapi(dbapi):
 	"""this tree will scan a portage directory located at root (passed to init)"""
 	portdbapi_instances = _dummy_list()
@@ -346,11 +408,14 @@ class portdbapi(dbapi):
 			return None
 
 	def getRepositories(self, catpkg=None):
+
 		"""
 		With catpkg=None, this will return a complete list of repositories in this dbapi. With catpkg set to a value,
 		this method will return a short-list of repositories that contain this catpkg. Use this second approach if
 		possible, to avoid exhaustively searching all repos for a particular catpkg. It's faster for this method to
-		find the catpkg than for you do it yourself.
+		find the catpkg than for you do it yourself. When specifying catpkg, you should have reasonable assurance that
+		the category is valid and PMS-compliant as the caching mechanism we use does not perform validation checks for
+		categories.
 
 		This function is required for GLEP 42 compliance.
 
@@ -358,7 +423,8 @@ class portdbapi(dbapi):
 		  catpkg; if None, return a list of all Repositories that contain a particular catpkg.
 		@return: a list of repositories.
 		"""
-		if catpkg is not None and self._better_cache is not None and catpkg in self._better_cache:
+
+		if catpkg is not None and self._better_cache is not None:
 			return [repo.name for repo in self._better_cache[catpkg]]
 		return self._ordered_repo_name_list
 
@@ -796,12 +862,7 @@ class portdbapi(dbapi):
 		elif self._better_cache is None:
 			mytrees = self.porttrees
 		else:
-			try:
-				repos = self._better_cache[mycp]
-			except KeyError:
-				mytrees = []
-			else:
-				mytrees = [repo.location for repo in repos]
+			mytrees = [repo.location for repo in self._better_cache[mycp]]
 		for oroot in mytrees:
 			try:
 				file_list = os.listdir(os.path.join(oroot, mycp))
@@ -850,50 +911,7 @@ class portdbapi(dbapi):
 			"minimum-all-ignore-profile", "minimum-visible"):
 			self.xcache[x]={}
 		self.frozen=1
-		self._better_cache = better_cache = collections.defaultdict(list)
-
-		# The purpose of self._better_cache is to perform an initial quick scan of all repositories
-		# using os.listdir(), which is less expensive IO-wise than exhaustively doing a stat on each
-		# repo. self._better_cache stores a list of repos in which particular catpkgs appear.
-		#
-		# For example, better_cache data may look like this:
-		#
-		# { "sys-apps/portage" : [ repo1, repo2 ] }
-		#
-		# Without this tweak, Portage will get slower and slower as more overlays are added.
-		#
-		# Also note that it is OK if this cache has some 'false positive' catpkgs in it. We use it
-		# to search for specific catpkgs listed in ebuilds. The likelihood of a false positive catpkg
-		# in our cache causing a problem is extremely low. Thus, the code below is optimized for
-		# speed rather than painstaking correctness.
-
-		valid_categories = self.settings.categories
-		for repo_loc in reversed(self.porttrees):
-			repo = self.repositories.get_repo_for_location(repo_loc)
-			try:
-				categories = os.listdir(repo_loc)
-			except OSError as e:
-				if e.errno not in (errno.ENOTDIR, errno.ENOENT, errno.ESTALE):
-					raise
-				continue
-
-			for cat in categories:
-				if cat not in valid_categories:
-					continue
-				cat_dir = repo_loc + "/" + cat
-				try:
-					pkg_list = os.listdir(cat_dir)
-				except OSError as e:
-					if e.errno != errno.ENOTDIR:
-						raise
-					continue
-
-				for p in pkg_list:
-					catpkg_dir = cat_dir + "/" + p
-					if not os.path.isdir(catpkg_dir):
-						continue
-					catpkg = cat + "/" + p
-					better_cache[catpkg].append(repo)
+		self._better_cache = _better_cache(self.repositories)
 
 	def melt(self):
 		self.xcache = {}

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Fabian Groffen]

commit:     aac6f2ab43afb78bd183a6d4a06c131cf70bba51
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Aug 24 13:21:26 2017 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Aug 27 09:33:17 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=aac6f2ab

_collision_protect: report progress in work todo

Currently Portage reports its progress in checking collisions forward
every 1000th file like so:

 * checking 4149 files for package collisions
1000 files checked ...
2000 files checked ...
3000 files checked ...
4000 files checked ...
>>> Merging sys-apps/portage-2.3.8 to /

Change it to countdown style so it is easier to anticipate what the
next action will be:

 * checking 4149 files for package collisions
3149 files remaining ...
2149 files remaining ...
1149 files remaining ...
149 files remaining ...
>>> Merging sys-apps/portage-2.3.8 to /

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 7c8f150bb..04a40b732 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3420,13 +3420,14 @@ class dblink(object):
 			dirs_ro = set()
 			symlink_collisions = []
 			destroot = self.settings['ROOT']
+			totfiles = len(file_list) + len(symlink_list)
 			showMessage(_(" %s checking %d files for package collisions\n") % \
-				(colorize("GOOD", "*"), len(file_list) + len(symlink_list)))
+				(colorize("GOOD", "*"), totfiles))
 			for i, (f, f_type) in enumerate(chain(
 				((f, "reg") for f in file_list),
 				((f, "sym") for f in symlink_list))):
 				if i % 1000 == 0 and i != 0:
-					showMessage(_("%d files checked ...\n") % i)
+					showMessage(_("%d files remaining ...\n") % (totfiles - i))
 
 				dest_path = normalize_path(
 					os.path.join(destroot, f.lstrip(os.path.sep)))

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Michał Górny]

commit:     ef1860e39a634696eb6e457cb0cafba95324f53f
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 28 09:27:08 2017 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Tue Feb 28 22:07:08 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ef1860e3

vartree: Use hashlib lazy-import, kill py<2.5 compat

 pym/portage/dbapi/vartree.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 5053801e6..2a612d162 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2014 Gentoo Foundation
+# Copyright 1998-2017 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import division, unicode_literals
@@ -9,6 +9,7 @@ __all__ = [
 
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
+	'hashlib:md5',
 	'portage.checksum:_perform_md5_merge@perform_md5',
 	'portage.data:portage_gid,portage_uid,secpass',
 	'portage.dbapi.dep_expand:dep_expand',
@@ -1118,11 +1119,7 @@ class vardbapi(dbapi):
 		packages that own it. This is used to optimize owner lookups
 		by narrowing the search down to a smaller number of packages.
 		"""
-		try:
-			from hashlib import md5 as _new_hash
-		except ImportError:
-			from md5 import new as _new_hash
-
+		_new_hash = md5
 		_hash_bits = 16
 		_hex_chars = _hash_bits // 4
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Michał Górny]

commit:     0dc2b065e16f1bec22b1730c2b2faf06e01d10af
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 28 09:36:55 2017 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Tue Feb 28 22:07:09 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0dc2b065

vartree: Replace uses of internal portage.checksum._new_md5 var

Use the public hashlib.md5 method that is required for other code in
the module already.

 pym/portage/dbapi/vartree.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 2a612d162..c421dc50b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4690,8 +4690,7 @@ class dblink(object):
 					os.unlink(mysrc)
 					os.symlink(myto, mysrc)
 
-				mymd5 = portage.checksum._new_md5(
-					_unicode_encode(myto)).hexdigest()
+				mymd5 = md5(_unicode_encode(myto)).hexdigest()
 
 			protected = False
 			if stat.S_ISLNK(mymode) or stat.S_ISREG(mymode):
@@ -4725,8 +4724,7 @@ class dblink(object):
 						# of the link target path string is used
 						# for cfgfiledict (symlinks are
 						# protected since bug #485598).
-						destmd5 = portage.checksum._new_md5(
-							_unicode_encode(mydest_link)).hexdigest()
+						destmd5 = md5(_unicode_encode(mydest_link)).hexdigest()
 
 					elif stat.S_ISREG(mydmode):
 						destmd5 = perform_md5(mydest,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     fb25c5d908c22bdd6868c7db44c56ef5cd459fe4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Nov  9 09:01:13 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Nov 10 19:20:59 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=fb25c5d9

preserve-libs: handle manually removed libraries better (bug 599240)

Before a package is merged, prune any libraries from the registry that no
longer exist on disk, in case they have been manually removed. This has
to be done prior to merge, since after merge it is non-trivial to
distinguish these files from files that have just been merged.

The performance impact of this change is negligible. It uses one lstat
call for each preserved library, and the plib_registry.store() call does
not actually write to disk unless it has found something to remove from
the registry.

X-Gentoo-Bug: 599240
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=599240
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 2cb96ea..5053801 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4252,6 +4252,20 @@ class dblink(object):
 		#if we have a file containing previously-merged config file md5sums, grab it.
 		self.vartree.dbapi._fs_lock()
 		try:
+			# This prunes any libraries from the registry that no longer
+			# exist on disk, in case they have been manually removed.
+			# This has to be done prior to merge, since after merge it
+			# is non-trivial to distinguish these files from files
+			# that have just been merged.
+			plib_registry = self.vartree.dbapi._plib_registry
+			if plib_registry:
+				plib_registry.lock()
+				try:
+					plib_registry.load()
+					plib_registry.store()
+				finally:
+					plib_registry.unlock()
+
 			# Always behave like --noconfmem is enabled for downgrades
 			# so that people who don't know about this option are less
 			# likely to get confused when doing upgrade/downgrade cycles.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3d5382f3e2c437285a009db356c719bc3d77bdd9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 19 21:32:32 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun 19 22:17:13 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d5382f3

unmerge-backup: check BUILD_TIME of existing binary package (bug 586410)

Fix the dblink._quickpkg_dblink method to search for a binary package
having identical BUILD_TIME to the installed instance.

X-Gentoo-Bug: 586410
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=586410
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index bfbe356..28ae584 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -5231,12 +5231,19 @@ class dblink(object):
 
 	def _quickpkg_dblink(self, backup_dblink, background, logfile):
 
+		build_time = backup_dblink.getfile('BUILD_TIME')
+		try:
+			build_time = long(build_time.strip())
+		except ValueError:
+			build_time = 0
+
 		trees = QueryCommand.get_db()[self.settings["EROOT"]]
 		bintree = trees["bintree"]
-		binpkg_path = bintree.getname(backup_dblink.mycpv)
-		if os.path.exists(binpkg_path) and \
-			catsplit(backup_dblink.mycpv)[1] not in bintree.invalids:
-			return os.EX_OK
+
+		for binpkg in reversed(
+			bintree.dbapi.match('={}'.format(backup_dblink.mycpv))):
+			if binpkg.build_time == build_time:
+				return os.EX_OK
 
 		self.lockdb()
 		try:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     90e142b8fccba0507687b747478c5e060effcb00
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jun  2 15:46:41 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jun  2 15:47:49 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=90e142b8

binarytree._populate: suppress PORTAGE_SSH_OPTS KeyError (bug 572494)

Use the "get" method to avoid triggering a KeyError message.

Fixes: 39d81c514c33 ("[...]config.__getitem__(): Partially drop backward compatibility for nonexistent keys.")
X-Gentoo-Bug: 572494
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=572494

 pym/portage/dbapi/bintree.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index dc18d30..f483059 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -905,10 +905,9 @@ class binarytree(object):
 						}
 
 						for k in ("PORTAGE_SSH_OPTS",):
-							try:
-								fcmd_vars[k] = self.settings[k]
-							except KeyError:
-								pass
+							v = self.settings.get(k)
+							if v is not None:
+								fcmd_vars[k] = v
 
 						success = portage.getbinpkg.file_get(
 							fcmd=fcmd, fcmd_vars=fcmd_vars)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a0f48531213976c20190e61dea7100b190f5139f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar  5 06:02:42 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar  5 08:16:19 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a0f48531

portdbapi.aux_get: don't cache in memory unless frozen

Fix the portdbapi aux_get method to cache metadata in memory only when
the "frozen" flag has been set, so that applications do not consume more
memory than necessary. This change does not affect depgraph performance,
since depgraph caches the metadata separately (in Package instances).

X-Gentoo-Bug: 576488
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=576488
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/porttree.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 23f3169..1f4374e 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -550,7 +550,7 @@ class portdbapi(dbapi):
 		#finally, we look at our internal cache entry and return the requested data.
 		returnme = [mydata.get(x, "") for x in mylist]
 
-		if cache_me:
+		if cache_me and self.frozen:
 			aux_cache = {}
 			for x in self._aux_cache_keys:
 				aux_cache[x] = mydata.get(x, "")
@@ -817,6 +817,7 @@ class portdbapi(dbapi):
 
 	def melt(self):
 		self.xcache = {}
+		self._aux_cache = {}
 		self.frozen = 0
 
 	def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f0070ca55e61ad8cf3b259797e675c22b17f2bc6
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 24 10:31:50 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jan 24 10:32:36 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f0070ca5

binarytree: fix PORTAGE_BINHOST KeyError (bug 572746)

Use the "get" method to avoid triggering a KeyError.

Fixes: 39d81c514c33 ("[...]config.__getitem__(): Partially drop backward compatibility for nonexistent keys.")
X-Gentoo-Bug: 572746
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=572746

 pym/portage/dbapi/bintree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 9f47436..dc18d30 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -775,7 +775,7 @@ class binarytree(object):
 				self._update_pkgindex_header(pkgindex.header)
 				self._pkgindex_write(pkgindex)
 
-		if getbinpkgs and not self.settings["PORTAGE_BINHOST"]:
+		if getbinpkgs and not self.settings.get("PORTAGE_BINHOST"):
 			writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
 				noiselevel=-1)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     9073762274c112f3d080367ea0fb24edb88564be
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Sep 24 20:25:46 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Sep 24 20:30:06 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=90737622

fakedbapi.cpv_exists: handle KeyError from _instance_key

Since commit 7921e61065502fd0bb08d9dfef6a4493657961bf, cpv_exists
needs to handle KeyError from _instance_key_multi_instance.

Fixes: 7921e6106550 ("binpkg-multi-instance 2 of 7")

 pym/portage/dbapi/virtual.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py
index a59c3b5..f2e841f 100644
--- a/pym/portage/dbapi/virtual.py
+++ b/pym/portage/dbapi/virtual.py
@@ -98,8 +98,12 @@ class fakedbapi(dbapi):
 		return result[:]
 
 	def cpv_exists(self, mycpv, myrepo=None):
-		return self._instance_key(mycpv,
-			support_string=True) in self.cpvdict
+		try:
+			return self._instance_key(mycpv,
+				support_string=True) in self.cpvdict
+		except KeyError:
+			# _instance_key failure
+			return False
 
 	def cp_list(self, mycp, use_cache=1, myrepo=None):
 		# NOTE: Cache can be safely shared with the match cache, since the

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     25c31a95fd1421952f78ed36dab69d8ff8f1a877
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 16 19:03:45 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 16 19:13:23 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=25c31a95

binarytree.get_pkgindex_uri: handle --gebinpkg=n (bug 549666)

Since commit 328dd4712f88cbb8ef390ae9eb471afa1ef781d7,
get_pkgindex_uri triggers AttributeError if --rebuild* options are
used together with --getbinpkg=n.

Fixes: 328dd4712f88 ("binpkg-multi-instance 3 of 7")
X-Gentoo-Bug: 549666
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=549666
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/bintree.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 4043016..f415a63 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1441,9 +1441,10 @@ class binarytree(object):
 	def get_pkgindex_uri(self, cpv):
 		"""Returns the URI to the Packages file for a given package."""
 		uri = None
-		metadata = self._remotepkgs.get(self.dbapi._instance_key(cpv))
-		if metadata is not None:
-			uri = metadata["PKGINDEX_URI"]
+		if self._remotepkgs is not None:
+			metadata = self._remotepkgs.get(self.dbapi._instance_key(cpv))
+			if metadata is not None:
+				uri = metadata["PKGINDEX_URI"]
 		return uri
 
 	def gettbz2(self, pkgname):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8207c1ea97456f71fc32e2c264be27b348677853
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May  9 05:46:04 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May  9 07:31:56 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8207c1ea

VdbMetadataDelta.applyDelta: handle "remove" events properly (bug 547532)

Since commit d800d224ab38c0f524d3fe858ebe201cbfa903c1, emerge --search
could report removed packages as installed, since applyDelta failed to
apply "remove" events unless the cpv exactly matched. Fix it to apply
the "remove" event to each affected package slot, using the same code
which is used to handle replaced packages for "add" events.

Fixes: d800d224ab38 ("Log changes between vdb_metadata.pickle updates")
X-Gentoo-Bug: 547532
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=547532
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/_VdbMetadataDelta.py | 39 ++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/pym/portage/dbapi/_VdbMetadataDelta.py b/pym/portage/dbapi/_VdbMetadataDelta.py
index 2dbb07a..7461f87 100644
--- a/pym/portage/dbapi/_VdbMetadataDelta.py
+++ b/pym/portage/dbapi/_VdbMetadataDelta.py
@@ -139,12 +139,12 @@ class VdbMetadataDelta(object):
 
 	def applyDelta(self, data):
 		packages = self._vardb._aux_cache["packages"]
-		added_slots = {}
+		deltas = {}
 		for delta in data["deltas"]:
 			cpv = delta["package"] + "-" + delta["version"]
+			deltas[cpv] = delta
 			event = delta["event"]
 			if event == "add":
-				added_slots[cpv] = delta
 				# Use aux_get to populate the cache
 				# for this cpv.
 				if cpv not in packages:
@@ -155,19 +155,22 @@ class VdbMetadataDelta(object):
 			elif event == "remove":
 				packages.pop(cpv, None)
 
-		# Remove replaced versions from updated slots
-		for cached_cpv, (mtime, metadata) in list(packages.items()):
-			if cached_cpv in added_slots:
-				continue
-			replaced = False
-			for cpv, delta in added_slots.items():
-				if (cached_cpv.startswith(delta["package"]) and
-					metadata.get("SLOT") == delta["slot"] and
-					cpv_getkey(cached_cpv) == delta["package"]):
-					replaced = True
-					break
-			if replaced:
-				del packages[cached_cpv]
-				del added_slots[cpv]
-				if not added_slots:
-					break
+		if deltas:
+			# Delete removed or replaced versions from affected slots
+			for cached_cpv, (mtime, metadata) in list(packages.items()):
+				if cached_cpv in deltas:
+					continue
+
+				removed = False
+				for cpv, delta in deltas.items():
+					if (cached_cpv.startswith(delta["package"]) and
+						metadata.get("SLOT") == delta["slot"] and
+						cpv_getkey(cached_cpv) == delta["package"]):
+						removed = True
+						break
+
+				if removed:
+					del packages[cached_cpv]
+					del deltas[cpv]
+					if not deltas:
+						break

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1e43b60d734f601cde727d690c7ae4d576146659
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May  4 04:49:19 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May  4 05:13:31 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1e43b60d

bintree.populate: binhost connection failure triggers TypeError (bug 532784)

Since commit 4496ee37d6fa327ada635c67500e82f830141a9e, binhost
connection errors result in a TypeError. Fix it to call the unicode
function correctly (with only a single argument). Also, handle
a possible UnicodeDecodeError.

Fixes: 4496ee37d6fa ("bintree.py: fix str() calls for Python 2 (532784)")
X-Gentoo-Bug: 532784
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532784
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/bintree.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index b37f388..4043016 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -961,8 +961,12 @@ class binarytree(object):
 				# With Python 2, the EnvironmentError message may
 				# contain bytes or unicode, so use _unicode to ensure
 				# safety with all locales (bug #532784).
-				writemsg("!!! %s\n\n" % _unicode(e,
-					_encodings["stdio"], errors="replace"))
+				try:
+					error_msg = _unicode(e)
+				except UnicodeDecodeError as uerror:
+					error_msg = _unicode(uerror.object,
+						encoding='utf_8', errors='replace')
+				writemsg("!!! %s\n\n" % error_msg)
 				del e
 				pkgindex = None
 			if proc is not None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     ba5814b0d747800152e87fc42a348114ff2c6145
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May  2 19:23:17 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May  2 23:10:48 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ba5814b0

dblink: elog failed postinst (bug 547778)

This ensures that there's still an eerror elog message, even if the
failed postinst did not call eerror itself.

X-Gentoo-Bug: 547778
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=547778
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index cacbf96..fca84d1 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4376,8 +4376,9 @@ class dblink(object):
 		if a != os.EX_OK:
 			# It's stupid to bail out here, so keep going regardless of
 			# phase return code.
-			showMessage(_("!!! FAILED postinst: ")+str(a)+"\n",
-				level=logging.ERROR, noiselevel=-1)
+			self._elog("eerror", "postinst", [
+				_("FAILED postinst: %s") % (a,),
+			])
 
 		#update environment settings, library paths. DO NOT change symlinks.
 		env_update(

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3d4abaf758af2106c1112b95ca93a8d85eaa6c8e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 29 02:57:30 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Apr 29 04:18:54 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d4abaf7

_unmerge_protected_symlinks: suggest UNINSTALL_IGNORE (bug 428098)

X-Gentoo-Bug: 428098
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=428098
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c59d778..cacbf96 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2590,6 +2590,9 @@ class dblink(object):
 				os.path.join(real_root, f.lstrip(os.path.sep)))
 
 		msg.append("")
+		msg.append("Use the UNINSTALL_IGNORE variable to exempt specific symlinks")
+		msg.append("from the following search (see the make.conf man page).")
+		msg.append("")
 		msg.append(_("Searching all installed"
 			" packages for files installed via above symlink(s)..."))
 		msg.append("")

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e968291aed91a955c30844ec6d3a99d09c340ddb
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr  6 04:24:50 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr  6 05:04:48 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e968291a

ro_checker: skip parents of EPREFIX dir (bug 544624)

The ro_checker code added in commit
47ef9a0969474f963dc8e52bfbbb8bc075e8d73c incorrectly asserts that the
parent directories of EPREFIX be writable. Fix it to skip the parents,
since they are irrelevant. This does not affect the case where EPREFIX
is empty, since all directories are checked in that case.

Fixes 47ef9a096947: ("Test for read-only filesystems, fixes bug 378869")
X-Gentoo-Bug: 544624
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=544624
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/vartree.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 277c2f1..1c0deab 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3751,6 +3751,7 @@ class dblink(object):
 		line_ending_re = re.compile('[\n\r]')
 		srcroot_len = len(srcroot)
 		ed_len = len(self.settings["ED"])
+		eprefix_len = len(self.settings["EPREFIX"])
 
 		while True:
 
@@ -3792,7 +3793,11 @@ class dblink(object):
 					break
 
 				relative_path = parent[srcroot_len:]
-				dirlist.append(os.path.join(destroot, relative_path))
+				if len(relative_path) >= eprefix_len:
+					# Files are never installed outside of the prefix,
+					# therefore we skip the readonly filesystem check for
+					# parent directories of the prefix (see bug 544624).
+					dirlist.append(os.path.join(destroot, relative_path))
 
 				for fname in files:
 					try:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     c2bebcc4cb631c9c0478bf7545cbe317a72bcd34
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Apr  1 18:39:31 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Apr  1 19:15:50 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c2bebcc4

binarytree.move_ent: fix binpkg-multi-instance _pkg_paths corruption (bug 545252)

When binpkg-multi-instance support was added in commit
328dd4712f88cbb8ef390ae9eb471afa1ef781d7, the code that adjusts
self._pkg_paths in binarytree.move_ent was not fixed to account for the
extra directory in the path.

Fixes: 328dd4712f88 ("binpkg-multi-instance 3 of 7")
X-Gentoo-Bug: 545252
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=545252
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/dbapi/bintree.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 2f0bc47..b37f388 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -462,8 +462,7 @@ class binarytree(object):
 			mynewcpv = _pkg_str(mynewcpv, metadata=metadata)
 			new_path = self.getname(mynewcpv)
 			self._pkg_paths[
-				self.dbapi._instance_key(mynewcpv)] = os.path.join(
-				*new_path.split(os.path.sep)[-2:])
+				self.dbapi._instance_key(mynewcpv)] = new_path[len(self.pkgdir)+1:]
 			if new_path != mytbz2:
 				self._ensure_dir(os.path.dirname(new_path))
 				_movefile(tbz2path, new_path, mysettings=self.settings)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     bd9569342d1f6ca1c21cf34770bc403f8508c1ad
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 18 02:25:04 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar  4 21:32:07 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=bd956934

binpkg-multi-instance 6 of 7

Remove unused binarytree _remove_symlink, _create_symlink,
prevent_collision, _move_to_all, and _move_from_all methods. These are
all related to the oldest PKGDIR layout, which put all of the tbz2
files in $PKGDIR/All, and created symlinks to them in the category
directories. The $PKGDIR/All layout should be practically extinct by
now. Now portage recognizes all existing layouts, or mixtures of them,
and uses the old packages in place. It never puts new packages in
$PKGDIR/All, so there's no need to move packages around to prevent file
name collisions between packages from different categories. It also
only uses regular files (any symlinks are ignored).

 pym/portage/dbapi/bintree.py | 117 ++-----------------------------------------
 1 file changed, 4 insertions(+), 113 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index f5e7303..2f0bc47 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -471,89 +471,11 @@ class binarytree(object):
 
 		return moves
 
-	def _remove_symlink(self, cpv):
-		"""Remove a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink and also remove
-		the ${PKGDIR}/${CATEGORY} directory if empty.  The file will not be
-		removed if os.path.islink() returns False."""
-		mycat, mypkg = catsplit(cpv)
-		mylink = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
-		if os.path.islink(mylink):
-			"""Only remove it if it's really a link so that this method never
-			removes a real package that was placed here to avoid a collision."""
-			os.unlink(mylink)
-		try:
-			os.rmdir(os.path.join(self.pkgdir, mycat))
-		except OSError as e:
-			if e.errno not in (errno.ENOENT,
-				errno.ENOTEMPTY, errno.EEXIST):
-				raise
-			del e
-
-	def _create_symlink(self, cpv):
-		"""Create a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink (and
-		${PKGDIR}/${CATEGORY} directory, if necessary).  Any file that may
-		exist in the location of the symlink will first be removed."""
-		mycat, mypkg = catsplit(cpv)
-		full_path = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
-		self._ensure_dir(os.path.dirname(full_path))
-		try:
-			os.unlink(full_path)
-		except OSError as e:
-			if e.errno != errno.ENOENT:
-				raise
-			del e
-		os.symlink(os.path.join("..", "All", mypkg + ".tbz2"), full_path)
-
 	def prevent_collision(self, cpv):
-		"""Make sure that the file location ${PKGDIR}/All/${PF}.tbz2 is safe to
-		use for a given cpv.  If a collision will occur with an existing
-		package from another category, the existing package will be bumped to
-		${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist."""
-		if not self._all_directory:
-			return
-
-		# Copy group permissions for new directories that
-		# may have been created.
-		for path in ("All", catsplit(cpv)[0]):
-			path = os.path.join(self.pkgdir, path)
-			self._ensure_dir(path)
-			if not os.access(path, os.W_OK):
-				raise PermissionDenied("access('%s', W_OK)" % path)
-
-		full_path = self.getname(cpv)
-		if "All" == full_path.split(os.path.sep)[-2]:
-			return
-		"""Move a colliding package if it exists.  Code below this point only
-		executes in rare cases."""
-		mycat, mypkg = catsplit(cpv)
-		myfile = mypkg + ".tbz2"
-		mypath = os.path.join("All", myfile)
-		dest_path = os.path.join(self.pkgdir, mypath)
-
-		try:
-			st = os.lstat(dest_path)
-		except OSError:
-			st = None
-		else:
-			if stat.S_ISLNK(st.st_mode):
-				st = None
-				try:
-					os.unlink(dest_path)
-				except OSError:
-					if os.path.exists(dest_path):
-						raise
-
-		if st is not None:
-			# For invalid packages, other_cat could be None.
-			other_cat = portage.xpak.tbz2(dest_path).getfile(b"CATEGORY")
-			if other_cat:
-				other_cat = _unicode_decode(other_cat,
-					encoding=_encodings['repo.content'], errors='replace')
-				other_cat = other_cat.strip()
-				other_cpv = other_cat + "/" + mypkg
-				self._move_from_all(other_cpv)
-				self.inject(other_cpv)
-		self._move_to_all(cpv)
+		warnings.warn("The "
+			"portage.dbapi.bintree.binarytree.prevent_collision "
+			"method is deprecated.",
+			DeprecationWarning, stacklevel=2)
 
 	def _ensure_dir(self, path):
 		"""
@@ -589,37 +511,6 @@ class binarytree(object):
 			except PortageException:
 				pass
 
-	def _move_to_all(self, cpv):
-		"""If the file exists, move it.  Whether or not it exists, update state
-		for future getname() calls."""
-		mycat, mypkg = catsplit(cpv)
-		myfile = mypkg + ".tbz2"
-		self._pkg_paths[cpv] = os.path.join("All", myfile)
-		src_path = os.path.join(self.pkgdir, mycat, myfile)
-		try:
-			mystat = os.lstat(src_path)
-		except OSError as e:
-			mystat = None
-		if mystat and stat.S_ISREG(mystat.st_mode):
-			self._ensure_dir(os.path.join(self.pkgdir, "All"))
-			dest_path = os.path.join(self.pkgdir, "All", myfile)
-			_movefile(src_path, dest_path, mysettings=self.settings)
-			self._create_symlink(cpv)
-			self.inject(cpv)
-
-	def _move_from_all(self, cpv):
-		"""Move a package from ${PKGDIR}/All/${PF}.tbz2 to
-		${PKGDIR}/${CATEGORY}/${PF}.tbz2 and update state from getname calls."""
-		self._remove_symlink(cpv)
-		mycat, mypkg = catsplit(cpv)
-		myfile = mypkg + ".tbz2"
-		mypath = os.path.join(mycat, myfile)
-		dest_path = os.path.join(self.pkgdir, mypath)
-		self._ensure_dir(os.path.dirname(dest_path))
-		src_path = os.path.join(self.pkgdir, "All", myfile)
-		_movefile(src_path, dest_path, mysettings=self.settings)
-		self._pkg_paths[cpv] = mypath
-
 	def populate(self, getbinpkgs=0):
 		"populates the binarytree"
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7921e61065502fd0bb08d9dfef6a4493657961bf
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 17 23:04:07 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar  4 21:32:07 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7921e610

binpkg-multi-instance 2 of 7

Add multi-instance support to fakedbapi, which allows multiple instances
with the same cpv to be stored simultaneously, as long as they are
distinguishable using the new _pkg_str build_id, build_time, file_size,
and mtime attributes. This will be used to add multi-instance support to
the bindbapi class (which inherits from fakedbapi).

 pym/portage/dbapi/virtual.py | 113 +++++++++++++++++++++++++++++++++----------
 1 file changed, 87 insertions(+), 26 deletions(-)

diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py
index ba9745c..3b7d10e 100644
--- a/pym/portage/dbapi/virtual.py
+++ b/pym/portage/dbapi/virtual.py
@@ -11,12 +11,17 @@ class fakedbapi(dbapi):
 	"""A fake dbapi that allows consumers to inject/remove packages to/from it
 	portage.settings is required to maintain the dbAPI.
 	"""
-	def __init__(self, settings=None, exclusive_slots=True):
+	def __init__(self, settings=None, exclusive_slots=True,
+		multi_instance=False):
 		"""
 		@param exclusive_slots: When True, injecting a package with SLOT
 			metadata causes an existing package in the same slot to be
 			automatically removed (default is True).
 		@type exclusive_slots: Boolean
+		@param multi_instance: When True, multiple instances with the
+			same cpv may be stored simultaneously, as long as they are
+			distinguishable (default is False).
+		@type multi_instance: Boolean
 		"""
 		self._exclusive_slots = exclusive_slots
 		self.cpvdict = {}
@@ -25,6 +30,56 @@ class fakedbapi(dbapi):
 			from portage import settings
 		self.settings = settings
 		self._match_cache = {}
+		self._set_multi_instance(multi_instance)
+
+	def _set_multi_instance(self, multi_instance):
+		"""
+		Enable or disable multi_instance mode. This should before any
+		packages are injected, so that all packages are indexed with
+		the same implementation of self._instance_key.
+		"""
+		if self.cpvdict:
+			raise AssertionError("_set_multi_instance called after "
+				"packages have already been added")
+		self._multi_instance = multi_instance
+		if multi_instance:
+			self._instance_key = self._instance_key_multi_instance
+		else:
+			self._instance_key = self._instance_key_cpv
+
+	def _instance_key_cpv(self, cpv, support_string=False):
+		return cpv
+
+	def _instance_key_multi_instance(self, cpv, support_string=False):
+		try:
+			return (cpv, cpv.build_id, cpv.file_size, cpv.build_time,
+				cpv.mtime)
+		except AttributeError:
+			if not support_string:
+				raise
+
+		# Fallback for interfaces such as aux_get where API consumers
+		# may pass in a plain string.
+		latest = None
+		for pkg in self.cp_list(cpv_getkey(cpv)):
+			if pkg == cpv and (
+				latest is None or
+				latest.build_time < pkg.build_time):
+				latest = pkg
+
+		if latest is not None:
+			return (latest, latest.build_id, latest.file_size,
+				latest.build_time, latest.mtime)
+
+		raise KeyError(cpv)
+
+	def clear(self):
+		"""
+		Remove all packages.
+		"""
+		self._clear_cache()
+		self.cpvdict.clear()
+		self.cpdict.clear()
 
 	def _clear_cache(self):
 		if self._categories is not None:
@@ -43,7 +98,8 @@ class fakedbapi(dbapi):
 		return result[:]
 
 	def cpv_exists(self, mycpv, myrepo=None):
-		return mycpv in self.cpvdict
+		return self._instance_key(mycpv,
+			support_string=True) in self.cpvdict
 
 	def cp_list(self, mycp, use_cache=1, myrepo=None):
 		# NOTE: Cache can be safely shared with the match cache, since the
@@ -63,7 +119,10 @@ class fakedbapi(dbapi):
 		return list(self.cpdict)
 
 	def cpv_all(self):
-		return list(self.cpvdict)
+		if self._multi_instance:
+			return [x[0] for x in self.cpvdict]
+		else:
+			return list(self.cpvdict)
 
 	def cpv_inject(self, mycpv, metadata=None):
 		"""Adds a cpv to the list of available packages. See the
@@ -99,13 +158,14 @@ class fakedbapi(dbapi):
 			except AttributeError:
 				pass
 
-		self.cpvdict[mycpv] = metadata
+		instance_key = self._instance_key(mycpv)
+		self.cpvdict[instance_key] = metadata
 		if not self._exclusive_slots:
 			myslot = None
 		if myslot and mycp in self.cpdict:
 			# If necessary, remove another package in the same SLOT.
 			for cpv in self.cpdict[mycp]:
-				if mycpv != cpv:
+				if instance_key != self._instance_key(cpv):
 					try:
 						other_slot = cpv.slot
 					except AttributeError:
@@ -115,40 +175,41 @@ class fakedbapi(dbapi):
 							self.cpv_remove(cpv)
 							break
 
-		cp_list = self.cpdict.get(mycp)
-		if cp_list is None:
-			cp_list = []
-			self.cpdict[mycp] = cp_list
-		try:
-			cp_list.remove(mycpv)
-		except ValueError:
-			pass
+		cp_list = self.cpdict.get(mycp, [])
+		cp_list = [x for x in cp_list
+			if self._instance_key(x) != instance_key]
 		cp_list.append(mycpv)
+		self.cpdict[mycp] = cp_list
 
 	def cpv_remove(self,mycpv):
 		"""Removes a cpv from the list of available packages."""
 		self._clear_cache()
 		mycp = cpv_getkey(mycpv)
-		if mycpv in self.cpvdict:
-			del	self.cpvdict[mycpv]
-		if mycp not in self.cpdict:
-			return
-		while mycpv in self.cpdict[mycp]:
-			del self.cpdict[mycp][self.cpdict[mycp].index(mycpv)]
-		if not len(self.cpdict[mycp]):
-			del self.cpdict[mycp]
+		instance_key = self._instance_key(mycpv)
+		self.cpvdict.pop(instance_key, None)
+		cp_list = self.cpdict.get(mycp)
+		if cp_list is not None:
+			cp_list = [x for x in cp_list
+				if self._instance_key(x) != instance_key]
+			if cp_list:
+				self.cpdict[mycp] = cp_list
+			else:
+				del self.cpdict[mycp]
 
 	def aux_get(self, mycpv, wants, myrepo=None):
-		if not self.cpv_exists(mycpv):
+		metadata = self.cpvdict.get(
+			self._instance_key(mycpv, support_string=True))
+		if metadata is None:
 			raise KeyError(mycpv)
-		metadata = self.cpvdict[mycpv]
-		if not metadata:
-			return ["" for x in wants]
 		return [metadata.get(x, "") for x in wants]
 
 	def aux_update(self, cpv, values):
 		self._clear_cache()
-		self.cpvdict[cpv].update(values)
+		metadata = self.cpvdict.get(
+			self._instance_key(cpv, support_string=True))
+		if metadata is None:
+			raise KeyError(cpv)
+		metadata.update(values)
 
 class testdbapi(object):
 	"""A dbapi instance with completely fake functions to get by hitting disk

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     cc5abc989dc6adf31f4caf377cf0761a9431ed2b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 18 02:20:17 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar  4 21:32:07 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cc5abc98

binpkg-multi-instance 5 of 7

Remove unused bintree _pkgindex_cpv_map_latest_build function. This
function is used by binhost clients running older versions of portage
to select the latest builds when their binhost server switches to
FEATURES=binpkg-multi-instance. The function is now unused because
portage is now capable of examining multiple builds and it sorts them
by BUILD_TIME in order to ensure that the latest builds are preferred
when appropriate.

 pym/portage/dbapi/bintree.py | 42 ------------------------------------------
 1 file changed, 42 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 9bc5d98..f5e7303 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -261,48 +261,6 @@ class bindbapi(fakedbapi):
 
 		return filesdict
 
-def _pkgindex_cpv_map_latest_build(pkgindex):
-	"""
-	Given a PackageIndex instance, create a dict of cpv -> metadata map.
-	If multiple packages have identical CPV values, prefer the package
-	with latest BUILD_TIME value.
-	@param pkgindex: A PackageIndex instance.
-	@type pkgindex: PackageIndex
-	@rtype: dict
-	@return: a dict containing entry for the give cpv.
-	"""
-	cpv_map = {}
-
-	for d in pkgindex.packages:
-		cpv = d["CPV"]
-
-		try:
-			cpv = _pkg_str(cpv)
-		except InvalidData:
-			writemsg(_("!!! Invalid remote binary package: %s\n") % cpv,
-				noiselevel=-1)
-			continue
-
-		btime = d.get('BUILD_TIME', '')
-		try:
-			btime = int(btime)
-		except ValueError:
-			btime = None
-
-		other_d = cpv_map.get(cpv)
-		if other_d is not None:
-			other_btime = other_d.get('BUILD_TIME', '')
-			try:
-				other_btime = int(other_btime)
-			except ValueError:
-				other_btime = None
-			if other_btime and (not btime or other_btime > btime):
-				continue
-
-		cpv_map[_pkg_str(cpv)] = d
-
-	return cpv_map
-
 class binarytree(object):
 	"this tree scans for a list of all packages available in PKGDIR"
 	def __init__(self, _unused=DeprecationWarning, pkgdir=None,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4d2987df494105f00021c9fce9e503f673c19549
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 17 08:31:47 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 17 18:27:41 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4d2987df

binarytree: avoid unecessary index regeneration

Since commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de, binarytree
regenerates the 'Packages' index unnecessarily, due to missing REQUIRES
and PROVIDES entries. These entries are not required, so use default
empty values in order to avoid the unnecessary regeneration.

Fixes: f1c1b8a77eeb ("Generate soname dependency metadata (bug 282639)")
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

---
 pym/portage/dbapi/bintree.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 583e208..cd30b67 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -348,7 +348,9 @@ class binarytree(object):
 				"PDEPEND" : "",
 				"PROPERTIES" : "",
 				"PROVIDE" : "",
+				"PROVIDES": "",
 				"RDEPEND" : "",
+				"REQUIRES": "",
 				"RESTRICT": "",
 				"SLOT"    : "0",
 				"USE"     : "",

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4496ee37d6fa327ada635c67500e82f830141a9e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Dec 17 17:33:50 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Dec 17 22:12:37 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4496ee37

bintree.py: fix str() calls for Python 2 (532784)

Avoid a UnicodeDecodeError raised when str(e) converts an exception
to bytes with Python 2. Since this file has unicode_literals enabled,
use literal unicode format strings to format messages for unicode
exceptions. However, with Python 2, an EnvironmentError exception
may contain either bytes or unicode, so use _unicode(errors="replace")
to ensure safety for EnvironmentError with all locales.

Also, convert remaining str() calls to use _unicode() for uniform
behavior regardless of python version.

X-Gentoo-Bug: 532784
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532784
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

---
 pym/portage/dbapi/bintree.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index d7c7f95..1156b66 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -391,7 +391,7 @@ class binarytree(object):
 		# sanity check
 		for atom in (origcp, newcp):
 			if not isjustname(atom):
-				raise InvalidPackageName(str(atom))
+				raise InvalidPackageName(_unicode(atom))
 		mynewcat = catsplit(newcp)[0]
 		origmatches=self.dbapi.cp_list(origcp)
 		moves = 0
@@ -803,8 +803,8 @@ class binarytree(object):
 
 					d["CPV"] = mycpv
 					d["SLOT"] = slot
-					d["MTIME"] = str(s[stat.ST_MTIME])
-					d["SIZE"] = str(s.st_size)
+					d["MTIME"] = _unicode(s[stat.ST_MTIME])
+					d["SIZE"] = _unicode(s.st_size)
 
 					d.update(zip(self._pkgindex_aux_keys,
 						self.dbapi.aux_get(mycpv, self._pkgindex_aux_keys)))
@@ -1024,7 +1024,11 @@ class binarytree(object):
 			except EnvironmentError as e:
 				writemsg(_("\n\n!!! Error fetching binhost package" \
 					" info from '%s'\n") % _hide_url_passwd(base_url))
-				writemsg("!!! %s\n\n" % str(e))
+				# With Python 2, the EnvironmentError message may
+				# contain bytes or unicode, so use _unicode to ensure
+				# safety with all locales (bug #532784).
+				writemsg("!!! %s\n\n" % _unicode(e,
+					_encodings["stdio"], errors="replace"))
 				del e
 				pkgindex = None
 			if proc is not None:
@@ -1242,8 +1246,8 @@ class binarytree(object):
 
 		d["CPV"] = cpv
 		st = os.stat(pkg_path)
-		d["MTIME"] = str(st[stat.ST_MTIME])
-		d["SIZE"] = str(st.st_size)
+		d["MTIME"] = _unicode(st[stat.ST_MTIME])
+		d["SIZE"] = _unicode(st.st_size)
 
 		rel_path = self._pkg_paths[cpv]
 		# record location if it's non-default
@@ -1270,7 +1274,7 @@ class binarytree(object):
 			if profile_path.startswith(profiles_base):
 				profile_path = profile_path[len(profiles_base):]
 			header["PROFILE"] = profile_path
-		header["VERSION"] = str(self._pkgindex_version)
+		header["VERSION"] = _unicode(self._pkgindex_version)
 		base_uri = self.settings.get("PORTAGE_BINHOST_HEADER_URI")
 		if base_uri:
 			header["URI"] = base_uri
@@ -1316,8 +1320,7 @@ class binarytree(object):
 				deps = use_reduce(deps, uselist=use, token_class=token_class)
 				deps = paren_enclose(deps)
 			except portage.exception.InvalidDependString as e:
-				writemsg("%s: %s\n" % (k, str(e)),
-					noiselevel=-1)
+				writemsg("%s: %s\n" % (k, e), noiselevel=-1)
 				raise
 			metadata[k] = deps
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Arfrever Frehtes Taifersar Arahesis]

commit:     f63db9a4598b5a49a989b9176a5c86fa4a70c3c2
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Mon Dec 15 16:24:01 2014 +0000
Commit:     Arfrever Frehtes Taifersar Arahesis <arfrever <AT> apache <DOT> org>
CommitDate: Mon Dec 15 16:24:01 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f63db9a4

portage.dbapi.bintree.binarytree: Delete main-repository-reliant microoptimization in index of binary packages.

This microoptimization cannot work when no main repository exists.
1 call to deprecated portage.repository.config.RepoConfigLoader.mainRepo() function
has been deleted.

---
 pym/portage/dbapi/bintree.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index b56c8c1..d7c7f95 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -360,16 +360,6 @@ class binarytree(object):
 				"repository"   : "",
 			}
 
-			# It is especially important to populate keys like
-			# "repository" that save space when entries can
-			# inherit them from the header. If an existing
-			# pkgindex header already defines these keys, then
-			# they will appropriately override our defaults.
-			main_repo = self.settings.repositories.mainRepo()
-			if main_repo is not None and not main_repo.missing_repo_name:
-				self._pkgindex_default_header_data["repository"] = \
-					main_repo.name
-
 			self._pkgindex_translated_keys = (
 				("DESCRIPTION"   ,   "DESC"),
 				("repository"    ,   "REPO"),

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5424b91133b3b155b0e6ddc08fb46ba301d971f8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Nov  1 15:06:01 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec  7 23:10:48 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5424b911

Add IndexedVardb class.

Searching of installed packages is optimized to take advantage of
vardbdbapi._aux_cache, which is backed by vdb_metadata.pickle.
This class only implements a subset of vardbapi functionality that is
useful for searching incrementally. For this reason, the cp_all method
returns an ordered iterator instead of a list, so that search results
can be displayed incrementally.

X-Gentoo-Bug: 525718
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=525718

---
 pym/portage/dbapi/IndexedVardb.py | 94 +++++++++++++++++++++++++++++++++++++++
 pym/portage/dbapi/vartree.py      | 23 +++++++---
 2 files changed, 110 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/IndexedVardb.py b/pym/portage/dbapi/IndexedVardb.py
new file mode 100644
index 0000000..424defc
--- /dev/null
+++ b/pym/portage/dbapi/IndexedVardb.py
@@ -0,0 +1,94 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import portage
+from portage.dep import Atom
+from portage.versions import _pkg_str
+
+class IndexedVardb(object):
+	"""
+	A vardbapi interface that sacrifices validation in order to
+	improve performance. It takes advantage of vardbdbapi._aux_cache,
+	which is backed by vdb_metadata.pickle. Since _aux_cache is
+	not updated for every single merge/unmerge (see
+	_aux_cache_threshold), the list of packages is obtained directly
+	from the real vardbapi instance. If a package is missing from
+	_aux_cache, then its metadata is obtained using the normal
+	(validated) vardbapi.aux_get method.
+
+	For performance reasons, the match method only supports package
+	name and version constraints.
+	"""
+
+	# Match returns unordered results.
+	match_unordered = True
+
+	_copy_attrs = ('cpv_exists',
+		'_aux_cache_keys', '_cpv_sort_ascending')
+
+	def __init__(self, vardb):
+		self._vardb = vardb
+
+		for k in self._copy_attrs:
+			setattr(self, k, getattr(vardb, k))
+
+		self._cp_map = None
+
+	def cp_all(self):
+		"""
+		Returns an ordered iterator instead of a list, so that search
+		results can be displayed incrementally.
+		"""
+		if self._cp_map is not None:
+			return iter(sorted(self._cp_map))
+
+		return self._iter_cp_all()
+
+	def _iter_cp_all(self):
+		self._cp_map = cp_map = {}
+		previous_cp = None
+		for cpv in self._vardb._iter_cpv_all(sort = True):
+			cp = portage.cpv_getkey(cpv)
+			if cp is not None:
+				cp_list = cp_map.get(cp)
+				if cp_list is None:
+					cp_list = []
+					cp_map[cp] = cp_list
+				cp_list.append(cpv)
+				if previous_cp is not None and \
+					previous_cp != cp:
+					yield previous_cp
+				previous_cp = cp
+
+		if previous_cp is not None:
+			yield previous_cp
+
+	def match(self, atom):
+		"""
+		For performance reasons, only package name and version
+		constraints are supported, and the returned list is
+		unordered.
+		"""
+		if not isinstance(atom, Atom):
+			atom = Atom(atom)
+		cp_list = self._cp_map.get(atom.cp)
+		if cp_list is None:
+			return []
+
+		if atom == atom.cp:
+			return cp_list[:]
+		else:
+			return portage.match_from_list(atom, cp_list)
+
+	def aux_get(self, cpv, attrs, myrepo=None):
+		pkg_data = self._vardb._aux_cache["packages"].get(cpv)
+		if not isinstance(pkg_data, tuple) or \
+			len(pkg_data) != 2 or \
+			not isinstance(pkg_data[1], dict):
+			pkg_data = None
+		if pkg_data is None:
+			# It may be missing from _aux_cache due to
+			# _aux_cache_threshold.
+			return self._vardb.aux_get(cpv, attrs)
+		metadata = pkg_data[1]
+		return [metadata.get(k, "") for k in attrs]

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 4840492..9c8b276 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -435,6 +435,9 @@ class vardbapi(dbapi):
 		(generally this is only necessary in critical sections that
 		involve merge or unmerge of packages).
 		"""
+		return list(self._iter_cpv_all(use_cache=use_cache))
+
+	def _iter_cpv_all(self, use_cache=True, sort=False):
 		returnme = []
 		basepath = os.path.join(self._eroot, VDB_PATH) + os.path.sep
 
@@ -451,26 +454,32 @@ class vardbapi(dbapi):
 					del e
 					return []
 
-		for x in listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1):
+		catdirs = listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1)
+		if sort:
+			catdirs.sort()
+
+		for x in catdirs:
 			if self._excluded_dirs.match(x) is not None:
 				continue
 			if not self._category_re.match(x):
 				continue
-			for y in listdir(basepath + x, EmptyOnError=1, dirsonly=1):
+
+			pkgdirs = listdir(basepath + x, EmptyOnError=1, dirsonly=1)
+			if sort:
+				pkgdirs.sort()
+
+			for y in pkgdirs:
 				if self._excluded_dirs.match(y) is not None:
 					continue
 				subpath = x + "/" + y
 				# -MERGING- should never be a cpv, nor should files.
 				try:
-					if catpkgsplit(subpath) is None:
-						self.invalidentry(self.getpath(subpath))
-						continue
+					subpath = _pkg_str(subpath)
 				except InvalidData:
 					self.invalidentry(self.getpath(subpath))
 					continue
-				returnme.append(subpath)
 
-		return returnme
+				yield subpath
 
 	def cp_all(self, use_cache=1):
 		mylist = self.cpv_all(use_cache=use_cache)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d800d224ab38c0f524d3fe858ebe201cbfa903c1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Nov  6 08:33:03 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec  7 23:10:48 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d800d224

Log changes between vdb_metadata.pickle updates

This adds add support to generate a vdb_metadata_delta.json file
which tracks package merges / unmerges that occur between updates to
vdb_metadata.pickle. IndexedVardb can use the delta together with
vdb_metadata.pickle to reconstruct a complete view of /var/db/pkg,
so that it can avoid expensive listdir calls in /var/db/pkg/*.
Note that vdb_metadata.pickle is only updated periodically, in
order to avoid excessive re-writes of a large file.

In order to test the performance gains from this patch, you need to
generate /var/cache/edb/vdb_metadata_delta.json first, which will
happen automatically if you run 'emerge -p anything' with root
privileges.

---
 pym/portage/dbapi/IndexedVardb.py      |  22 ++++-
 pym/portage/dbapi/_VdbMetadataDelta.py | 153 +++++++++++++++++++++++++++++++++
 pym/portage/dbapi/vartree.py           |  42 ++++++---
 3 files changed, 206 insertions(+), 11 deletions(-)

diff --git a/pym/portage/dbapi/IndexedVardb.py b/pym/portage/dbapi/IndexedVardb.py
index 424defc..38bfeed 100644
--- a/pym/portage/dbapi/IndexedVardb.py
+++ b/pym/portage/dbapi/IndexedVardb.py
@@ -3,6 +3,7 @@
 
 import portage
 from portage.dep import Atom
+from portage.exception import InvalidData
 from portage.versions import _pkg_str
 
 class IndexedVardb(object):
@@ -42,7 +43,26 @@ class IndexedVardb(object):
 		if self._cp_map is not None:
 			return iter(sorted(self._cp_map))
 
-		return self._iter_cp_all()
+		delta_data = self._vardb._cache_delta.loadRace()
+		if delta_data is None:
+			return self._iter_cp_all()
+
+		self._vardb._cache_delta.applyDelta(delta_data)
+
+		self._cp_map = cp_map = {}
+		for cpv in self._vardb._aux_cache["packages"]:
+			try:
+				cpv = _pkg_str(cpv)
+			except InvalidData:
+				continue
+
+			cp_list = cp_map.get(cpv.cp)
+			if cp_list is None:
+				cp_list = []
+				cp_map[cpv.cp] = cp_list
+			cp_list.append(cpv)
+
+		return iter(sorted(self._cp_map))
 
 	def _iter_cp_all(self):
 		self._cp_map = cp_map = {}

diff --git a/pym/portage/dbapi/_VdbMetadataDelta.py b/pym/portage/dbapi/_VdbMetadataDelta.py
new file mode 100644
index 0000000..3e3ff18
--- /dev/null
+++ b/pym/portage/dbapi/_VdbMetadataDelta.py
@@ -0,0 +1,153 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import io
+import json
+import os
+
+from portage import _encodings
+from portage.util import atomic_ofstream
+
+class VdbMetadataDelta(object):
+
+	_format_version  = "1"
+
+	def __init__(self, vardb):
+		self._vardb = vardb
+
+	def initialize(self, timestamp):
+		f = atomic_ofstream(self._vardb._cache_delta_filename, 'w',
+			encoding=_encodings['repo.content'], errors='strict')
+		json.dump({
+			"version": self._format_version,
+			"timestamp": timestamp
+			}, f, ensure_ascii=False)
+		f.close()
+
+	def load(self):
+
+		if not os.path.exists(self._vardb._aux_cache_filename):
+			# If the primary cache doesn't exist yet, then
+			# we can't record a delta against it.
+			return None
+
+		try:
+			with io.open(self._vardb._cache_delta_filename, 'r',
+				encoding=_encodings['repo.content'],
+				errors='strict') as f:
+				cache_obj = json.load(f)
+		except EnvironmentError as e:
+			if e.errno not in (errno.ENOENT, errno.ESTALE):
+				raise
+		except (SystemExit, KeyboardInterrupt):
+			raise
+		except Exception:
+			# Corrupt, or not json format.
+			pass
+		else:
+			try:
+				version = cache_obj["version"]
+			except KeyError:
+				pass
+			else:
+				# Verify that the format version is compatible,
+				# since a newer version of portage may have
+				# written an incompatible file.
+				if version == self._format_version:
+					try:
+						deltas = cache_obj["deltas"]
+					except KeyError:
+						cache_obj["deltas"] = deltas = []
+
+					if isinstance(deltas, list):
+						return cache_obj
+
+		return None
+
+	def loadRace(self):
+		"""
+		This calls self.load() and validates the timestamp
+		against the currently loaded self._vardb._aux_cache. If a
+		concurrent update causes the timestamps to be inconsistent,
+		then it reloads the caches and tries one more time before
+		it aborts. In practice, the race is very unlikely, so
+		this will usually succeed on the first try.
+		"""
+
+		tries = 2
+		while tries:
+			tries -= 1
+			cache_delta = self.load()
+			if cache_delta is not None and \
+				cache_delta.get("timestamp") != \
+				self._vardb._aux_cache.get("timestamp", False):
+				self._vardb._aux_cache_obj = None
+			else:
+				return cache_delta
+
+		return None
+
+	def recordEvent(self, event, cpv, slot, counter):
+
+		self._vardb.lock()
+		try:
+			deltas_obj = self.load()
+
+			if deltas_obj is None:
+				# We can't record meaningful deltas without
+				# a pre-existing state.
+				return
+
+			delta_node = {
+				"event": event,
+				"package": cpv.cp,
+				"version": cpv.version,
+				"slot": slot,
+				"counter": "%s" % counter
+			}
+
+			deltas_obj["deltas"].append(delta_node)
+
+			# Eliminate earlier nodes cancelled out by later nodes
+			# that have identical package and slot attributes.
+			filtered_list = []
+			slot_keys = set()
+			version_keys = set()
+			for delta_node in reversed(deltas_obj["deltas"]):
+				slot_key = (delta_node["package"],
+					delta_node["slot"])
+				version_key = (delta_node["package"],
+					delta_node["version"])
+				if not (slot_key in slot_keys or \
+					version_key in version_keys):
+					filtered_list.append(delta_node)
+					slot_keys.add(slot_key)
+					version_keys.add(version_key)
+
+			filtered_list.reverse()
+			deltas_obj["deltas"] = filtered_list
+
+			f = atomic_ofstream(self._vardb._cache_delta_filename,
+				mode='w', encoding=_encodings['repo.content'])
+			json.dump(deltas_obj, f, ensure_ascii=False)
+			f.close()
+
+		finally:
+			self._vardb.unlock()
+
+	def applyDelta(self, data):
+		packages = self._vardb._aux_cache["packages"]
+		for delta in data["deltas"]:
+			cpv = delta["package"] + "-" + delta["version"]
+			event = delta["event"]
+			if event == "add":
+				# Use aux_get to populate the cache
+				# for this cpv.
+				if cpv not in packages:
+					try:
+						self._vardb.aux_get(cpv, ["DESCRIPTION"])
+					except KeyError:
+						pass
+			elif event == "remove":
+				packages.pop(cpv, None)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 9c8b276..2d4d32d 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -64,6 +64,7 @@ from portage import _os_merge
 from portage import _selinux_merge
 from portage import _unicode_decode
 from portage import _unicode_encode
+from ._VdbMetadataDelta import VdbMetadataDelta
 
 from _emerge.EbuildBuildDir import EbuildBuildDir
 from _emerge.EbuildPhase import EbuildPhase
@@ -179,6 +180,9 @@ class vardbapi(dbapi):
 		self._aux_cache_obj = None
 		self._aux_cache_filename = os.path.join(self._eroot,
 			CACHE_PATH, "vdb_metadata.pickle")
+		self._cache_delta_filename = os.path.join(self._eroot,
+			CACHE_PATH, "vdb_metadata_delta.json")
+		self._cache_delta = VdbMetadataDelta(self)
 		self._counter_path = os.path.join(self._eroot,
 			CACHE_PATH, "counter")
 
@@ -569,22 +573,31 @@ class vardbapi(dbapi):
 		long as at least part of the cache is still valid)."""
 		if self._flush_cache_enabled and \
 			self._aux_cache is not None and \
-			len(self._aux_cache["modified"]) >= self._aux_cache_threshold and \
-			secpass >= 2:
+			secpass >= 2 and \
+			(len(self._aux_cache["modified"]) >= self._aux_cache_threshold or
+			not os.path.exists(self._cache_delta_filename)):
+
+			ensure_dirs(os.path.dirname(self._aux_cache_filename))
+
 			self._owners.populate() # index any unindexed contents
 			valid_nodes = set(self.cpv_all())
 			for cpv in list(self._aux_cache["packages"]):
 				if cpv not in valid_nodes:
 					del self._aux_cache["packages"][cpv]
 			del self._aux_cache["modified"]
-			try:
-				f = atomic_ofstream(self._aux_cache_filename, 'wb')
-				pickle.dump(self._aux_cache, f, protocol=2)
-				f.close()
-				apply_secpass_permissions(
-					self._aux_cache_filename, gid=portage_gid, mode=0o644)
-			except (IOError, OSError) as e:
-				pass
+			timestamp = time.time()
+			self._aux_cache["timestamp"] = timestamp
+
+			f = atomic_ofstream(self._aux_cache_filename, 'wb')
+			pickle.dump(self._aux_cache, f, protocol=2)
+			f.close()
+			apply_secpass_permissions(
+				self._aux_cache_filename, mode=0o644)
+
+			self._cache_delta.initialize(timestamp)
+			apply_secpass_permissions(
+				self._cache_delta_filename, mode=0o644)
+
 			self._aux_cache["modified"] = set()
 
 	@property
@@ -1622,6 +1635,13 @@ class dblink(object):
 				self.dbdir, noiselevel=-1)
 			return
 
+		if self.dbdir is self.dbpkgdir:
+			counter, = self.vartree.dbapi.aux_get(
+				self.mycpv, ["COUNTER"])
+			self.vartree.dbapi._cache_delta.recordEvent(
+				"remove", self.mycpv,
+				self.settings["SLOT"].split("/")[0], counter)
+
 		shutil.rmtree(self.dbdir)
 		# If empty, remove parent category directory.
 		try:
@@ -4232,6 +4252,8 @@ class dblink(object):
 			self.delete()
 			_movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings)
 			self._merged_path(self.dbpkgdir, os.lstat(self.dbpkgdir))
+			self.vartree.dbapi._cache_delta.recordEvent(
+				"add", self.mycpv, slot, counter)
 		finally:
 			self.unlockdb()
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     96c2e57685659211c1e33281e08dfaff04d05b58
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Nov  1 15:05:30 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec  7 23:10:48 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=96c2e576

Add IndexedPortdb class.

The IndexedPortdb class uses pkg_desc_index to optimize searchs for
package names and descriptions. If the package description index is
missing from a particular repository, then all metadata for that
repository is obtained using the normal pordbapi.aux_get method.

This class only implements a subset of portdbapi functionality that is
useful for searching pkg_desc_index incrementally. For this reason,
the cp_all method returns an ordered iterator instead of a list, so
that search results can be displayed incrementally.

X-Gentoo-Bug: 525718
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=525718

---
 pym/portage/dbapi/IndexedPortdb.py | 171 +++++++++++++++++++++++++++++++++++++
 1 file changed, 171 insertions(+)

diff --git a/pym/portage/dbapi/IndexedPortdb.py b/pym/portage/dbapi/IndexedPortdb.py
new file mode 100644
index 0000000..e96f83c
--- /dev/null
+++ b/pym/portage/dbapi/IndexedPortdb.py
@@ -0,0 +1,171 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import io
+import functools
+import operator
+import os
+
+import portage
+from portage import _encodings
+from portage.dep import Atom
+from portage.exception import FileNotFound
+from portage.cache.index.IndexStreamIterator import IndexStreamIterator
+from portage.cache.index.pkg_desc_index import \
+	pkg_desc_index_line_read, pkg_desc_index_node
+from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy
+from portage.versions import _pkg_str
+
+class IndexedPortdb(object):
+	"""
+	A portdbapi interface that uses a package description index to
+	improve performance. If the description index is missing for a
+	particular repository, then all metadata for that repository is
+	obtained using the normal pordbapi.aux_get method.
+
+	For performance reasons, the match method only supports package
+	name and version constraints. For the same reason, the xmatch
+	method is not implemented.
+	"""
+
+	# Match returns unordered results.
+	match_unordered = True
+
+	_copy_attrs = ('cpv_exists', 'findname', 'getFetchMap',
+		'_aux_cache_keys', '_cpv_sort_ascending',
+		'_have_root_eclass_dir')
+
+	def __init__(self, portdb):
+
+		self._portdb = portdb
+
+		for k in self._copy_attrs:
+			setattr(self, k, getattr(portdb, k))
+
+		self._desc_cache = None
+		self._cp_map = None
+		self._unindexed_cp_map = None
+
+	def _init_index(self):
+
+		cp_map = {}
+		desc_cache = {}
+		self._desc_cache = desc_cache
+		self._cp_map = cp_map
+		index_missing = []
+
+		streams = []
+		for repo_path in self._portdb.porttrees:
+			outside_repo = os.path.join(self._portdb.depcachedir,
+				repo_path.lstrip(os.sep))
+			filenames = []
+			for parent_dir in (repo_path, outside_repo):
+				filenames.append(os.path.join(parent_dir,
+					"metadata", "pkg_desc_index"))
+
+			repo_name = self._portdb.getRepositoryName(repo_path)
+
+			try:
+				f = None
+				for filename in filenames:
+					try:
+						f = io.open(filename,
+							encoding=_encodings["repo.content"])
+					except IOError as e:
+						if e.errno not in (errno.ENOENT, errno.ESTALE):
+							raise
+					else:
+						break
+
+				if f is None:
+					raise FileNotFound(filename)
+
+				streams.append(iter(IndexStreamIterator(f,
+					functools.partial(pkg_desc_index_line_read,
+					repo = repo_name))))
+			except FileNotFound:
+				index_missing.append(repo_path)
+
+		if index_missing:
+			self._unindexed_cp_map = {}
+
+			class _NonIndexedStream(object):
+				def __iter__(self_):
+					for cp in self._portdb.cp_all(
+						trees = index_missing):
+						# Don't call cp_list yet, since it's a waste
+						# if the package name does not match the current
+						# search.
+						self._unindexed_cp_map[cp] = index_missing
+						yield pkg_desc_index_node(cp, (), None)
+
+			streams.append(iter(_NonIndexedStream()))
+
+		if streams:
+			if len(streams) == 1:
+				cp_group_iter = ([node] for node in streams[0])
+			else:
+				cp_group_iter = MultiIterGroupBy(streams,
+					key = operator.attrgetter("cp"))
+
+			for cp_group in cp_group_iter:
+
+				new_cp = None
+				cp_list = cp_map.get(cp_group[0].cp)
+				if cp_list is None:
+					new_cp = cp_group[0].cp
+					cp_list = []
+					cp_map[cp_group[0].cp] = cp_list
+
+				for entry in cp_group:
+					cp_list.extend(entry.cpv_list)
+					if entry.desc is not None:
+						for cpv in entry.cpv_list:
+							desc_cache[cpv] = entry.desc
+
+				if new_cp is not None:
+					yield cp_group[0].cp
+
+	def cp_all(self):
+		"""
+		Returns an ordered iterator instead of a list, so that search
+		results can be displayed incrementally.
+		"""
+		if self._cp_map is None:
+			return self._init_index()
+		return iter(sorted(self._cp_map))
+
+	def match(self, atom):
+		"""
+		For performance reasons, only package name and version
+		constraints are supported, and the returned list is
+		unordered.
+		"""
+		if not isinstance(atom, Atom):
+			atom = Atom(atom)
+		cp_list = self._cp_map.get(atom.cp)
+		if cp_list is None:
+			return []
+
+		if self._unindexed_cp_map is not None:
+			try:
+				unindexed = self._unindexed_cp_map.pop(atom.cp)
+			except KeyError:
+				pass
+			else:
+				cp_list.extend(self._portdb.cp_list(atom.cp,
+					mytree=unindexed))
+
+		if atom == atom.cp:
+			return cp_list[:]
+		else:
+			return portage.match_from_list(atom, cp_list)
+
+	def aux_get(self, cpv, attrs, myrepo=None):
+		if len(attrs) == 1 and attrs[0] == "DESCRIPTION":
+			try:
+				return [self._desc_cache[cpv]]
+			except KeyError:
+				pass
+		return self._portdb.aux_get(cpv, attrs)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     cea71687f6bbe0077d3c6271a39b15f6cc8bf325
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Dec  7 00:39:54 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec  7 00:57:22 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cea71687

dblink._protect: disable config protect for identical files (531854)

Since commit 02417188225758b1822d176abd8902a92300a371, config
protection was triggered for identical files if
config-protect-if-modified was not enabled in FEATURES. Fix it
to skip config protection in this case.

Fixes: 024171882257 ("CONFIG_PROTECT: protect symlinks, bug #485598")
X-Gentoo-Bug: 531854
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=531854
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

---
 pym/portage/dbapi/vartree.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index df031cd..4840492 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4856,8 +4856,10 @@ class dblink(object):
 
 		if protected and dest_mode is not None:
 			# we have a protection path; enable config file management.
-			if src_md5 != dest_md5 and \
-				src_md5 == cfgfiledict.get(dest_real, [None])[0]:
+			if src_md5 == dest_md5:
+				protected = False
+
+			elif src_md5 == cfgfiledict.get(dest_real, [None])[0]:
 				# An identical update has previously been
 				# merged.  Skip it unless the user has chosen
 				# --noconfmem.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     317629c7e7acbae32b56d1dec86d11bc038fc976
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 13 00:28:45 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Nov 18 01:35:55 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=317629c7

dblink: case insensitive support for bug #524236

This adds a dblink._contents attribute with methods that provide
an interface for contents operations with "implicit" case handling.
The new methods are implemented in a separate
ContentsCaseSensitivityManager class, in order to avoid adding more
bloat to vartree.py.

X-Gentoo-Bug: 524236
X-Gentoo-Url: https://bugs.gentoo.org/show_bug.cgi?id=524236
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

---
 .../dbapi/_ContentsCaseSensitivityManager.py       | 93 ++++++++++++++++++++++
 pym/portage/dbapi/vartree.py                       |  3 +
 2 files changed, 96 insertions(+)

diff --git a/pym/portage/dbapi/_ContentsCaseSensitivityManager.py b/pym/portage/dbapi/_ContentsCaseSensitivityManager.py
new file mode 100644
index 0000000..c479ec9
--- /dev/null
+++ b/pym/portage/dbapi/_ContentsCaseSensitivityManager.py
@@ -0,0 +1,93 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+class ContentsCaseSensitivityManager(object):
+	"""
+	Implicitly handles case transformations that are needed for
+	case-insensitive support.
+	"""
+
+	def __init__(self, db):
+		"""
+		@param db: A dblink instance
+		@type db: vartree.dblink
+		"""
+		self.getcontents = db.getcontents
+
+		if "case-insensitive-fs" in db.settings.features:
+			self.unmap_key = self._unmap_key_case_insensitive
+			self.contains = self._contains_case_insensitive
+			self.keys = self._keys_case_insensitive
+
+		self._contents_insensitive = None
+		self._reverse_key_map = None
+
+	def clear_cache(self):
+		"""
+		Clear all cached contents data.
+		"""
+		self._contents_insensitive = None
+		self._reverse_key_map = None
+
+	def keys(self):
+		"""
+		Iterate over all contents keys, which are transformed to
+		lowercase when appropriate, for use in case-insensitive
+		comparisons.
+		@rtype: iterator
+		@return: An iterator over all the contents keys
+		"""
+		return iter(self.getcontents())
+
+	def contains(self, key):
+		"""
+		Check if the given key is contained in the contents, using
+		case-insensitive comparison when appropriate.
+		@param key: A filesystem path (including ROOT and EPREFIX)
+		@type key: str
+		@rtype: bool
+		@return: True if the given key is contained in the contents,
+			False otherwise
+		"""
+		return key in self.getcontents()
+
+	def unmap_key(self, key):
+		"""
+		Map a key (from the keys method) back to its case-preserved
+		form.
+		@param key: A filesystem path (including ROOT and EPREFIX)
+		@type key: str
+		@rtype: str
+		@return: The case-preserved form of key
+		"""
+		return key
+
+	def _case_insensitive_init(self):
+		"""
+		Initialize data structures for case-insensitive support.
+		"""
+		self._contents_insensitive = dict(
+			(k.lower(), v) for k, v in self.getcontents().items())
+		self._reverse_key_map = dict(
+			(k.lower(), k) for k in self.getcontents())
+
+	def _keys_case_insensitive(self):
+		if self._contents_insensitive is None:
+			self._case_insensitive_init()
+		return iter(self._contents_insensitive)
+
+	_keys_case_insensitive.__doc__ = keys.__doc__
+
+	def _contains_case_insensitive(self, key):
+		if self._contents_insensitive is None:
+			self._case_insensitive_init()
+		return key.lower() in self._contents_insensitive
+
+	_contains_case_insensitive.__doc__ = contains.__doc__
+
+	def _unmap_key_case_insensitive(self, key):
+		if self._reverse_key_map is None:
+			self._case_insensitive_init()
+		return self._reverse_key_map[key]
+
+	_unmap_key_case_insensitive.__doc__ = unmap_key.__doc__

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 8b06f4c..81059b1 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -69,6 +69,7 @@ from _emerge.EbuildPhase import EbuildPhase
 from _emerge.emergelog import emergelog
 from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
 from _emerge.SpawnProcess import SpawnProcess
+from ._ContentsCaseSensitivityManager import ContentsCaseSensitivityManager
 
 import errno
 import fnmatch
@@ -1525,6 +1526,7 @@ class dblink(object):
 		# When necessary, this attribute is modified for
 		# compliance with RESTRICT=preserve-libs.
 		self._preserve_libs = "preserve-libs" in mysettings.features
+		self._contents = ContentsCaseSensitivityManager(self)
 
 	def __hash__(self):
 		return hash(self._hash_key)
@@ -1612,6 +1614,7 @@ class dblink(object):
 		self.contentscache = None
 		self._contents_inodes = None
 		self._contents_basenames = None
+		self._contents.clear_cache()
 
 	def getcontents(self):
 		"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     42ccbf74f36ae7ed31045607ac872717808a065b
Author:     David James <davidjames <AT> google <DOT> com>
AuthorDate: Sun Nov  2 18:07:53 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Nov  2 18:07:53 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=42ccbf74

If a binhost file sets a TTL header, honor it.

Reviewed-on: https://chromium-review.googlesource.com/225279
Reviewed-on: https://chromium-review.googlesource.com/225423
Tested-by: David James <davidjames <AT> chromium.org>
Reviewed-by: Mike Frysinger <vapier <AT> chromium.org>
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

---
 pym/portage/dbapi/bintree.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 229ce3b..a5d7ac9 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -43,6 +43,7 @@ import subprocess
 import sys
 import tempfile
 import textwrap
+import time
 import traceback
 import warnings
 from gzip import GzipFile
@@ -879,6 +880,11 @@ class binarytree(object):
 				if e.errno != errno.ENOENT:
 					raise
 			local_timestamp = pkgindex.header.get("TIMESTAMP", None)
+			try:
+				download_timestamp = \
+					float(pkgindex.header.get("DOWNLOAD_TIMESTAMP", 0))
+			except ValueError:
+				download_timestamp = 0
 			remote_timestamp = None
 			rmt_idx = self._new_pkgindex()
 			proc = None
@@ -890,6 +896,15 @@ class binarytree(object):
 				url = base_url.rstrip("/") + "/Packages"
 				f = None
 
+				try:
+					ttl = float(pkgindex.header.get("TTL", 0))
+				except ValueError:
+					pass
+				else:
+					if download_timestamp and ttl and \
+						download_timestamp + ttl > time.time():
+						raise UseCachedCopyOfRemoteIndex()
+
 				# Don't use urlopen for https, since it doesn't support
 				# certificate/hostname verification (bug #469888).
 				if parsed_url.scheme not in ('https',):
@@ -1022,6 +1037,7 @@ class binarytree(object):
 					pass
 			if pkgindex is rmt_idx:
 				pkgindex.modified = False # don't update the header
+				pkgindex.header["DOWNLOAD_TIMESTAMP"] = "%d" % time.time()
 				try:
 					ensure_dirs(os.path.dirname(pkgindex_file))
 					f = atomic_ofstream(pkgindex_file)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     34e3661b76e6c8a95e57c95d78ba673be32cc2b1
Author:     Bertrand SIMONNET <bsimonnet <AT> chromium <DOT> org>
AuthorDate: Thu Sep 11 08:06:37 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Sep 11 23:28:46 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=34e3661b

Fix read only detection logic

Portage should check for a read only file system relative to ROOT and
not /.

Acked-by: Zac Medico <zmedico <AT> gentoo.org>
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

---
 pym/portage/dbapi/vartree.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 5b947dd..8bc6073 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3723,7 +3723,7 @@ class dblink(object):
 					break
 
 				relative_path = parent[srcroot_len:]
-				dirlist.append(os.path.join("/", relative_path))
+				dirlist.append(os.path.join(destroot, relative_path))
 
 				for fname in files:
 					try:
@@ -3849,8 +3849,7 @@ class dblink(object):
 			msg = textwrap.wrap(msg, 70)
 			msg.append("")
 			for f in rofilesystems:
-				msg.append("\t%s" % os.path.join(destroot,
-					f.lstrip(os.path.sep)))
+				msg.append("\t%s" % f)
 			msg.append("")
 			self._elog("eerror", "preinst", msg)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Brian Dolbec]

commit:     dcb095ec652562961a317effcca6334ee2cb2c0d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Apr  4 15:32:08 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Apr  4 20:26:05 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=dcb095ec

dbapi/vartree.py: whitespace cleanup

---
 pym/portage/dbapi/vartree.py | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 6417a56..4a27092 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1465,7 +1465,7 @@ class dblink(object):
 		"""
 		Creates a DBlink object for a given CPV.
 		The given CPV may not be present in the database already.
-		
+
 		@param cat: Category
 		@type cat: String
 		@param pkg: Package (PV)
@@ -1796,7 +1796,7 @@ class dblink(object):
 		calls postrm
 		calls cleanrm
 		calls env_update
-		
+
 		@param pkgfiles: files to unmerge (generally self.getcontents() )
 		@type pkgfiles: Dictionary
 		@param trimworld: Unused
@@ -2105,10 +2105,10 @@ class dblink(object):
 
 	def _unmerge_pkgfiles(self, pkgfiles, others_in_slot):
 		"""
-		
+
 		Unmerges the contents of a package from the liveFS
 		Removes the VDB entry for self
-		
+
 		@param pkgfiles: typically self.getcontents()
 		@type pkgfiles: Dictionary { filename: [ 'type', '?', 'md5sum' ] }
 		@param others_in_slot: all dblink instances in this slot, excluding self
@@ -2183,7 +2183,7 @@ class dblink(object):
 					# in a totally undefined state, hence we just bleed
 					# like hell and continue to hopefully finish all our
 					# administrative and pkg_postinst stuff.
-					self._eerror("postrm", 
+					self._eerror("postrm",
 						["Could not chmod or unlink '%s': %s" % \
 						(file_name, ose)])
 				else:
@@ -2237,7 +2237,7 @@ class dblink(object):
 						_unicode_encode(obj,
 							encoding=_encodings['merge'], errors='strict')
 					except UnicodeEncodeError:
-						# The package appears to have been merged with a 
+						# The package appears to have been merged with a
 						# different value of sys.getfilesystemencoding(),
 						# so fall back to utf_8 if appropriate.
 						try:
@@ -2681,7 +2681,7 @@ class dblink(object):
 							show_unmerge("!!!", "", "sym", obj)
 
 	def isowner(self, filename, destroot=None):
-		""" 
+		"""
 		Check if a file belongs to this package. This may
 		result in a stat call for the parent directory of
 		every installed file, since the inode numbers are
@@ -2813,7 +2813,7 @@ class dblink(object):
 								encoding=_encodings['merge'],
 								errors='strict')
 					except UnicodeEncodeError:
-						# The package appears to have been merged with a 
+						# The package appears to have been merged with a
 						# different value of sys.getfilesystemencoding(),
 						# so fall back to utf_8 if appropriate.
 						try:
@@ -2925,7 +2925,7 @@ class dblink(object):
 					_unicode_encode(f_abs,
 						encoding=_encodings['merge'], errors='strict')
 				except UnicodeEncodeError:
-					# The package appears to have been merged with a 
+					# The package appears to have been merged with a
 					# different value of sys.getfilesystemencoding(),
 					# so fall back to utf_8 if appropriate.
 					try:
@@ -3390,7 +3390,7 @@ class dblink(object):
 					_unicode_encode(path,
 						encoding=_encodings['merge'], errors='strict')
 				except UnicodeEncodeError:
-					# The package appears to have been merged with a 
+					# The package appears to have been merged with a
 					# different value of sys.getfilesystemencoding(),
 					# so fall back to utf_8 if appropriate.
 					try:
@@ -3436,7 +3436,7 @@ class dblink(object):
 			for path, s in path_list:
 				msg.append("\t%s" % path)
 		msg.append("")
-		msg.append(_("See the Gentoo Security Handbook " 
+		msg.append(_("See the Gentoo Security Handbook "
 			"guide for advice on how to proceed."))
 
 		self._eerror("preinst", msg)
@@ -3507,9 +3507,9 @@ class dblink(object):
 	def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
 		mydbapi=None, prev_mtimes=None, counter=None):
 		"""
-		
+
 		This function does the following:
-		
+
 		calls get_ro_checker to retrieve a function for checking whether Portage
 		will write to a read-only filesystem, then runs it against the directory list
 		calls self._preserve_libs if FEATURES=preserve-libs
@@ -3519,7 +3519,7 @@ class dblink(object):
 		unmerges old version (if required)
 		calls doebuild(mydo=pkg_postinst)
 		calls env_update
-		
+
 		@param srcroot: Typically this is ${D}
 		@type srcroot: String (Path)
 		@param destroot: ignored, self.settings['ROOT'] is used instead
@@ -3536,7 +3536,7 @@ class dblink(object):
 		@return:
 		1. 0 on success
 		2. 1 on failure
-		
+
 		secondhand is a list of symlinks that have been skipped due to their target
 		not existing; we will merge these symlinks at a later time.
 		"""
@@ -3891,7 +3891,7 @@ class dblink(object):
 			msg = _("This package will overwrite one or more files that"
 			" may belong to other packages (see list below).")
 			if not (collision_protect or protect_owned):
-				msg += _(" Add either \"collision-protect\" or" 
+				msg += _(" Add either \"collision-protect\" or"
 				" \"protect-owned\" to FEATURES in"
 				" make.conf if you would like the merge to abort"
 				" in cases like this. See the make.conf man page for"
@@ -4404,10 +4404,10 @@ class dblink(object):
 
 	def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime):
 		"""
-		
+
 		This function handles actual merging of the package contents to the livefs.
 		It also handles config protection.
-		
+
 		@param srcroot: Where are we copying files from (usually ${D})
 		@type srcroot: String (Path)
 		@param destroot: Typically ${ROOT}
@@ -4428,7 +4428,7 @@ class dblink(object):
 		@return:
 		1. True on failure
 		2. None otherwise
-		
+
 		"""
 
 		showMessage = self._display_merge
@@ -4689,7 +4689,7 @@ class dblink(object):
 				protected = self.isprotected(mydest)
 				if mydmode != None:
 					# destination file exists
-					
+
 					if stat.S_ISDIR(mydmode):
 						# install of destination is blocked by an existing directory with the same name
 						newdest = self._new_backup_path(mydest)
@@ -4931,7 +4931,7 @@ class dblink(object):
 		if not os.path.exists(self.dbdir+"/"+fname):
 			return ""
 		with io.open(_unicode_encode(os.path.join(self.dbdir, fname),
-			encoding=_encodings['fs'], errors='strict'), 
+			encoding=_encodings['fs'], errors='strict'),
 			mode='r', encoding=_encodings['repo.content'], errors='replace'
 			) as f:
 			return f.read()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Alexander Berntsen]

commit:     099941cc4788f469b30b9ba7222f92fba28e6b28
Author:     Alexander Berntsen <alexander <AT> plaimi <DOT> net>
AuthorDate: Mon Feb 24 00:33:43 2014 +0000
Commit:     Alexander Berntsen <bernalex <AT> gentoo <DOT> org>
CommitDate: Mon Feb 24 00:50:54 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=099941cc

Handle bad PORTAGE_BINHOST gracefully (bug 502288)

---
 pym/portage/dbapi/bintree.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 022300e..7b61c1f 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -909,6 +909,11 @@ class binarytree(object):
 								traceback.print_exc()
 
 							raise
+					except ValueError as _:
+						writemsg("\n\n!!! Invalid PORTAGE_BINHOST value "
+								 "'%s'.\nSee man make.conf(5) for more info."
+								 "\n" % url.lstrip(), noiselevel=-1)
+						exit(1)
 
 				if f is None:
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Chris Reffett]

commit:     dd7b8544d5f2553284d22a27ca7fd6c43229fac9
Author:     Chris Reffett <creffett <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 27 04:44:37 2014 +0000
Commit:     Chris Reffett <creffett <AT> gentoo <DOT> org>
CommitDate: Fri Feb  7 22:34:52 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=dd7b8544

Remove "my" prefix from my{dir,link,file}list

---
 pym/portage/dbapi/vartree.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index b593365..6417a56 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3688,9 +3688,9 @@ class dblink(object):
 			unicode_error = False
 			eagain_error = False
 
-			myfilelist = []
-			mydirlist = []
-			mylinklist = []
+			filelist = []
+			dirlist = []
+			linklist = []
 			paths_with_newlines = []
 			def onerror(e):
 				raise
@@ -3723,7 +3723,7 @@ class dblink(object):
 					break
 
 				relative_path = parent[srcroot_len:]
-				mydirlist.append(os.path.join("/", relative_path))
+				dirlist.append(os.path.join("/", relative_path))
 
 				for fname in files:
 					try:
@@ -3754,12 +3754,12 @@ class dblink(object):
 
 					file_mode = os.lstat(fpath).st_mode
 					if stat.S_ISREG(file_mode):
-						myfilelist.append(relative_path)
+						filelist.append(relative_path)
 					elif stat.S_ISLNK(file_mode):
 						# Note: os.walk puts symlinks to directories in the "dirs"
 						# list and it does not traverse them since that could lead
 						# to an infinite recursion loop.
-						mylinklist.append(relative_path)
+						linklist.append(relative_path)
 
 						myto = _unicode_decode(
 							_os.readlink(_unicode_encode(fpath,
@@ -3794,7 +3794,7 @@ class dblink(object):
 		# If there are no files to merge, and an installed package in the same
 		# slot has files, it probably means that something went wrong.
 		if self.settings.get("PORTAGE_PACKAGE_EMPTY_ABORT") == "1" and \
-			not myfilelist and not mylinklist and others_in_slot:
+			not filelist and not linklist and others_in_slot:
 			installed_files = None
 			for other_dblink in others_in_slot:
 				installed_files = other_dblink.getcontents()
@@ -3839,7 +3839,7 @@ class dblink(object):
 
 		# Check for read-only filesystems.
 		ro_checker = get_ro_checker()
-		rofilesystems = ro_checker(mydirlist)
+		rofilesystems = ro_checker(dirlist)
 
 		if rofilesystems:
 			msg = _("One or more files installed to this package are "
@@ -3868,7 +3868,7 @@ class dblink(object):
 			blockers = []
 		collisions, symlink_collisions, plib_collisions = \
 			self._collision_protect(srcroot, destroot,
-			others_in_slot + blockers, myfilelist, mylinklist)
+			others_in_slot + blockers, filelist, linklist)
 
 		if symlink_collisions:
 			# Symlink collisions need to be distinguished from other types

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Arfrever Frehtes Taifersar Arahesis]

commit:     be7e0cc700afa4c16cabdd7a692c082bc2c000b1
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Sun Jan 19 08:15:32 2014 +0000
Commit:     Arfrever Frehtes Taifersar Arahesis <arfrever <AT> apache <DOT> org>
CommitDate: Sun Jan 19 08:15:32 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=be7e0cc7

Deprecate portage.dbapi.porttree.portdbapi.porttree_root and portage.dbapi.porttree.portdbapi.eclassdb.

---
 pym/portage/dbapi/porttree.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index fc3fc03..25ff27f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2013 Gentoo Foundation
+# Copyright 1998-2014 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -111,10 +111,16 @@ class portdbapi(dbapi):
 
 	@property
 	def porttree_root(self):
+		warnings.warn("portage.dbapi.porttree.portdbapi.porttree_root is deprecated in favor of portage.repository.config.RepoConfig.location "
+			"(available as repositories[repo_name].location attribute of instances of portage.dbapi.porttree.portdbapi class)",
+			DeprecationWarning, stacklevel=2)
 		return self.settings.repositories.mainRepoLocation()
 
 	@property
 	def eclassdb(self):
+		warnings.warn("portage.dbapi.porttree.portdbapi.eclassdb is deprecated in favor of portage.repository.config.RepoConfig.eclass_db "
+			"(available as repositories[repo_name].eclass_db attribute of instances of portage.dbapi.porttree.portdbapi class)",
+			DeprecationWarning, stacklevel=2)
 		main_repo = self.repositories.mainRepo()
 		if main_repo is None:
 			return None

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Brian Dolbec]

commit:     c08e18723f34cb33a025b7cfa999f03cced9a67b
Author:     Sebastian Luther <SebastianLuther <AT> gmx <DOT> de>
AuthorDate: Thu Dec  5 13:36:51 2013 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  5 15:16:47 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c08e1872

Don't filter USE for binary packages with IUSE (bug 485920)

USE may contain values that aren't present in IUSE for any
supported EAPI. This for example breaks use dependencies on
USE_EXPAND-values.

The behavior for binary packages is now in line with what is
done for installed packages.

Note that this filtering was not active during merge.

URL: https://bugs.gentoo.org/485920

---
 pym/portage/dbapi/bintree.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 61ac6b5..b1f67ae 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1283,11 +1283,6 @@ class binarytree(object):
 
 	def _eval_use_flags(self, cpv, metadata):
 		use = frozenset(metadata["USE"].split())
-		raw_use = use
-		iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split())
-		use = [f for f in use if f in iuse]
-		use.sort()
-		metadata["USE"] = " ".join(use)
 		for k in self._pkgindex_use_evaluated_keys:
 			if k.endswith('DEPEND'):
 				token_class = Atom
@@ -1296,7 +1291,7 @@ class binarytree(object):
 
 			try:
 				deps = metadata[k]
-				deps = use_reduce(deps, uselist=raw_use, token_class=token_class)
+				deps = use_reduce(deps, uselist=use, token_class=token_class)
 				deps = paren_enclose(deps)
 			except portage.exception.InvalidDependString as e:
 				writemsg("%s: %s\n" % (k, str(e)),

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     b7c7b359af59d857c0c0c64e99cec623df97c137
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 17 23:32:17 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 17 23:32:17 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b7c7b359

Handle newline in symlink target for bug #485226.

---
 pym/portage/dbapi/vartree.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index cbdf75b..ed62323 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3753,6 +3753,13 @@ class dblink(object):
 						# to an infinite recursion loop.
 						mylinklist.append(relative_path)
 
+						myto = _unicode_decode(
+							_os.readlink(_unicode_encode(fpath,
+							encoding=_encodings['merge'], errors='strict')),
+							encoding=_encodings['merge'], errors='replace')
+						if line_ending_re.search(myto) is not None:
+							paths_with_newlines.append(relative_path)
+
 				if unicode_error:
 					break
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     ef14017dfaeeaa0b1dacdc4641c30583128f7f08
Author:     Alexander Berntsen <alexander <AT> plaimi <DOT> net>
AuthorDate: Sun Aug 11 22:18:40 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug 12 03:22:08 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ef14017d

vartree.py: Add link for collisions, bug #449918

Link to <http://wiki.gentoo.org/wiki/Knowledge_Base:Blockers> on
file collisions.

---
 pym/portage/dbapi/vartree.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index fe61229..cbdf75b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3870,7 +3870,9 @@ class dblink(object):
 				" enough information to determine if a real problem"
 				" exists. Please do NOT file a bug report at"
 				" http://bugs.gentoo.org unless you report exactly which"
-				" two packages install the same file(s). Once again,"
+				" two packages install the same file(s). See"
+				" http://wiki.gentoo.org/wiki/Knowledge_Base:Blockers"
+				" for tips on how to solve the problem. And once again,"
 				" please do NOT file a bug report unless you have"
 				" completely understood the above message.")
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     85f53097ed9fad43b713ac9c13f6beff43da760e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Aug  4 22:08:43 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Aug  4 22:08:43 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=85f53097

Include implicit IUSE vars in binhost Packages.

These values may be useful for using a binhost without having a local
copy of the profile (bug #470006).

---
 pym/portage/dbapi/bintree.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 7e30208..61ac6b5 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -317,7 +317,9 @@ class binarytree(object):
 				"ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
 				"ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD",
 				"CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
-				"GENTOO_MIRRORS", "INSTALL_MASK", "USE"])
+				"GENTOO_MIRRORS", "INSTALL_MASK", "IUSE_IMPLICIT", "USE",
+				"USE_EXPAND", "USE_EXPAND_HIDDEN", "USE_EXPAND_IMPLICIT",
+				"USE_EXPAND_UNPREFIXED"])
 			self._pkgindex_default_pkg_data = {
 				"BUILD_TIME"         : "",
 				"DEFINED_PHASES"     : "",
@@ -1259,6 +1261,16 @@ class binarytree(object):
 			else:
 				header.pop(k, None)
 
+		# These values may be useful for using a binhost without
+		# having a local copy of the profile (bug #470006).
+		for k in self.settings.get("USE_EXPAND_IMPLICIT", "").split():
+			k = "USE_EXPAND_VALUES_" + k
+			v = self.settings.get(k)
+			if v:
+				header[k] = v
+			else:
+				header.pop(k, None)
+
 	def _pkgindex_version_supported(self, pkgindex):
 		version = pkgindex.header.get("VERSION")
 		if version:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     6a43553134eefadaa2a6da312e4457cfcb128a5c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul  7 03:22:50 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul  7 03:22:50 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6a435531

MergeProcess: flush stderr for traceback

---
 pym/portage/dbapi/_MergeProcess.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 6c41041..0c20776 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -233,6 +233,8 @@ class MergeProcess(ForkProcess):
 				raise
 			except:
 				traceback.print_exc()
+				# os._exit() skips stderr flush!
+				sys.stderr.flush()
 			finally:
 				os._exit(rval)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e2a79ef5008ea491deefa2246c67798eee9c8bb4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun  8 05:47:48 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun  8 05:47:48 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e2a79ef5

porttree._dummy_list: fix infinite recursion

---
 pym/portage/dbapi/porttree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 1a6ffdd..b106de7 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -63,7 +63,7 @@ class _dummy_list(list):
 		# TODO: Trigger a DeprecationWarning here, after stable portage
 		# has dummy portdbapi_instances.
 		try:
-			self.remove(item)
+			list.remove(self, item)
 		except ValueError:
 			pass
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     55cd4ceae6ed39b52daaad89f111d87c25be9c9b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun  8 02:48:51 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun  8 03:00:03 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=55cd4cea

_iter_match_use: construct _pkg_str instances

Now _match_use does not have to construct them itself.

---
 pym/portage/dbapi/__init__.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index f954bad..a20a1e8 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -193,18 +193,27 @@ class dbapi(object):
 			except KeyError:
 				continue
 
+			try:
+				cpv.slot
+			except AttributeError:
+				try:
+					cpv = _pkg_str(cpv, metadata=metadata,
+						settings=self.settings)
+				except InvalidData:
+					continue
+
 			if not self._match_use(atom, cpv, metadata):
 				continue
 
 			yield cpv
 
-	def _match_use(self, atom, cpv, metadata):
+	def _match_use(self, atom, pkg, metadata):
 		eapi_attrs = _get_eapi_attrs(metadata["EAPI"])
 		if eapi_attrs.iuse_effective:
 			iuse_implicit_match = self.settings._iuse_effective_match
 		else:
 			iuse_implicit_match = self.settings._iuse_implicit_match
-		usealiases = self.settings._use_manager.getUseAliases(self._pkg_str(cpv, metadata["repository"]))
+		usealiases = self.settings._use_manager.getUseAliases(pkg)
 		iuse = Package._iuse(None, metadata["IUSE"].split(), iuse_implicit_match, usealiases, metadata["EAPI"])
 
 		for x in atom.unevaluated_atom.use.required:
@@ -245,12 +254,6 @@ class dbapi(object):
 
 		elif not self.settings.local_config:
 			# Check masked and forced flags for repoman.
-			try:
-				cpv.slot
-			except AttributeError:
-				pkg = _pkg_str(cpv, metadata=metadata, settings=self.settings)
-			else:
-				pkg = cpv
 			usemask = self.settings._getUseMask(pkg,
 				stable=self.settings._parent_stable)
 			if any(x in usemask for x in atom.use.enabled):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     b3e5f8a85a2e4f74ff3e26e6b889c3e493bc0172
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun  8 02:48:51 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun  8 02:48:51 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b3e5f8a8

_iter_match_use: construct _pkg_str instances

Now _match_use does not have to construct them itself.

---
 pym/portage/dbapi/__init__.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index f954bad..6938bf1 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -193,18 +193,23 @@ class dbapi(object):
 			except KeyError:
 				continue
 
+			try:
+				cpv.slot
+			except AttributeError:
+				cpv = _pkg_str(cpv, metadata=metadata, settings=self.settings)
+
 			if not self._match_use(atom, cpv, metadata):
 				continue
 
 			yield cpv
 
-	def _match_use(self, atom, cpv, metadata):
+	def _match_use(self, atom, pkg, metadata):
 		eapi_attrs = _get_eapi_attrs(metadata["EAPI"])
 		if eapi_attrs.iuse_effective:
 			iuse_implicit_match = self.settings._iuse_effective_match
 		else:
 			iuse_implicit_match = self.settings._iuse_implicit_match
-		usealiases = self.settings._use_manager.getUseAliases(self._pkg_str(cpv, metadata["repository"]))
+		usealiases = self.settings._use_manager.getUseAliases(pkg)
 		iuse = Package._iuse(None, metadata["IUSE"].split(), iuse_implicit_match, usealiases, metadata["EAPI"])
 
 		for x in atom.unevaluated_atom.use.required:
@@ -245,12 +250,6 @@ class dbapi(object):
 
 		elif not self.settings.local_config:
 			# Check masked and forced flags for repoman.
-			try:
-				cpv.slot
-			except AttributeError:
-				pkg = _pkg_str(cpv, metadata=metadata, settings=self.settings)
-			else:
-				pkg = cpv
 			usemask = self.settings._getUseMask(pkg,
 				stable=self.settings._parent_stable)
 			if any(x in usemask for x in atom.use.enabled):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Arfrever Frehtes Taifersar Arahesis]

commit:     c8530deda6b85dad1f7557d490f8a200e8bfa54e
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Fri Jun  7 20:41:28 2013 +0000
Commit:     Arfrever Frehtes Taifersar Arahesis <arfrever.fta <AT> gmail <DOT> com>
CommitDate: Fri Jun  7 20:41:28 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c8530ded

portage.dbapi.dbapi._match_use(): Use _emerge.Package.Package._iuse.get_real_flag().

It fixes support for USE aliases in:
  best_version
  has_version
  emerge --keep-going
  emerge --resume

---
 pym/portage/dbapi/__init__.py | 40 +++++++++++++++++-----------------------
 1 file changed, 17 insertions(+), 23 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index ab4306d..f954bad 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -204,10 +204,11 @@ class dbapi(object):
 			iuse_implicit_match = self.settings._iuse_effective_match
 		else:
 			iuse_implicit_match = self.settings._iuse_implicit_match
-		iuse = frozenset(x.lstrip('+-') for x in metadata["IUSE"].split())
+		usealiases = self.settings._use_manager.getUseAliases(self._pkg_str(cpv, metadata["repository"]))
+		iuse = Package._iuse(None, metadata["IUSE"].split(), iuse_implicit_match, usealiases, metadata["EAPI"])
 
 		for x in atom.unevaluated_atom.use.required:
-			if x not in iuse and not iuse_implicit_match(x):
+			if iuse.get_real_flag(x) is None:
 				return False
 
 		if atom.use is None:
@@ -220,27 +221,24 @@ class dbapi(object):
 			# data corruption). The enabled flags must be consistent
 			# with implicit IUSE, in order to avoid potential
 			# inconsistencies in USE dep matching (see bug #453400).
-			use = frozenset(x for x in metadata["USE"].split()
-				if x in iuse or iuse_implicit_match(x))
-			missing_enabled = frozenset(x for x in
-				atom.use.missing_enabled if not
-				(x in iuse or iuse_implicit_match(x)))
-			missing_disabled = frozenset(x for x in
-				atom.use.missing_disabled if not
-				(x in iuse or iuse_implicit_match(x)))
-
-			if atom.use.enabled:
-				if any(x in atom.use.enabled for x in missing_disabled):
+			use = frozenset(x for x in metadata["USE"].split() if iuse.get_real_flag(x) is not None)
+			missing_enabled = frozenset(x for x in atom.use.missing_enabled if iuse.get_real_flag(x) is None)
+			missing_disabled = frozenset(x for x in atom.use.missing_disabled if iuse.get_real_flag(x) is None)
+			enabled = frozenset((iuse.get_real_flag(x) or x) for x in atom.use.enabled)
+			disabled = frozenset((iuse.get_real_flag(x) or x) for x in atom.use.disabled)
+
+			if enabled:
+				if any(x in enabled for x in missing_disabled):
 					return False
-				need_enabled = atom.use.enabled.difference(use)
+				need_enabled = enabled.difference(use)
 				if need_enabled:
 					if any(x not in missing_enabled for x in need_enabled):
 						return False
 
-			if atom.use.disabled:
-				if any(x in atom.use.disabled for x in missing_enabled):
+			if disabled:
+				if any(x in disabled for x in missing_enabled):
 					return False
-				need_disabled = atom.use.disabled.intersection(use)
+				need_disabled = disabled.intersection(use)
 				if need_disabled:
 					if any(x not in missing_disabled for x in need_disabled):
 						return False
@@ -266,15 +264,11 @@ class dbapi(object):
 
 			# Check unsatisfied use-default deps
 			if atom.use.enabled:
-				missing_disabled = frozenset(x for x in
-					atom.use.missing_disabled if not
-					(x in iuse or iuse_implicit_match(x)))
+				missing_disabled = frozenset(x for x in atom.use.missing_disabled if iuse.get_real_flag(x) is None)
 				if any(x in atom.use.enabled for x in missing_disabled):
 					return False
 			if atom.use.disabled:
-				missing_enabled = frozenset(x for x in
-					atom.use.missing_enabled if not
-					(x in iuse or iuse_implicit_match(x)))
+				missing_enabled = frozenset(x for x in atom.use.missing_enabled if iuse.get_real_flag(x) is None)
 				if any(x in atom.use.disabled for x in missing_enabled):
 					return False
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     076c6637583328dc814828e1d10affb4e2666368
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  2 22:42:38 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  2 22:46:27 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=076c6637

treewalk: skip CHOST warning for binary packages

The CHOST of a binary package does not necessarily match the make.conf
setting, especially if ACCEPT_CHOSTS is configured to match other CHOST
values.

---
 pym/portage/dbapi/vartree.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c6eaabf..e1a4753 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3555,6 +3555,7 @@ class dblink(object):
 				level=logging.ERROR, noiselevel=-1)
 			return 1
 
+		is_binpkg = self.settings.get("EMERGE_FROM") == "binary"
 		slot = ''
 		for var_name in ('CHOST', 'SLOT'):
 			if var_name == 'CHOST' and self.cat == 'virtual':
@@ -3588,7 +3589,9 @@ class dblink(object):
 						return 1
 					write_atomic(os.path.join(inforoot, var_name), slot + '\n')
 
-			if val != self.settings.get(var_name, ''):
+			# This check only applies when built from source, since
+			# inforoot values are written just after src_install.
+			if not is_binpkg and val != self.settings.get(var_name, ''):
 				self._eqawarn('preinst',
 					[_("QA Notice: Expected %(var_name)s='%(expected_value)s', got '%(actual_value)s'\n") % \
 					{"var_name":var_name, "expected_value":self.settings.get(var_name, ''), "actual_value":val}])

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     75a524e9a53ef4dc941947cb99d34fdf1ccab6e4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri May 17 02:06:41 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri May 17 02:06:41 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=75a524e9

PORTAGE_BINHOST: sftp with FETCHCOMMAND_SFTP

Do not generate the sftp command for downloading the Packages file, and
simply use the FETCHCOMMAND_SFTP setting.

---
 pym/portage/dbapi/bintree.py |   18 ++++--------------
 1 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 7f2d017..14d05ad 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -911,20 +911,10 @@ class binarytree(object):
 
 					path = parsed_url.path.rstrip("/") + "/Packages"
 
-					if parsed_url.scheme == 'sftp':
-						# The sftp command complains about 'Illegal seek' if
-						# we try to make it write to /dev/stdout, so use a
-						# temp file instead.
-						fd, tmp_filename = tempfile.mkstemp()
-						os.close(fd)
-						if port is not None:
-							port_args = ['-P', "%s" % (port,)]
-						proc = subprocess.Popen(['sftp'] + port_args + \
-							[user_passwd + host + ":" + path, tmp_filename])
-						if proc.wait() != os.EX_OK:
-							raise EnvironmentError("sftp failed")
-						f = open(tmp_filename, 'rb')
-					elif parsed_url.scheme == 'ssh':
+					if parsed_url.scheme == 'ssh':
+						# Use a pipe so that we can terminate the download
+						# early if we detect that the TIMESTAMP header
+						# matches that of the cached Packages file.
 						if port is not None:
 							port_args = ['-p', "%s" % (port,)]
 						proc = subprocess.Popen(['ssh'] + port_args + \

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     b5969af9f575e4e4b669f44e76ad01f0dbc2dd27
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May 15 07:39:47 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May 15 07:39:47 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b5969af9

PORTAGE_BINHOST: https FETCHCOMMAND, bug #469888

Don't use urlopen for https, since it doesn't support
certificate/hostname verification (bug #469888).

---
 pym/portage/dbapi/bintree.py |   38 +++++++++++++++++++++++++++++---------
 1 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 44fc93b..7f2d017 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -43,6 +43,7 @@ import subprocess
 import sys
 import tempfile
 import textwrap
+import traceback
 import warnings
 from gzip import GzipFile
 from itertools import chain
@@ -884,13 +885,29 @@ class binarytree(object):
 				# protocols and requires the base url to have a trailing
 				# slash, so join manually...
 				url = base_url.rstrip("/") + "/Packages"
-				try:
-					f = _urlopen(url, if_modified_since=local_timestamp)
-					if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
-						remote_timestamp = f.headers.get('timestamp')
-				except IOError as err:
-					if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp)
-						raise UseCachedCopyOfRemoteIndex()
+				f = None
+
+				# Don't use urlopen for https, since it doesn't support
+				# certificate/hostname verification (bug #469888).
+				if parsed_url.scheme not in ('https',):
+					try:
+						f = _urlopen(url, if_modified_since=local_timestamp)
+						if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
+							remote_timestamp = f.headers.get('timestamp')
+					except IOError as err:
+						if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp)
+							raise UseCachedCopyOfRemoteIndex()
+
+						if parsed_url.scheme in ('ftp', 'http', 'https'):
+							# This protocol is supposedly supported by urlopen,
+							# so apparently there's a problem with the url
+							# or a bug in urlopen.
+							if self.settings.get("PORTAGE_DEBUG", "0") != "0":
+								traceback.print_exc()
+
+							raise
+
+				if f is None:
 
 					path = parsed_url.path.rstrip("/") + "/Packages"
 
@@ -905,7 +922,7 @@ class binarytree(object):
 						proc = subprocess.Popen(['sftp'] + port_args + \
 							[user_passwd + host + ":" + path, tmp_filename])
 						if proc.wait() != os.EX_OK:
-							raise
+							raise EnvironmentError("sftp failed")
 						f = open(tmp_filename, 'rb')
 					elif parsed_url.scheme == 'ssh':
 						if port is not None:
@@ -918,7 +935,10 @@ class binarytree(object):
 						setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
 						fcmd = self.settings.get(setting)
 						if not fcmd:
-							raise
+							fcmd = self.settings.get('FETCHCOMMAND')
+							if not fcmd:
+								raise EnvironmentError("FETCHCOMMAND is unset")
+
 						fd, tmp_filename = tempfile.mkstemp()
 						tmp_dirname, tmp_basename = os.path.split(tmp_filename)
 						os.close(fd)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5a812ae15a6c7895c71bfe78941398558c15346b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue May  7 03:49:12 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May  7 03:49:12 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5a812ae1

binarytree: evaluate RESTRICT conditionals

---
 pym/portage/dbapi/bintree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 948fec9..44fc93b 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -311,7 +311,7 @@ class binarytree(object):
 			self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
 			self._pkgindex_use_evaluated_keys = \
 				("DEPEND", "HDEPEND", "LICENSE", "RDEPEND",
-				"PDEPEND", "PROPERTIES", "PROVIDE")
+				"PDEPEND", "PROPERTIES", "PROVIDE", "RESTRICT")
 			self._pkgindex_header_keys = set([
 				"ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
 				"ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD",

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7043592d0c20bcdaa3007f07e060f194a23d02ab
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May  6 08:18:06 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May  6 08:18:06 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7043592d

binarytree: index RESTRICT, for ACCEPT_RESTRICT

---
 pym/portage/dbapi/bintree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index cdd10b3..948fec9 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -306,7 +306,7 @@ class binarytree(object):
 			self._pkgindex_aux_keys = \
 				["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI",
 				"HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES",
-				"PROVIDE", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
+				"PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
 				"BASE_URI"]
 			self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
 			self._pkgindex_use_evaluated_keys = \

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a65efb0b622bc745bbeb1a8fa4f05b1b936a761b
Author:     Aviv Keshet <akeshet <AT> chromium <DOT> org>
AuthorDate: Fri Apr 12 00:51:34 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Apr 12 17:22:00 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a65efb0b

refactor contents file writing to its own function

This CL refactors out the functionality of writing a contents dicionary
to file. That functionality is required by a change in the chromeos
chromite repository.

BUG=chromium:229234
TEST=None

Change-Id: I851724408b1d10827eee2ea8d67bdca9ad90c455

---
 pym/portage/dbapi/vartree.py |   23 +++++++++++++++++------
 1 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 9bc6e89..0d4a756 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1010,12 +1010,23 @@ class vardbapi(dbapi):
 				removed += 1
 
 		if removed:
-			self._bump_mtime(pkg.mycpv)
-			f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS"))
-			write_contents(new_contents, root, f)
-			f.close()
-			self._bump_mtime(pkg.mycpv)
-			pkg._clear_contents_cache()
+			self.writeContentsToContentsFile(pkg, new_contents)
+
+	def writeContentsToContentsFile(self, pkg, new_contents):
+		"""
+		@param pkg: package to write contents file for
+		@type pkg: dblink
+		@param new_contents: contents to write to CONTENTS file
+		@type new_contents: contents dictionary of the form
+					{u'/path/to/file' : (contents_attribute 1, ...), ...}
+		"""
+		root = self.settings['ROOT']
+		self._bump_mtime(pkg.mycpv)
+		f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS"))
+		write_contents(new_contents, root, f)
+		f.close()
+		self._bump_mtime(pkg.mycpv)
+		pkg._clear_contents_cache()
 
 	class _owners_cache(object):
 		"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f8c222902193ca5005bdc56f26525d7172866488
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr  1 17:25:34 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr  1 17:25:34 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f8c22290

binarytree: copy g+rw from PKGDIR to files

---
 pym/portage/dbapi/bintree.py |   19 +++++++++++++++++++
 1 files changed, 19 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 875e955..01dbad1 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -563,6 +563,20 @@ class binarytree(object):
 			if not os.path.isdir(path):
 				raise
 
+	def _file_permissions(self, path):
+		try:
+			pkgdir_st = os.stat(self.pkgdir)
+		except OSError:
+			pass
+		else:
+			pkgdir_gid = pkgdir_st.st_gid
+			pkgdir_grp_mode = 0o0060 & pkgdir_st.st_mode
+			try:
+				portage.util.apply_permissions(path, gid=pkgdir_gid,
+					mode=pkgdir_grp_mode, mask=0)
+			except PortageException:
+				pass
+
 	def _move_to_all(self, cpv):
 		"""If the file exists, move it.  Whether or not it exists, update state
 		for future getname() calls."""
@@ -1075,6 +1089,10 @@ class binarytree(object):
 				if not samefile:
 					self._ensure_dir(os.path.dirname(new_filename))
 					_movefile(filename, new_filename, mysettings=self.settings)
+				full_path = new_filename
+
+			self._file_permissions(full_path)
+
 			if self._all_directory and \
 				self.getname(cpv).split(os.path.sep)[-2] == "All":
 				self._create_symlink(cpv)
@@ -1147,6 +1165,7 @@ class binarytree(object):
 			f.close()
 			if f_close is not None:
 				f_close.close()
+			self._file_permissions(fname)
 			# some seconds might have elapsed since TIMESTAMP
 			os.utime(fname, (atime, mtime))
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8cfe2dc28e02a9cdd739a83db4ed871a31468114
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 16 05:39:38 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 16 05:39:38 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8cfe2dc2

Use lexists for symlink target checks.

---
 pym/portage/dbapi/vartree.py |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 95b67dd..d011c33 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4484,7 +4484,10 @@ class dblink(object):
 					pass
 
 				if mymtime != None:
-					if not os.path.exists(myrealto) and not os.path.exists(join(srcroot, myabsto)):
+					# Use lexists, since if the target happens to be a broken
+					# symlink then that should trigger an independent warning.
+					if not (os.path.lexists(myrealto) or
+						os.path.lexists(join(srcroot, myabsto))):
 						self._eqawarn('preinst',
 							[_("QA Notice: Symbolic link /%s points to /%s which does not exist.")
 							% (relative_path, myabsto)])

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     ca7467006b112f65a57c4db7816729841a2de694
Author:     Tom Wijsman <tomwij <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 16 05:28:03 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 16 05:28:03 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ca746700

Warn when merging broken symlinks for bug #446.

---
 pym/portage/dbapi/vartree.py |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index f918c2c..95b67dd 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4484,6 +4484,11 @@ class dblink(object):
 					pass
 
 				if mymtime != None:
+					if not os.path.exists(myrealto) and not os.path.exists(join(srcroot, myabsto)):
+						self._eqawarn('preinst',
+							[_("QA Notice: Symbolic link /%s points to /%s which does not exist.")
+							% (relative_path, myabsto)])
+
 					showMessage(">>> %s -> %s\n" % (mydest, myto))
 					if sys.hexversion >= 0x3030000:
 						outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n")

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e6971d5000178a2ab84e0a3217f0b5b031d491ef
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 16 01:46:07 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 16 01:46:07 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e6971d50

mergeme: fix myrealto join with lstrip(os.sep)

---
 pym/portage/dbapi/vartree.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index f918c2c..971ab7a 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4434,7 +4434,8 @@ class dblink(object):
 						myto = myto[len(self.settings["D"])-1:]
 				# myrealto contains the path of the real file to which this symlink points.
 				# we can simply test for existence of this file to see if the target has been merged yet
-				myrealto = normalize_path(os.path.join(destroot, myabsto))
+				myrealto = normalize_path(os.path.join(destroot,
+					myabsto.lstrip(os.sep)))
 				if mydmode!=None:
 					#destination exists
 					if stat.S_ISDIR(mydmode):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     cefaa921a68c017910a804c6f889f6f19fc6e8b9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 28 21:18:00 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jan 28 21:19:10 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cefaa921

removeFromContents: avoid KeyError, bug #454400

It's possible for two different paths to refer to the same
contents_key, due to directory symlinks. This could happen if a
directory was changed to a symlink after a package was merged, either
by the user or by a pkg_* phase from some random ebuild.

---
 pym/portage/dbapi/vartree.py |    6 +++++-
 1 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index bbcf61c..a2fbf86 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1002,7 +1002,11 @@ class vardbapi(dbapi):
 				relative_filename = filename[root_len:]
 			contents_key = pkg._match_contents(relative_filename)
 			if contents_key:
-				del new_contents[contents_key]
+				# It's possible for two different paths to refer to the same
+				# contents_key, due to directory symlinks. Therefore, pass a
+				# default value to pop, in order to avoid a KeyError which
+				# could otherwise be triggered (see bug #454400).
+				new_contents.pop(contents_key, None)
 				removed += 1
 
 		if removed:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     25cd343a65fb4ff460e98aeb704f85b284dfc3a9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 24 18:29:41 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 24 18:29:41 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=25cd343a

_new_backup_path: fix TypeError, bug #453892

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 20a114b..bbcf61c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4222,7 +4222,7 @@ class dblink(object):
 		x = -1
 		while True:
 			x += 1
-			backup_p = p + '.backup.' + str(x).rjust(4, '0')
+			backup_p = '%s.backup.%04d' % (p, x)
 			try:
 				os.lstat(backup_p)
 			except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     36f922331bb90f828d21fa9a5a8b545cbb5bdf44
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 17 14:19:56 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 17 14:19:56 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=36f92233

Use 'with file' more.

This helps to minimize ResourceWarning triggered by ^C with python3.

---
 pym/portage/dbapi/vartree.py |  107 ++++++++++++++++++------------------------
 1 files changed, 46 insertions(+), 61 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index be93e1f..09130fb 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -577,18 +577,16 @@ class vardbapi(dbapi):
 			# appears to be solved in Python >=3.2 at least).
 			open_kwargs["buffering"] = 0
 		try:
-			f = open(_unicode_encode(self._aux_cache_filename,
+			with open(_unicode_encode(self._aux_cache_filename,
 				encoding=_encodings['fs'], errors='strict'),
-				mode='rb', **open_kwargs)
-			mypickle = pickle.Unpickler(f)
-			try:
-				mypickle.find_global = None
-			except AttributeError:
-				# TODO: If py3k, override Unpickler.find_class().
-				pass
-			aux_cache = mypickle.load()
-			f.close()
-			del f
+				mode='rb', **open_kwargs) as f:
+				mypickle = pickle.Unpickler(f)
+				try:
+					mypickle.find_global = None
+				except AttributeError:
+					# TODO: If py3k, override Unpickler.find_class().
+					pass
+				aux_cache = mypickle.load()
 		except (SystemExit, KeyboardInterrupt):
 			raise
 		except Exception as e:
@@ -741,15 +739,12 @@ class vardbapi(dbapi):
 				results[x] = st[stat.ST_MTIME]
 				continue
 			try:
-				myf = io.open(
+				with io.open(
 					_unicode_encode(os.path.join(mydir, x),
 					encoding=_encodings['fs'], errors='strict'),
 					mode='r', encoding=_encodings['repo.content'],
-					errors='replace')
-				try:
-					myd = myf.read()
-				finally:
-					myf.close()
+					errors='replace') as f:
+					myd = f.read()
 			except IOError:
 				if x not in self._aux_cache_keys and \
 					self._aux_cache_keys_re.match(x) is None:
@@ -895,11 +890,17 @@ class vardbapi(dbapi):
 		del myroot
 		counter = -1
 		try:
-			cfile = io.open(
+			with io.open(
 				_unicode_encode(self._counter_path,
 				encoding=_encodings['fs'], errors='strict'),
 				mode='r', encoding=_encodings['repo.content'],
-				errors='replace')
+				errors='replace') as f:
+				try:
+					counter = long(f.readline().strip())
+				except (OverflowError, ValueError) as e:
+					writemsg(_("!!! COUNTER file is corrupt: '%s'\n") %
+						self._counter_path, noiselevel=-1)
+					writemsg("!!! %s\n" % (e,), noiselevel=-1)
 		except EnvironmentError as e:
 			# Silently allow ENOENT since files under
 			# /var/cache/ are allowed to disappear.
@@ -908,17 +909,6 @@ class vardbapi(dbapi):
 					self._counter_path, noiselevel=-1)
 				writemsg("!!! %s\n" % str(e), noiselevel=-1)
 			del e
-		else:
-			try:
-				try:
-					counter = long(cfile.readline().strip())
-				finally:
-					cfile.close()
-			except (OverflowError, ValueError) as e:
-				writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % \
-					self._counter_path, noiselevel=-1)
-				writemsg("!!! %s\n" % str(e), noiselevel=-1)
-				del e
 
 		if self._cached_counter == counter:
 			max_counter = counter
@@ -1591,18 +1581,18 @@ class dblink(object):
 			return self.contentscache
 		pkgfiles = {}
 		try:
-			myc = io.open(_unicode_encode(contents_file,
+			with io.open(_unicode_encode(contents_file,
 				encoding=_encodings['fs'], errors='strict'),
 				mode='r', encoding=_encodings['repo.content'],
-				errors='replace')
+				errors='replace') as f:
+				mylines = f.readlines()
 		except EnvironmentError as e:
 			if e.errno != errno.ENOENT:
 				raise
 			del e
 			self.contentscache = pkgfiles
 			return pkgfiles
-		mylines = myc.readlines()
-		myc.close()
+
 		null_byte = "\0"
 		normalize_needed = self._normalize_needed
 		contents_re = self._contents_re
@@ -3532,22 +3522,18 @@ class dblink(object):
 					pass
 				continue
 
-			f = None
 			try:
-				f = io.open(_unicode_encode(
+				with io.open(_unicode_encode(
 					os.path.join(inforoot, var_name),
 					encoding=_encodings['fs'], errors='strict'),
 					mode='r', encoding=_encodings['repo.content'],
-					errors='replace')
-				val = f.readline().strip()
+					errors='replace') as f:
+					val = f.readline().strip()
 			except EnvironmentError as e:
 				if e.errno != errno.ENOENT:
 					raise
 				del e
 				val = ''
-			finally:
-				if f is not None:
-					f.close()
 
 			if var_name == 'SLOT':
 				slot = val
@@ -3973,12 +3959,11 @@ class dblink(object):
 		# write local package counter for recording
 		if counter is None:
 			counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
-		f = io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
+		with io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
 			encoding=_encodings['fs'], errors='strict'),
 			mode='w', encoding=_encodings['repo.content'],
-			errors='backslashreplace')
-		f.write(_unicode_decode(str(counter)))
-		f.close()
+			errors='backslashreplace') as f:
+			f.write(_unicode_decode(str(counter)))
 
 		self.updateprotect()
 
@@ -4823,11 +4808,12 @@ class dblink(object):
 		"returns contents of a file with whitespace converted to spaces"
 		if not os.path.exists(self.dbdir+"/"+name):
 			return ""
-		mydata = io.open(
+		with io.open(
 			_unicode_encode(os.path.join(self.dbdir, name),
 			encoding=_encodings['fs'], errors='strict'),
 			mode='r', encoding=_encodings['repo.content'], errors='replace'
-			).read().split()
+			) as f:
+			mydata = f.read().split()
 		return " ".join(mydata)
 
 	def copyfile(self,fname):
@@ -4836,10 +4822,11 @@ class dblink(object):
 	def getfile(self,fname):
 		if not os.path.exists(self.dbdir+"/"+fname):
 			return ""
-		return io.open(_unicode_encode(os.path.join(self.dbdir, fname),
+		with io.open(_unicode_encode(os.path.join(self.dbdir, fname),
 			encoding=_encodings['fs'], errors='strict'), 
 			mode='r', encoding=_encodings['repo.content'], errors='replace'
-			).read()
+			) as f:
+			return f.read()
 
 	def setfile(self,fname,data):
 		kwargs = {}
@@ -4853,11 +4840,12 @@ class dblink(object):
 	def getelements(self,ename):
 		if not os.path.exists(self.dbdir+"/"+ename):
 			return []
-		mylines = io.open(_unicode_encode(
+		with io.open(_unicode_encode(
 			os.path.join(self.dbdir, ename),
 			encoding=_encodings['fs'], errors='strict'),
 			mode='r', encoding=_encodings['repo.content'], errors='replace'
-			).readlines()
+			) as f:
+			mylines = f.readlines()
 		myreturn = []
 		for x in mylines:
 			for y in x[:-1].split():
@@ -4865,14 +4853,13 @@ class dblink(object):
 		return myreturn
 
 	def setelements(self,mylist,ename):
-		myelement = io.open(_unicode_encode(
+		with io.open(_unicode_encode(
 			os.path.join(self.dbdir, ename),
 			encoding=_encodings['fs'], errors='strict'),
 			mode='w', encoding=_encodings['repo.content'],
-			errors='backslashreplace')
-		for x in mylist:
-			myelement.write(_unicode_decode(x+"\n"))
-		myelement.close()
+			errors='backslashreplace') as f:
+			for x in mylist:
+				f.write(_unicode_decode(x + "\n"))
 
 	def isregular(self):
 		"Is this a regular package (does it have a CATEGORY file?  A dblink can be virtual *and* regular)"
@@ -5140,13 +5127,11 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None):
 				tar.addfile(tarinfo, f)
 				f.close()
 			else:
-				f = open(_unicode_encode(path,
+				with open(_unicode_encode(path,
 					encoding=encoding,
-					errors='strict'), 'rb')
-				try:
+					errors='strict'), 'rb') as f:
 					tar.addfile(tarinfo, f)
-				finally:
-					f.close()
+
 		else:
 			tar.addfile(tarinfo)
 		if onProgress:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     c811cbf837e9a81f656d7883a231e23c209c73de
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 10 14:06:34 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 10 14:06:34 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c811cbf8

Use EventLoop, no SchedulerInterface if possible.

---
 pym/portage/dbapi/vartree.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index ba149b7..be93e1f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4746,8 +4746,8 @@ class dblink(object):
 
 			proc = SyncfsProcess(paths=paths,
 				scheduler=(self._scheduler or
-				SchedulerInterface(portage._internal_caller and
-					global_event_loop() or EventLoop(main=False))))
+					portage._internal_caller and global_event_loop() or
+					EventLoop(main=False)))
 			proc.start()
 			returncode = proc.wait()
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3b3405694dd3cdaaf7587db5fdd657bc93b73f41
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  6 20:48:04 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jan  6 20:48:04 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3b340569

portdbapi.cp_all(): add reverse sort parameter

---
 pym/portage/dbapi/porttree.py |    5 +++--
 1 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index d8cd261..cf4b142 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -646,13 +646,14 @@ class portdbapi(dbapi):
 		else:
 			return 0
 
-	def cp_all(self, categories=None, trees=None):
+	def cp_all(self, categories=None, trees=None, reverse=False):
 		"""
 		This returns a list of all keys in our tree or trees
 		@param categories: optional list of categories to search or 
 			defaults to self.settings.categories
 		@param trees: optional list of trees to search the categories in or
 			defaults to self.porttrees
+		@param reverse: reverse sort order (default is False)
 		@rtype list of [cat/pkg,...]
 		"""
 		d = {}
@@ -671,7 +672,7 @@ class portdbapi(dbapi):
 						continue
 					d[atom.cp] = None
 		l = list(d)
-		l.sort()
+		l.sort(reverse=reverse)
 		return l
 
 	def cp_list(self, mycp, use_cache=1, mytree=None):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     400a2a983b374ec54e7b574789d1f94752581b81
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan  3 22:42:41 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan  3 22:42:41 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=400a2a98

portdbapi: use global_event_loop when safe

---
 pym/portage/dbapi/porttree.py |   13 ++++++++++++-
 1 files changed, 12 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 35f4e2d..ebb1be8 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -34,6 +34,7 @@ from portage import _encodings
 from portage import _unicode_encode
 from portage import OrderedDict
 from portage.util._eventloop.EventLoop import EventLoop
+from portage.util._eventloop.global_event_loop import global_event_loop
 from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
 
 import os as _os
@@ -96,7 +97,6 @@ class portdbapi(dbapi):
 		# this purpose because doebuild makes many changes to the config
 		# instance that is passed in.
 		self.doebuild_settings = config(clone=self.settings)
-		self._event_loop = EventLoop(main=False)
 		self.depcachedir = os.path.realpath(self.settings.depcachedir)
 		
 		if os.environ.get("SANDBOX_ON") == "1":
@@ -195,6 +195,17 @@ class portdbapi(dbapi):
 		self._aux_cache = {}
 		self._broken_ebuilds = set()
 
+	@property
+	def _event_loop(self):
+		if portage._internal_warnings:
+			# For internal portage usage, the global_event_loop is safe.
+			return global_event_loop()
+		else:
+			# For external API consumers, use a local EventLoop, since
+			# we don't want to assume that it's safe to override the
+			# global SIGCHLD handler.
+			return EventLoop(main=False)
+
 	def _create_pregen_cache(self, tree):
 		conf = self.repositories.get_repo_for_location(tree)
 		cache = conf.get_pregenerated_cache(

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7ebb2f54877edb28621c33e380f8777b1b1dc201
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Dec 30 09:33:10 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec 30 09:33:10 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7ebb2f54

Use ctypes in subprocess for bug #448858.

Isolate ctypes usage in a subprocess, in order to avoid potential
problems with stale cached libraries as described in bug #448858,
comment #14 (also see http://bugs.python.org/issue14597).

---
 pym/portage/dbapi/_SyncfsProcess.py |   53 +++++++++++++++++++++++++++++++++++
 pym/portage/dbapi/vartree.py        |   47 +++++++++---------------------
 2 files changed, 67 insertions(+), 33 deletions(-)

diff --git a/pym/portage/dbapi/_SyncfsProcess.py b/pym/portage/dbapi/_SyncfsProcess.py
new file mode 100644
index 0000000..7518214
--- /dev/null
+++ b/pym/portage/dbapi/_SyncfsProcess.py
@@ -0,0 +1,53 @@
+# Copyright 2012 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage import os
+from portage.util._ctypes import find_library, LoadLibrary
+from portage.util._async.ForkProcess import ForkProcess
+
+class SyncfsProcess(ForkProcess):
+	"""
+	Isolate ctypes usage in a subprocess, in order to avoid
+	potential problems with stale cached libraries as
+	described in bug #448858, comment #14 (also see
+	http://bugs.python.org/issue14597).
+	"""
+
+	__slots__ = ('paths',)
+
+	@staticmethod
+	def _get_syncfs():
+
+		filename = find_library("c")
+		if filename is not None:
+			library = LoadLibrary(filename)
+			if library is not None:
+				try:
+					return library.syncfs
+				except AttributeError:
+					pass
+
+		return None
+
+	def _run(self):
+
+		syncfs_failed = False
+		syncfs = self._get_syncfs()
+
+		if syncfs is not None:
+			for path in self.paths:
+				try:
+					fd = os.open(path, os.O_RDONLY)
+				except OSError:
+					pass
+				else:
+					try:
+						if syncfs(fd) != 0:
+							# Happens with PyPy (bug #446610)
+							syncfs_failed = True
+					finally:
+						os.close(fd)
+
+		if syncfs is None or syncfs_failed:
+			return 1
+		return os.EX_OK

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 840e796..7a930e5 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -11,6 +11,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.data:portage_gid,portage_uid,secpass',
 	'portage.dbapi.dep_expand:dep_expand',
 	'portage.dbapi._MergeProcess:MergeProcess',
+	'portage.dbapi._SyncfsProcess:SyncfsProcess',
 	'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \
 	 	'use_reduce,_get_slot_re,_slot_separator,_repo_separator',
 	'portage.eapi:_get_eapi_attrs',
@@ -29,7 +30,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util.env_update:env_update',
 	'portage.util.listdir:dircache,listdir',
 	'portage.util.movefile:movefile',
-	'portage.util._ctypes:find_library,LoadLibrary',
 	'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry',
 	'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap',
 	'portage.util._async.SchedulerInterface:SchedulerInterface',
@@ -4731,28 +4731,22 @@ class dblink(object):
 			"merge-sync" not in self.settings.features:
 			return
 
-		syncfs_failed = False
-		syncfs = _get_syncfs()
+		returncode = None
+		if platform.system() == "Linux":
 
-		if syncfs is not None:
+			paths = []
 			for path in self._device_path_map.values():
-				if path is False:
-					continue
-				try:
-					fd = os.open(path, os.O_RDONLY)
-				except OSError:
-					pass
-				else:
-					try:
-						if syncfs(fd) != 0:
-							# Happens with PyPy (bug #446610)
-							syncfs_failed = True
-					except OSError:
-						pass
-					finally:
-						os.close(fd)
+				if path is not False:
+					paths.append(path)
+			paths = tuple(paths)
+
+			proc = SyncfsProcess(paths=paths,
+				scheduler=(self._scheduler or
+				SchedulerInterface(EventLoop(main=False))))
+			proc.start()
+			returncode = proc.wait()
 
-		if syncfs is None or syncfs_failed:
+		if returncode is None or returncode != os.EX_OK:
 			try:
 				proc = subprocess.Popen(["sync"])
 			except EnvironmentError:
@@ -4938,19 +4932,6 @@ class dblink(object):
 		finally:
 			self.unlockdb()
 
-def _get_syncfs():
-	if platform.system() == "Linux":
-		filename = find_library("c")
-		if filename is not None:
-			library = LoadLibrary(filename)
-			if library is not None:
-				try:
-					return library.syncfs
-				except AttributeError:
-					pass
-
-	return None
-
 def merge(mycat, mypkg, pkgloc, infloc,
 	myroot=None, settings=None, myebuild=None,
 	mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     07f38da52b836127f0252982277945a985150c45
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 28 22:44:34 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Dec 28 22:44:34 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=07f38da5

_MergeProcess: tweak merge-sync library lookup

The library handle is no longer cached, since commit
9e37cca4f54260bd8c45a3041fcee00938c71649, so skip the LoadLibrary
call and just call find_library instead.

---
 pym/portage/dbapi/_MergeProcess.py |    7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index cd9b122..e2534cf 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import io
+import platform
 import signal
 import sys
 import traceback
@@ -10,7 +11,7 @@ import errno
 import fcntl
 import portage
 from portage import os, _unicode_decode
-from portage.dbapi.vartree import _get_syncfs
+from portage.util._ctypes import find_library
 import portage.elog.messages
 from portage.util._async.ForkProcess import ForkProcess
 
@@ -44,7 +45,9 @@ class MergeProcess(ForkProcess):
 		# This caches the libc library lookup in the current
 		# process, so that it's only done once rather than
 		# for each child process.
-		_get_syncfs()
+		if platform.system() == "Linux" and \
+			"merge-sync" in settings.features:
+			find_library("c")
 
 		# Inherit stdin by default, so that the pdb SIGUSR1
 		# handler is usable for the subprocess.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     9f65889534279731f4b00af243c1edc885eece09
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 27 03:10:16 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Dec 27 03:10:16 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9f658895

merge-sync: handle PyPy syncfs fail, bug #446610

---
 pym/portage/dbapi/vartree.py |   23 ++++++++++++++---------
 1 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 25128bf..840e796 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4731,15 +4731,10 @@ class dblink(object):
 			"merge-sync" not in self.settings.features:
 			return
 
+		syncfs_failed = False
 		syncfs = _get_syncfs()
-		if syncfs is None:
-			try:
-				proc = subprocess.Popen(["sync"])
-			except EnvironmentError:
-				pass
-			else:
-				proc.wait()
-		else:
+
+		if syncfs is not None:
 			for path in self._device_path_map.values():
 				if path is False:
 					continue
@@ -4749,12 +4744,22 @@ class dblink(object):
 					pass
 				else:
 					try:
-						syncfs(fd)
+						if syncfs(fd) != 0:
+							# Happens with PyPy (bug #446610)
+							syncfs_failed = True
 					except OSError:
 						pass
 					finally:
 						os.close(fd)
 
+		if syncfs is None or syncfs_failed:
+			try:
+				proc = subprocess.Popen(["sync"])
+			except EnvironmentError:
+				pass
+			else:
+				proc.wait()
+
 	def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
 		mydbapi=None, prev_mtimes=None, counter=None):
 		"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     074becc02bf28e3783d672967cae08aa35116809
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 24 21:12:03 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Dec 24 21:12:03 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=074becc0

vardbapi: tweak deprecated constructor param

---
 pym/portage/dbapi/vartree.py |    9 +++++----
 1 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 344a4f9..25128bf 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -112,7 +112,8 @@ class vardbapi(dbapi):
 	_aux_cache_keys_re = re.compile(r'^NEEDED\..*$')
 	_aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$')
 
-	def __init__(self, _unused_param=None, categories=None, settings=None, vartree=None):
+	def __init__(self, _unused_param=DeprecationWarning,
+		categories=None, settings=None, vartree=None):
 		"""
 		The categories parameter is unused since the dbapi class
 		now has a categories property that is generated from the
@@ -142,11 +143,11 @@ class vardbapi(dbapi):
 			settings = portage.settings
 		self.settings = settings
 
-		if _unused_param is not None and _unused_param != settings['ROOT']:
+		if _unused_param is not DeprecationWarning:
 			warnings.warn("The first parameter of the "
 				"portage.dbapi.vartree.vardbapi"
-				" constructor is now unused. Use "
-				"settings['ROOT'] instead.",
+				" constructor is now unused. Instead "
+				"settings['ROOT'] is used.",
 				DeprecationWarning, stacklevel=2)
 
 		self._eroot = settings['EROOT']

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8d27739e6b395ad36f349752a02f55b58e731204
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 24 21:06:43 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Dec 24 21:06:43 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8d27739e

binarytree: tweak deprecated constructor param

---
 pym/portage/dbapi/bintree.py |   10 +++++-----
 1 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 2203aaf..890e8e1 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -258,7 +258,7 @@ def _pkgindex_cpv_map_latest_build(pkgindex):
 
 class binarytree(object):
 	"this tree scans for a list of all packages available in PKGDIR"
-	def __init__(self, _unused=None, pkgdir=None,
+	def __init__(self, _unused=DeprecationWarning, pkgdir=None,
 		virtual=DeprecationWarning, settings=None):
 
 		if pkgdir is None:
@@ -267,11 +267,11 @@ class binarytree(object):
 		if settings is None:
 			raise TypeError("settings parameter is required")
 
-		if _unused is not None and _unused != settings['ROOT']:
-			warnings.warn("The root parameter of the "
+		if _unused is not DeprecationWarning:
+			warnings.warn("The first parameter of the "
 				"portage.dbapi.bintree.binarytree"
-				" constructor is now unused. Use "
-				"settings['ROOT'] instead.",
+				" constructor is now unused. Instead "
+				"settings['ROOT'] is used.",
 				DeprecationWarning, stacklevel=2)
 
 		if virtual is not DeprecationWarning:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a5fdf591cffaa787ac8c579f853e5ea8c20b4543
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 24 20:20:58 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Dec 24 20:20:58 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a5fdf591

porttree.py: tweak deprecated constructor params

Trigger a DeprecationWarning even when None is passed in for one of the
deprecated parameters, ensuring that any potentially unsafe usage
triggers the warning.

---
 pym/portage/dbapi/porttree.py |    9 +++++----
 1 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 9f0140c..35f4e2d 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -65,7 +65,7 @@ class portdbapi(dbapi):
 			return None
 		return main_repo.eclass_db
 
-	def __init__(self, _unused_param=None, mysettings=None):
+	def __init__(self, _unused_param=DeprecationWarning, mysettings=None):
 		"""
 		@param _unused_param: deprecated, use mysettings['PORTDIR'] instead
 		@type _unused_param: None
@@ -81,7 +81,7 @@ class portdbapi(dbapi):
 			from portage import settings
 			self.settings = config(clone=settings)
 
-		if _unused_param is not None:
+		if _unused_param is not DeprecationWarning:
 			warnings.warn("The first parameter of the " + \
 				"portage.dbapi.porttree.portdbapi" + \
 				" constructor is unused since portage-2.1.8. " + \
@@ -986,7 +986,8 @@ def close_portdbapi_caches():
 portage.process.atexit_register(portage.portageexit)
 
 class portagetree(object):
-	def __init__(self, root=None, virtual=DeprecationWarning, settings=None):
+	def __init__(self, root=DeprecationWarning, virtual=DeprecationWarning,
+		settings=None):
 		"""
 		Constructor for a PortageTree
 		
@@ -1002,7 +1003,7 @@ class portagetree(object):
 			settings = portage.settings
 		self.settings = settings
 
-		if root is not None and root != settings['ROOT']:
+		if root is not DeprecationWarning:
 			warnings.warn("The root parameter of the " + \
 				"portage.dbapi.porttree.portagetree" + \
 				" constructor is now unused. Use " + \

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     57a37fa6e1e30823b0798f34c7291eec081c1ec2
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 10 08:55:18 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Dec 10 08:55:18 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=57a37fa6

dblink._elog_process: obey phasefilter arg

This would make a difference when collecting messages for prerm/postrm
when uninstalling the previous instance of a package that's been
updated.

---
 pym/portage/dbapi/vartree.py |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 735cf3b..344a4f9 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3436,7 +3436,10 @@ class dblink(object):
 		else:
 			logdir = os.path.join(self.settings["T"], "logging")
 			ebuild_logentries = collect_ebuild_messages(logdir)
-			py_logentries = collect_messages(key=cpv).get(cpv, {})
+			# phasefilter is irrelevant for the above collect_ebuild_messages
+			# call, since this package instance has a private logdir. However,
+			# it may be relevant for the following collect_messages call.
+			py_logentries = collect_messages(key=cpv, phasefilter=phasefilter).get(cpv, {})
 			logentries = _merge_logentries(py_logentries, ebuild_logentries)
 			funcnames = {
 				"INFO": "einfo",

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     db157a4096dac7a7028357c7f1b5ae5c05eb82ae
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 10 08:38:51 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Dec 10 08:38:51 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=db157a40

MergeProcess: avoid duplicate elog, bug #446136

---
 pym/portage/dbapi/_MergeProcess.py |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 9be68d3..cd9b122 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -152,6 +152,9 @@ class MergeProcess(ForkProcess):
 				self._elog_reader_fd = elog_reader_fd
 				self._buf = ""
 				self._elog_keys = set()
+				# Discard messages which will be collected by the subprocess,
+				# in order to avoid duplicates (bug #446136).
+				portage.elog.messages.collect_messages(key=mylink.mycpv)
 
 				# invalidate relevant vardbapi caches
 				if self.vartree.dbapi._categories is not None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     c3bc709ae99074ed65584ad68ff403d2e75ae04f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 26 03:26:46 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Nov 26 03:26:46 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c3bc709a

similar_name_search: fix lower/upper case bugs

---
 pym/portage/dbapi/_similar_name_search.py |    8 ++++----
 1 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/_similar_name_search.py b/pym/portage/dbapi/_similar_name_search.py
index d569fbf..b6e4a1f 100644
--- a/pym/portage/dbapi/_similar_name_search.py
+++ b/pym/portage/dbapi/_similar_name_search.py
@@ -7,8 +7,8 @@ from portage.versions import catsplit
 
 def similar_name_search(dbs, atom):
 
-	cp = atom.cp.lower()
-	cat, pkg = catsplit(cp)
+	cp_lower = atom.cp.lower()
+	cat, pkg = catsplit(cp_lower)
 	if cat == "null":
 		cat = None
 
@@ -17,7 +17,7 @@ def similar_name_search(dbs, atom):
 		all_cp.update(db.cp_all())
 
 	# discard dir containing no ebuilds
-	all_cp.discard(cp)
+	all_cp.discard(atom.cp)
 
 	orig_cp_map = {}
 	for cp_orig in all_cp:
@@ -25,7 +25,7 @@ def similar_name_search(dbs, atom):
 	all_cp = set(orig_cp_map)
 
 	if cat:
-		matches = difflib.get_close_matches(cp, all_cp)
+		matches = difflib.get_close_matches(cp_lower, all_cp)
 	else:
 		pkg_to_cp = {}
 		for other_cp in list(all_cp):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     c974a023882485b8eeae35bac35c1f00d1a0725b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 15 00:02:38 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Nov 15 00:02:38 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c974a023

vardbapi.move_ent: rm wrong fixdbentries call

The parent argument was wrong, as reported in bug #367215 comment #22,
and this call was uneeded anyway because dbapi.update_ents handles
it with update_dbentries.

---
 pym/portage/dbapi/vartree.py |    3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 27e204e..0a74f47 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -22,7 +22,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
 		'_merge_unicode_error', '_spawn_phase',
 	'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
 	'portage.package.ebuild._ipc.QueryCommand:QueryCommand',
-	'portage.update:fixdbentries',
 	'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \
 		'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \
 		'grabdict,normalize_path,new_protect_filename',
@@ -364,7 +363,7 @@ class vardbapi(dbapi):
 					del e
 			write_atomic(os.path.join(newpath, "PF"), new_pf+"\n")
 			write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n")
-			fixdbentries([mylist], newpath, parent=mycpv)
+
 		return moves
 
 	def cp_list(self, mycp, use_cache=1):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5acf1392fdcac2470e504b211d40ee7606d2090a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 30 23:11:35 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct 30 23:11:35 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5acf1392

dblink.mergeme(): use stack instead of recursion

Maybe this helps reduce memory consumption when merging packages with
lots of subdirectories like gentoo-sources (bug #320859).

---
 pym/portage/dbapi/vartree.py |   22 +++++++++++-----------
 1 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index aae23e6..8d908fc 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4356,18 +4356,18 @@ class dblink(object):
 		# this is supposed to merge a list of files.  There will be 2 forms of argument passing.
 		if isinstance(stufftomerge, basestring):
 			#A directory is specified.  Figure out protection paths, listdir() it and process it.
-			mergelist = os.listdir(join(srcroot, stufftomerge))
-			offset = stufftomerge
+			mergelist = [join(stufftomerge, child) for child in \
+				os.listdir(join(srcroot, stufftomerge))]
 		else:
-			mergelist = stufftomerge
-			offset = ""
+			mergelist = stufftomerge[:]
 
-		for i, x in enumerate(mergelist):
+		while mergelist:
 
-			mysrc = join(srcroot, offset, x)
-			mydest = join(destroot, offset, x)
+			relative_path = mergelist.pop()
+			mysrc = join(srcroot, relative_path)
+			mydest = join(destroot, relative_path)
 			# myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/")
-			myrealdest = join(sep, offset, x)
+			myrealdest = join(sep, relative_path)
 			# stat file once, test using S_* macros many times (faster that way)
 			mystat = os.lstat(mysrc)
 			mymode = mystat[stat.ST_MODE]
@@ -4575,9 +4575,9 @@ class dblink(object):
 
 				outfile.write("dir "+myrealdest+"\n")
 				# recurse and merge this directory
-				if self.mergeme(srcroot, destroot, outfile, secondhand,
-					join(offset, x), cfgfiledict, thismtime):
-					return 1
+				mergelist.extend(join(relative_path, child) for child in
+					os.listdir(join(srcroot, relative_path)))
+
 			elif stat.S_ISREG(mymode):
 				# we are merging a regular file
 				mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     690312722e111cf813b591dd4dccb4400f5f7f79
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 25 19:48:56 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 25 19:48:56 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=69031272

merge-sync: handle unmerge corner cases

In dblink_merged_path we need to distinguish between existing and
non-existing, since we need an existing path of syncfs.

---
 pym/portage/dbapi/vartree.py |   17 +++++++++++++----
 1 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 5bcd077..aae23e6 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2156,7 +2156,7 @@ class dblink(object):
 					# here so that _unmerge_dirs can see that we've
 					# removed a file from this device, and will record
 					# the parent directory for a syncfs call.
-					self._merged_path(file_name, lstatobj)
+					self._merged_path(file_name, lstatobj, exists=False)
 
 				finally:
 					if bsd_chflags and pflags != 0:
@@ -4706,10 +4706,17 @@ class dblink(object):
 					outfile.write("dev %s\n" % myrealdest)
 				showMessage(zing + " " + mydest + "\n")
 
-	def _merged_path(self, path, lstatobj):
+	def _merged_path(self, path, lstatobj, exists=True):
 		previous_path = self._device_path_map.get(lstatobj.st_dev)
-		if previous_path is None or len(path) < len(previous_path):
-			self._device_path_map[lstatobj.st_dev] = path
+		if previous_path is None or previous_path is False or \
+			(exists and len(path) < len(previous_path)):
+			if exists:
+				self._device_path_map[lstatobj.st_dev] = path
+			else:
+				# This entry is used to indicate that we've unmerged
+				# a file from this device, and later, this entry is
+				# replaced by a parent directory.
+				self._device_path_map[lstatobj.st_dev] = False
 
 	def _post_merge_sync(self):
 		"""
@@ -4731,6 +4738,8 @@ class dblink(object):
 				proc.wait()
 		else:
 			for path in self._device_path_map.values():
+				if path is False:
+					continue
 				try:
 					fd = os.open(path, os.O_RDONLY)
 				except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3f2451c0901ee75e3e4e0f11628a0a3a981eff88
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 25 15:31:16 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 25 15:42:44 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3f2451c0

merge-sync: syncfs on parent of dir for unmerge

Note that we use a realpath and a regular stat here, since we want to
follow any symlinks back to the real device where the real parent
directory resides.

---
 pym/portage/dbapi/vartree.py |   41 ++++++++++++++++++++++++++++++++++-------
 1 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index beaeda7..5bcd077 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2151,10 +2151,12 @@ class dblink(object):
 						["Could not chmod or unlink '%s': %s" % \
 						(file_name, ose)])
 				else:
-					try:
-						self._merged_path(file_name, lstatobj)
-					except OSError:
-						pass
+
+					# Even though the file no longer exists, we log it
+					# here so that _unmerge_dirs can see that we've
+					# removed a file from this device, and will record
+					# the parent directory for a syncfs call.
+					self._merged_path(file_name, lstatobj)
 
 				finally:
 					if bsd_chflags and pflags != 0:
@@ -2576,15 +2578,19 @@ class dblink(object):
 								raise
 							del e
 							show_unmerge("!!!", "", "obj", child)
+
 			try:
+				parent_name = os.path.dirname(obj)
+				parent_stat = os.stat(parent_name)
+
 				if bsd_chflags:
 					lstatobj = os.lstat(obj)
 					if lstatobj.st_flags != 0:
 						bsd_chflags.lchflags(obj, 0)
-					parent_name = os.path.dirname(obj)
+
 					# Use normal stat/chflags for the parent since we want to
 					# follow any symlinks to the real parent directory.
-					pflags = os.stat(parent_name).st_flags
+					pflags = parent_stat.st_flags
 					if pflags != 0:
 						bsd_chflags.chflags(parent_name, 0)
 				try:
@@ -2593,13 +2599,34 @@ class dblink(object):
 					if bsd_chflags and pflags != 0:
 						# Restore the parent flags we saved before unlinking
 						bsd_chflags.chflags(parent_name, pflags)
+
+				# Record the parent directory for use in syncfs calls.
+				# Note that we use a realpath and a regular stat here, since
+				# we want to follow any symlinks back to the real device where
+				# the real parent directory resides.
+				self._merged_path(os.path.realpath(parent_name), parent_stat)
+
 				show_unmerge("<<<", "", "dir", obj)
 			except EnvironmentError as e:
 				if e.errno not in ignored_rmdir_errnos:
 					raise
 				if e.errno != errno.ENOENT:
 					show_unmerge("---", unmerge_desc["!empty"], "dir", obj)
-				del e
+
+				# Since we didn't remove this directory, record the directory
+				# itself for use in syncfs calls, if we have removed another
+				# file from the same device.
+				# Note that we use a realpath and a regular stat here, since
+				# we want to follow any symlinks back to the real device where
+				# the real directory resides.
+				try:
+					dir_stat = os.stat(obj)
+				except OSError:
+					pass
+				else:
+					if dir_stat.st_dev in self._device_path_map:
+						self._merged_path(os.path.realpath(obj), dir_stat)
+
 			else:
 				# When a directory is successfully removed, there's
 				# no need to protect symlinks that point to it.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     6d94d08143c3f198a8ebfca1573cebefaa3e8bb0
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 25 15:31:16 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 25 15:33:46 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6d94d081

merge-sync: syncfs on parent of dir for unmerge

Note that we use a realpath and a regular stat here, since we want to
follow any symlinks back to the real device where the real parent
directory resides.

---
 pym/portage/dbapi/vartree.py |   36 ++++++++++++++++++++++++++++--------
 1 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index beaeda7..0a03496 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2150,11 +2150,6 @@ class dblink(object):
 					self._eerror("postrm", 
 						["Could not chmod or unlink '%s': %s" % \
 						(file_name, ose)])
-				else:
-					try:
-						self._merged_path(file_name, lstatobj)
-					except OSError:
-						pass
 
 				finally:
 					if bsd_chflags and pflags != 0:
@@ -2576,15 +2571,19 @@ class dblink(object):
 								raise
 							del e
 							show_unmerge("!!!", "", "obj", child)
+
 			try:
+				parent_name = os.path.dirname(obj)
+				parent_stat = os.stat(parent_name)
+
 				if bsd_chflags:
 					lstatobj = os.lstat(obj)
 					if lstatobj.st_flags != 0:
 						bsd_chflags.lchflags(obj, 0)
-					parent_name = os.path.dirname(obj)
+
 					# Use normal stat/chflags for the parent since we want to
 					# follow any symlinks to the real parent directory.
-					pflags = os.stat(parent_name).st_flags
+					pflags = parent_stat.st_flags
 					if pflags != 0:
 						bsd_chflags.chflags(parent_name, 0)
 				try:
@@ -2593,13 +2592,34 @@ class dblink(object):
 					if bsd_chflags and pflags != 0:
 						# Restore the parent flags we saved before unlinking
 						bsd_chflags.chflags(parent_name, pflags)
+
+				# Record the parent directory for use in syncfs calls.
+				# Note that we use a realpath and a regular stat here, since
+				# we want to follow any symlinks back to the real device where
+				# the real parent directory resides.
+				self._merged_path(os.path.realpath(parent_name), parent_stat)
+
 				show_unmerge("<<<", "", "dir", obj)
 			except EnvironmentError as e:
 				if e.errno not in ignored_rmdir_errnos:
 					raise
 				if e.errno != errno.ENOENT:
 					show_unmerge("---", unmerge_desc["!empty"], "dir", obj)
-				del e
+
+				# Since we didn't remove this directory, record the directory
+				# itself for use in syncfs calls, if we have removed another
+				# file from the same device.
+				# Note that we use a realpath and a regular stat here, since
+				# we want to follow any symlinks back to the real device where
+				# the real directory resides.
+				try:
+					dir_stat = os.stat(obj)
+				except OSError:
+					pass
+				else:
+					if dir_stat.st_dev in self._device_path_map:
+						self._merged_path(os.path.realpath(obj), dir_stat)
+
 			else:
 				# When a directory is successfully removed, there's
 				# no need to protect symlinks that point to it.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3ddca7ff1046a48d64ee23fb75f3b6b2eec0e22f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 25 15:31:16 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 25 15:31:16 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3ddca7ff

merge-sync: syncfs on parent of dir for unmerge

Note that we use a realpath and a regular stat here, since we want to
follow any symlinks back to the real device where the real parent
directory resides.

---
 pym/portage/dbapi/vartree.py |   36 ++++++++++++++++++++++++++++--------
 1 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index beaeda7..c20377b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2150,11 +2150,6 @@ class dblink(object):
 					self._eerror("postrm", 
 						["Could not chmod or unlink '%s': %s" % \
 						(file_name, ose)])
-				else:
-					try:
-						self._merged_path(file_name, lstatobj)
-					except OSError:
-						pass
 
 				finally:
 					if bsd_chflags and pflags != 0:
@@ -2576,15 +2571,19 @@ class dblink(object):
 								raise
 							del e
 							show_unmerge("!!!", "", "obj", child)
+
 			try:
+				parent_name = os.path.dirname(obj)
+				parent_stat = os.stat(parent_name)
+
 				if bsd_chflags:
 					lstatobj = os.lstat(obj)
 					if lstatobj.st_flags != 0:
 						bsd_chflags.lchflags(obj, 0)
-					parent_name = os.path.dirname(obj)
+
 					# Use normal stat/chflags for the parent since we want to
 					# follow any symlinks to the real parent directory.
-					pflags = os.stat(parent_name).st_flags
+					pflags = parent_stat.st_flags
 					if pflags != 0:
 						bsd_chflags.chflags(parent_name, 0)
 				try:
@@ -2593,13 +2592,34 @@ class dblink(object):
 					if bsd_chflags and pflags != 0:
 						# Restore the parent flags we saved before unlinking
 						bsd_chflags.chflags(parent_name, pflags)
+
+				# Record the parent directory for use in synfs calls.
+				# Note that we use a realpath and a regular stat here, since
+				# we want to follow any symlinks back to the real device where
+				# the real parent directory resides.
+				self._merged_path(os.path.realpath(parent_name), parent_stat)
+
 				show_unmerge("<<<", "", "dir", obj)
 			except EnvironmentError as e:
 				if e.errno not in ignored_rmdir_errnos:
 					raise
 				if e.errno != errno.ENOENT:
 					show_unmerge("---", unmerge_desc["!empty"], "dir", obj)
-				del e
+
+				# Since we didn't remove this directory, record the directory
+				# itself for use in synfs calls, if we have removed another
+				# file from the same device.
+				# Note that we use a realpath and a regular stat here, since
+				# we want to follow any symlinks back to the real device where
+				# the real directory resides.
+				try:
+					dir_stat = os.stat(obj)
+				except OSError:
+					pass
+				else:
+					if dir_stat.st_dev in self._device_path_map:
+						self._merged_path(os.path.realpath(obj), dir_stat)
+
 			else:
 				# When a directory is successfully removed, there's
 				# no need to protect symlinks that point to it.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     fa83d9cbe167f5fb54052f544cad4ddd7a264843
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 25 09:32:35 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 25 09:35:02 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fa83d9cb

MergeProcess: cache syncfs libc library lookup

---
 pym/portage/dbapi/_MergeProcess.py |    6 ++++++
 pym/portage/dbapi/vartree.py       |   28 ++++++++++++++--------------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 829a3d7..9be68d3 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -10,6 +10,7 @@ import errno
 import fcntl
 import portage
 from portage import os, _unicode_decode
+from portage.dbapi.vartree import _get_syncfs
 import portage.elog.messages
 from portage.util._async.ForkProcess import ForkProcess
 
@@ -40,6 +41,11 @@ class MergeProcess(ForkProcess):
 			settings.reset()
 			settings.setcpv(cpv, mydb=self.mydbapi)
 
+		# This caches the libc library lookup in the current
+		# process, so that it's only done once rather than
+		# for each child process.
+		_get_syncfs()
+
 		# Inherit stdin by default, so that the pdb SIGUSR1
 		# handler is usable for the subprocess.
 		if self.fd_pipes is None:

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index edc477a..beaeda7 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4694,7 +4694,7 @@ class dblink(object):
 			"merge-sync" not in self.settings.features:
 			return
 
-		syncfs = self._get_syncfs()
+		syncfs = _get_syncfs()
 		if syncfs is None:
 			try:
 				proc = subprocess.Popen(["sync"])
@@ -4716,19 +4716,6 @@ class dblink(object):
 					finally:
 						os.close(fd)
 
-	def _get_syncfs(self):
-		if platform.system() == "Linux":
-			filename = find_library("c")
-			if filename is not None:
-				library = LoadLibrary(filename)
-				if library is not None:
-					try:
-						return library.syncfs
-					except AttributeError:
-						pass
-
-		return None
-
 	def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
 		mydbapi=None, prev_mtimes=None, counter=None):
 		"""
@@ -4907,6 +4894,19 @@ class dblink(object):
 		finally:
 			self.unlockdb()
 
+def _get_syncfs():
+	if platform.system() == "Linux":
+		filename = find_library("c")
+		if filename is not None:
+			library = LoadLibrary(filename)
+			if library is not None:
+				try:
+					return library.syncfs
+				except AttributeError:
+					pass
+
+	return None
+
 def merge(mycat, mypkg, pkgloc, infloc,
 	myroot=None, settings=None, myebuild=None,
 	mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     133d687b7d3a9d5345ef7628bb4c60663baf8309
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 25 09:32:35 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 25 09:32:35 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=133d687b

MergeProcess: cache syncfs libc library lookup

---
 pym/portage/dbapi/_MergeProcess.py |    6 ++++++
 pym/portage/dbapi/vartree.py       |   28 ++++++++++++++--------------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 829a3d7..cefd37e 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -10,6 +10,7 @@ import errno
 import fcntl
 import portage
 from portage import os, _unicode_decode
+from portage.dbapi.vartree import _get_syncfs
 import portage.elog.messages
 from portage.util._async.ForkProcess import ForkProcess
 
@@ -40,6 +41,11 @@ class MergeProcess(ForkProcess):
 			settings.reset()
 			settings.setcpv(cpv, mydb=self.mydbapi)
 
+		# This caches the libc library lookup in the current
+		# process, that it's only done once rather than
+		# for each child process.
+		_get_syncfs()
+
 		# Inherit stdin by default, so that the pdb SIGUSR1
 		# handler is usable for the subprocess.
 		if self.fd_pipes is None:

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index edc477a..beaeda7 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4694,7 +4694,7 @@ class dblink(object):
 			"merge-sync" not in self.settings.features:
 			return
 
-		syncfs = self._get_syncfs()
+		syncfs = _get_syncfs()
 		if syncfs is None:
 			try:
 				proc = subprocess.Popen(["sync"])
@@ -4716,19 +4716,6 @@ class dblink(object):
 					finally:
 						os.close(fd)
 
-	def _get_syncfs(self):
-		if platform.system() == "Linux":
-			filename = find_library("c")
-			if filename is not None:
-				library = LoadLibrary(filename)
-				if library is not None:
-					try:
-						return library.syncfs
-					except AttributeError:
-						pass
-
-		return None
-
 	def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
 		mydbapi=None, prev_mtimes=None, counter=None):
 		"""
@@ -4907,6 +4894,19 @@ class dblink(object):
 		finally:
 			self.unlockdb()
 
+def _get_syncfs():
+	if platform.system() == "Linux":
+		filename = find_library("c")
+		if filename is not None:
+			library = LoadLibrary(filename)
+			if library is not None:
+				try:
+					return library.syncfs
+				except AttributeError:
+					pass
+
+	return None
+
 def merge(mycat, mypkg, pkgloc, infloc,
 	myroot=None, settings=None, myebuild=None,
 	mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e440548da95775019dc89bb612f8049240eed4e9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 25 06:59:31 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 25 08:23:33 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e440548d

dblink: add hooks to sync disk for bug #439584

---
 pym/portage/dbapi/vartree.py |   59 +++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 58 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 46afea5..cb7b39b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1486,6 +1486,7 @@ class dblink(object):
 		self._contents_inodes = None
 		self._contents_basenames = None
 		self._linkmap_broken = False
+		self._device_path_map = {}
 		self._hardlink_merge_map = {}
 		self._hash_key = (self._eroot, self.mycpv)
 		self._protect_obj = None
@@ -1533,7 +1534,11 @@ class dblink(object):
 		"""
 		Remove this entry from the database
 		"""
-		if not os.path.exists(self.dbdir):
+		try:
+			os.lstat(self.dbdir)
+		except OSError as e:
+			if e.errno not in (errno.ENOENT, errno.ENOTDIR, errno.ESTALE):
+				raise
 			return
 
 		# Check validity of self.dbdir before attempting to remove it.
@@ -1550,6 +1555,14 @@ class dblink(object):
 			pass
 		self.vartree.dbapi._remove(self)
 
+		# Use self.dbroot since we need an existing path for syncfs.
+		try:
+			self._merged_path(self.dbroot, os.lstat(self.dbroot))
+		except OSError:
+			pass
+
+		self._post_merge_sync()
+
 	def clearcontents(self):
 		"""
 		For a given db entry (self), erase the CONTENTS values.
@@ -2135,6 +2148,12 @@ class dblink(object):
 					self._eerror("postrm", 
 						["Could not chmod or unlink '%s': %s" % \
 						(file_name, ose)])
+				else:
+					try:
+						self._merged_path(file_name, lstatobj)
+					except OSError:
+						pass
+
 				finally:
 					if bsd_chflags and pflags != 0:
 						# Restore the parent flags we saved before unlinking
@@ -4048,6 +4067,7 @@ class dblink(object):
 		try:
 			self.delete()
 			_movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings)
+			self._merged_path(self.dbpkgdir, os.lstat(self.dbpkgdir))
 		finally:
 			self.unlockdb()
 
@@ -4163,6 +4183,7 @@ class dblink(object):
 		# For gcc upgrades, preserved libs have to be removed after the
 		# the library path has been updated.
 		self._prune_plib_registry()
+		self._post_merge_sync()
 
 		return os.EX_OK
 
@@ -4412,6 +4433,12 @@ class dblink(object):
 				mymtime = movefile(mysrc, mydest, newmtime=thismtime,
 					sstat=mystat, mysettings=self.settings,
 					encoding=_encodings['merge'])
+
+				try:
+					self._merged_path(mydest, os.lstat(mydest))
+				except OSError:
+					pass
+
 				if mymtime != None:
 					showMessage(">>> %s -> %s\n" % (mydest, myto))
 					if sys.hexversion >= 0x3030000:
@@ -4511,6 +4538,12 @@ class dblink(object):
 					os.chmod(mydest, mystat[0])
 					os.chown(mydest, mystat[4], mystat[5])
 					showMessage(">>> %s/\n" % mydest)
+
+				try:
+					self._merged_path(mydest, os.lstat(mydest))
+				except OSError:
+					pass
+
 				outfile.write("dir "+myrealdest+"\n")
 				# recurse and merge this directory
 				if self.mergeme(srcroot, destroot, outfile, secondhand,
@@ -4610,6 +4643,11 @@ class dblink(object):
 					hardlink_candidates.append(mydest)
 					zing = ">>>"
 
+					try:
+						self._merged_path(mydest, os.lstat(mydest))
+					except OSError:
+						pass
+
 				if mymtime != None:
 					if sys.hexversion >= 0x3030000:
 						outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n")
@@ -4625,6 +4663,12 @@ class dblink(object):
 						sstat=mystat, mysettings=self.settings,
 						encoding=_encodings['merge']) is not None:
 						zing = ">>>"
+
+						try:
+							self._merged_path(mydest, os.lstat(mydest))
+						except OSError:
+							pass
+
 					else:
 						return 1
 				if stat.S_ISFIFO(mymode):
@@ -4633,6 +4677,19 @@ class dblink(object):
 					outfile.write("dev %s\n" % myrealdest)
 				showMessage(zing + " " + mydest + "\n")
 
+	def _merged_path(self, path, lstatobj):
+		previous_path = self._device_path_map.get(lstatobj.st_dev)
+		if previous_path is None or len(path) < len(previous_path):
+			self._device_path_map[lstatobj.st_dev] = path
+
+	def _post_merge_sync(self):
+		"""
+		Call this after merge or unmerge, in order to sync relevant files to
+		disk and avoid data-loss in the event of a power failure. This method
+		does nothing if FEATURES=merge-sync is disabled.
+		"""
+		pass
+
 	def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
 		mydbapi=None, prev_mtimes=None, counter=None):
 		"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8fd188a16dbee53f7f22f85f37497155ca2aac8e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 25 06:59:31 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 25 06:59:31 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8fd188a1

dblink: add hooks to sync disk for bug #439584

---
 pym/portage/dbapi/vartree.py |   54 +++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 53 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 46afea5..5f4466f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1486,6 +1486,7 @@ class dblink(object):
 		self._contents_inodes = None
 		self._contents_basenames = None
 		self._linkmap_broken = False
+		self._device_path_map = {}
 		self._hardlink_merge_map = {}
 		self._hash_key = (self._eroot, self.mycpv)
 		self._protect_obj = None
@@ -1533,7 +1534,11 @@ class dblink(object):
 		"""
 		Remove this entry from the database
 		"""
-		if not os.path.exists(self.dbdir):
+		try:
+			lstatobj = os.lstat(self.dbdir)
+		except OSError as e:
+			if e.errno not in (errno.ENOENT, errno.ENOTDIR, errno.ESTALE):
+				raise
 			return
 
 		# Check validity of self.dbdir before attempting to remove it.
@@ -1550,6 +1555,9 @@ class dblink(object):
 			pass
 		self.vartree.dbapi._remove(self)
 
+		self._merged_path(self.dbdir, lstatobj)
+		self._post_merge_sync()
+
 	def clearcontents(self):
 		"""
 		For a given db entry (self), erase the CONTENTS values.
@@ -2135,6 +2143,12 @@ class dblink(object):
 					self._eerror("postrm", 
 						["Could not chmod or unlink '%s': %s" % \
 						(file_name, ose)])
+				else:
+					try:
+						self._merged_path(file_name, lstatobj)
+					except OSError:
+						pass
+
 				finally:
 					if bsd_chflags and pflags != 0:
 						# Restore the parent flags we saved before unlinking
@@ -4048,6 +4062,7 @@ class dblink(object):
 		try:
 			self.delete()
 			_movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings)
+			self._merged_path(self.dbpkgdir, os.lstat(self.dbpkgdir))
 		finally:
 			self.unlockdb()
 
@@ -4163,6 +4178,7 @@ class dblink(object):
 		# For gcc upgrades, preserved libs have to be removed after the
 		# the library path has been updated.
 		self._prune_plib_registry()
+		self._post_merge_sync()
 
 		return os.EX_OK
 
@@ -4412,6 +4428,12 @@ class dblink(object):
 				mymtime = movefile(mysrc, mydest, newmtime=thismtime,
 					sstat=mystat, mysettings=self.settings,
 					encoding=_encodings['merge'])
+
+				try:
+					self._merged_path(mydest, os.lstat(mydest))
+				except OSError:
+					pass
+
 				if mymtime != None:
 					showMessage(">>> %s -> %s\n" % (mydest, myto))
 					if sys.hexversion >= 0x3030000:
@@ -4511,6 +4533,12 @@ class dblink(object):
 					os.chmod(mydest, mystat[0])
 					os.chown(mydest, mystat[4], mystat[5])
 					showMessage(">>> %s/\n" % mydest)
+
+				try:
+					self._merged_path(mydest, os.lstat(mydest))
+				except OSError:
+					pass
+
 				outfile.write("dir "+myrealdest+"\n")
 				# recurse and merge this directory
 				if self.mergeme(srcroot, destroot, outfile, secondhand,
@@ -4610,6 +4638,11 @@ class dblink(object):
 					hardlink_candidates.append(mydest)
 					zing = ">>>"
 
+					try:
+						self._merged_path(mydest, os.lstat(mydest))
+					except OSError:
+						pass
+
 				if mymtime != None:
 					if sys.hexversion >= 0x3030000:
 						outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n")
@@ -4625,6 +4658,12 @@ class dblink(object):
 						sstat=mystat, mysettings=self.settings,
 						encoding=_encodings['merge']) is not None:
 						zing = ">>>"
+
+						try:
+							self._merged_path(mydest, os.lstat(mydest))
+						except OSError:
+							pass
+
 					else:
 						return 1
 				if stat.S_ISFIFO(mymode):
@@ -4633,6 +4672,19 @@ class dblink(object):
 					outfile.write("dev %s\n" % myrealdest)
 				showMessage(zing + " " + mydest + "\n")
 
+	def _merged_path(self, path, lstatobj):
+		previous_path = self._device_path_map.get(lstatobj.st_dev)
+		if previous_path is None or len(path) < len(previous_path):
+			self._device_path_map[lstatobj.st_dev] = path
+
+	def _post_merge_sync(self):
+		"""
+		Call this after merge or unmerge, in order to sync relevant files to
+		disk and avoid data-loss in the event of a power failure. This method
+		does nothing if FEATURES=merge-sync is disabled.
+		"""
+		pass
+
 	def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
 		mydbapi=None, prev_mtimes=None, counter=None):
 		"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5608e26193249acf4d072f7658f88ea7e7222ae4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 18 03:32:20 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 18 03:32:20 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5608e261

bintree/portree: fix getslot for EAPI 5 sub-slot

---
 pym/portage/dbapi/bintree.py  |    6 ++----
 pym/portage/dbapi/porttree.py |    6 ++----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index cbcfa72..f90a493 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1484,9 +1484,7 @@ class binarytree(object):
 		"Get a slot for a catpkg; assume it exists."
 		myslot = ""
 		try:
-			myslot = self.dbapi.aux_get(mycatpkg,["SLOT"])[0]
-		except SystemExit as e:
-			raise
-		except Exception as e:
+			myslot = self.dbapi._pkg_str(mycatpkg, None).slot
+		except KeyError:
 			pass
 		return myslot

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index a5945be..115fc7f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1063,10 +1063,8 @@ class portagetree(object):
 		"Get a slot for a catpkg; assume it exists."
 		myslot = ""
 		try:
-			myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0]
-		except SystemExit:
-			raise
-		except Exception:
+			myslot = self.dbapi._pkg_str(mycatpkg, None).slot
+		except KeyError:
 			pass
 		return myslot
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     303741ace78498c28573216b8db5ac15c71e379f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 18 00:48:13 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 18 00:48:13 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=303741ac

vartree.py: account for EAPI 5 sub-slot in SLOT

---
 pym/portage/dbapi/vartree.py |   34 ++++++++++++++++++----------------
 1 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 07b3cdb..30b6cb1 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -35,7 +35,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util._async.SchedulerInterface:SchedulerInterface',
 	'portage.util._eventloop.EventLoop:EventLoop',
 	'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \
-		'_pkgsplit@pkgsplit,_pkg_str',
+		'_pkgsplit@pkgsplit,_pkg_str,_unknown_repo',
 	'subprocess',
 	'tarfile',
 )
@@ -1393,7 +1393,7 @@ class vartree(object):
 	def getslot(self, mycatpkg):
 		"Get a slot for a catpkg; assume it exists."
 		try:
-			return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0]
+			return self.dbapi._pkg_str(mycatpkg, None).slot
 		except KeyError:
 			return ""
 
@@ -1701,8 +1701,11 @@ class dblink(object):
 						unmerge_preserve = \
 							self._find_libs_to_preserve(unmerge=True)
 					counter = self.vartree.dbapi.cpv_counter(self.mycpv)
-					plib_registry.unregister(self.mycpv,
-						self.settings["SLOT"], counter)
+					try:
+						slot = self.mycpv.slot
+					except AttributeError:
+						slot = _pkg_str(self.mycpv, slot=self.settings["SLOT"]).slot
+					plib_registry.unregister(self.mycpv, slot, counter)
 					if unmerge_preserve:
 						for path in sorted(unmerge_preserve):
 							contents_key = self._match_contents(path)
@@ -1712,7 +1715,7 @@ class dblink(object):
 							self._display_merge(_(">>> needed   %s %s\n") % \
 								(obj_type, contents_key), noiselevel=-1)
 						plib_registry.register(self.mycpv,
-							self.settings["SLOT"], counter, unmerge_preserve)
+							slot, counter, unmerge_preserve)
 						# Remove the preserved files from our contents
 						# so that they won't be unmerged.
 						self.vartree.dbapi.removeFromContents(self,
@@ -1807,7 +1810,7 @@ class dblink(object):
 		# done for this slot, so it shouldn't be repeated until the next
 		# replacement or unmerge operation.
 		if others_in_slot is None:
-			slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0]
+			slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot
 			slot_matches = self.vartree.dbapi.match(
 				"%s:%s" % (portage.cpv_getkey(self.mycpv), slot))
 			others_in_slot = []
@@ -2075,7 +2078,7 @@ class dblink(object):
 
 		if others_in_slot is None:
 			others_in_slot = []
-			slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0]
+			slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot
 			slot_matches = self.vartree.dbapi.match(
 				"%s:%s" % (portage.cpv_getkey(self.mycpv), slot))
 			for cur_cpv in slot_matches:
@@ -3811,14 +3814,13 @@ class dblink(object):
 					self.vartree.dbapi.flush_cache()
 
 					for pkg in owners:
-						other_slot, other_repo = self.vartree.dbapi.aux_get(
-							pkg.mycpv, ["SLOT", "repository"])
-						pkg_info_str = "%s%s%s" % (pkg.mycpv,
-							_slot_separator, other_slot)
-						if other_repo:
+						pkg = self.vartree.dbapi._pkg_str(pkg.mycpv, None)
+						pkg_info_str = "%s%s%s" % (pkg,
+							_slot_separator, pkg.slot)
+						if pkg.repo != _unknown_repo:
 							pkg_info_str += "%s%s" % (_repo_separator,
-								other_repo)
-						pkg_info_strs[pkg.mycpv] = pkg_info_str
+								pkg.repo)
+						pkg_info_strs[pkg] = pkg_info_str
 
 				finally:
 					self.unlockdb()
@@ -4090,8 +4092,8 @@ class dblink(object):
 						self.vartree.dbapi.lock()
 						try:
 							try:
-								slot, counter = self.vartree.dbapi.aux_get(
-									cpv, ["SLOT", "COUNTER"])
+								slot = self.vartree.dbapi._pkg_str(cpv, None).slot
+								counter = self.vartree.dbapi.cpv_counter(cpv)
 							except KeyError:
 								pass
 							else:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     960792b9fd0e11b16efdc0b538231288b8727862
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  9 02:01:45 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct  9 02:01:45 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=960792b9

_parse_uri_map: preserve order of URIs

Returns a dict of tuples instead of a dict of sets.

---
 pym/portage/dbapi/porttree.py |   11 ++++++++---
 1 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index dba0ef5..a5945be 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1138,9 +1138,14 @@ def _parse_uri_map(cpv, metadata, use=None):
 
 		uri_set = uri_map.get(distfile)
 		if uri_set is None:
-			uri_set = set()
+			# Use OrderedDict to preserve order from SRC_URI
+			# while ensuring uniqueness.
+			uri_set = OrderedDict()
 			uri_map[distfile] = uri_set
-		uri_set.add(uri)
-		uri = None
+		uri_set[uri] = True
+
+	# Convert OrderedDicts to tuples.
+	for k, v in uri_map.items():
+		uri_map[k] = tuple(v)
 
 	return uri_map

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     83577043ac769db95e490ef180de14d01072f13a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  7 22:02:35 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct  7 22:02:35 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=83577043

Show slot + repo for colliding packages.

See bug #437516 for example.

---
 pym/portage/dbapi/vartree.py |   18 +++++++++++++++---
 1 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 11d9800..07b3cdb 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -12,7 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.dbapi.dep_expand:dep_expand',
 	'portage.dbapi._MergeProcess:MergeProcess',
 	'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \
-	 	'use_reduce,_get_slot_re',
+	 	'use_reduce,_get_slot_re,_slot_separator,_repo_separator',
 	'portage.eapi:_get_eapi_attrs',
 	'portage.elog:collect_ebuild_messages,collect_messages,' + \
 		'elog_process,_merge_logentries',
@@ -3803,17 +3803,29 @@ class dblink(object):
 					# get_owners is slow for large numbers of files, so
 					# don't look them all up.
 					collisions = collisions[:20]
+
+				pkg_info_strs = {}
 				self.lockdb()
 				try:
 					owners = self.vartree.dbapi._owners.get_owners(collisions)
 					self.vartree.dbapi.flush_cache()
+
+					for pkg in owners:
+						other_slot, other_repo = self.vartree.dbapi.aux_get(
+							pkg.mycpv, ["SLOT", "repository"])
+						pkg_info_str = "%s%s%s" % (pkg.mycpv,
+							_slot_separator, other_slot)
+						if other_repo:
+							pkg_info_str += "%s%s" % (_repo_separator,
+								other_repo)
+						pkg_info_strs[pkg.mycpv] = pkg_info_str
+
 				finally:
 					self.unlockdb()
 
 				for pkg, owned_files in owners.items():
-					cpv = pkg.mycpv
 					msg = []
-					msg.append("%s" % cpv)
+					msg.append(pkg_info_strs[pkg.mycpv])
 					for f in sorted(owned_files):
 						msg.append("\t%s" % os.path.join(destroot,
 							f.lstrip(os.path.sep)))

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0f07531fa54158a84ca0105b7652ffd9a3f2986e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Sep 26 03:47:39 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Sep 26 03:47:39 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0f07531f

update_ents: use Package._dep_keys

---
 pym/portage/dbapi/__init__.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index b7c0c7b..dbba22c 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -19,6 +19,7 @@ from portage import auxdbkeys
 from portage.eapi import _get_eapi_attrs
 from portage.exception import InvalidData
 from portage.localization import _
+from _emerge.Package import Package
 
 class dbapi(object):
 	_category_re = re.compile(r'^\w[-.+\w]*$', re.UNICODE)
@@ -290,8 +291,7 @@ class dbapi(object):
 		maxval = len(cpv_all)
 		aux_get = self.aux_get
 		aux_update = self.aux_update
-		meta_keys = ["DEPEND", "EAPI", "HDEPEND",
-			"PDEPEND", "PROVIDE", "RDEPEND", 'repository']
+		meta_keys = Package._dep_keys + ("EAPI", "PROVIDE", "repository")
 		repo_dict = None
 		if isinstance(updates, dict):
 			repo_dict = updates

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3da62f9d447c0bfa4089554deb485c4eeeae9879
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 16 21:16:41 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Sep 16 21:16:41 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3da62f9d

Fix "unsatisfied" comment typo.

---
 pym/portage/dbapi/__init__.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index e36a0ef..ad22f39 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -253,7 +253,7 @@ class dbapi(object):
 				for x in atom.use.disabled):
 				return False
 
-			# Check unsatsified use-default deps
+			# Check unsatisfied use-default deps
 			if atom.use.enabled:
 				missing_disabled = atom.use.missing_disabled.difference(iuse)
 				if any(x in atom.use.enabled for x in missing_disabled):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     ca79d9994fc43909203daad5f2c12f750a504b28
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 16 21:01:29 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Sep 16 21:01:29 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ca79d999

repoman: unsatsified use-default deps, bug 435242

---
 pym/portage/dbapi/__init__.py |   10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index dc0a3da..e36a0ef 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -253,6 +253,16 @@ class dbapi(object):
 				for x in atom.use.disabled):
 				return False
 
+			# Check unsatsified use-default deps
+			if atom.use.enabled:
+				missing_disabled = atom.use.missing_disabled.difference(iuse)
+				if any(x in atom.use.enabled for x in missing_disabled):
+					return False
+			if atom.use.disabled:
+				missing_enabled = atom.use.missing_enabled.difference(iuse)
+				if any(x in atom.use.disabled for x in missing_enabled):
+					return False
+
 		return True
 
 	def invalidentry(self, mypath):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e95a07267c7f642fdca2aca346ab4c12f46748bb
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Aug  8 20:34:15 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Aug  8 20:51:45 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e95a0726

bintree.inject(): write Packages.gz atomically

---
 pym/portage/dbapi/bintree.py |    9 ++++++---
 1 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 77fc0c4..8dd7aad 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1193,13 +1193,16 @@ class binarytree(object):
 			atime = mtime = long(pkgindex.header["TIMESTAMP"])
 
 			pkgindex_filename = os.path.join(self.pkgdir, "Packages")
-			output_files = [(atomic_ofstream(pkgindex_filename, mode="wb"), pkgindex_filename)]
+			output_files = [(atomic_ofstream(pkgindex_filename, mode="wb"), pkgindex_filename, None)]
 			if "compress-index" in self.settings.features:
 				gz_fname = pkgindex_filename + ".gz"
-				output_files.append((GzipFile(gz_fname, mode="wb"), gz_fname))
-			for f, fname in output_files:
+				fileobj = atomic_ofstream(gz_fname, mode="wb")
+				output_files.append((GzipFile(filename='', mode="wb", fileobj=fileobj, mtime=mtime), gz_fname, fileobj))
+			for f, fname, f_close in output_files:
 				f.write(contents)
 				f.close()
+				if f_close is not None:
+					f_close.close()
 				# some seconds might have elapsed since TIMESTAMP
 				os.utime(fname, (atime, mtime))
 		finally:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e5233e3c56a3be2db78c0806afabb1c5b7b96bc8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Aug  8 20:34:15 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Aug  8 20:34:15 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e5233e3c

bintree.inject(): write Packages.gz atomically

---
 pym/portage/dbapi/bintree.py |    9 ++++++---
 1 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 77fc0c4..2872e3b 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1193,15 +1193,18 @@ class binarytree(object):
 			atime = mtime = long(pkgindex.header["TIMESTAMP"])
 
 			pkgindex_filename = os.path.join(self.pkgdir, "Packages")
-			output_files = [(atomic_ofstream(pkgindex_filename, mode="wb"), pkgindex_filename)]
+			output_files = [(atomic_ofstream(pkgindex_filename, mode="wb"), pkgindex_filename, None)]
 			if "compress-index" in self.settings.features:
 				gz_fname = pkgindex_filename + ".gz"
-				output_files.append((GzipFile(gz_fname, mode="wb"), gz_fname))
-			for f, fname in output_files:
+				tmp_fname = "%s.%s" % (gz_fname, os.getpid())
+				output_files.append((GzipFile(tmp_fname, mode="wb"), tmp_fname, gz_fname))
+			for f, fname, fname_rename in output_files:
 				f.write(contents)
 				f.close()
 				# some seconds might have elapsed since TIMESTAMP
 				os.utime(fname, (atime, mtime))
+				if fname_rename is not None:
+					os.rename(fname, fname_rename)
 		finally:
 			if pkgindex_lock:
 				unlockfile(pkgindex_lock)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d9c77eec3c74c3b22b53911afaf2c167818fbc79
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Aug  2 20:22:02 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Aug  2 20:22:02 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d9c77eec

vardbapi: optimize pickle load for Python >=3.2

---
 pym/portage/dbapi/vartree.py |    5 +++--
 1 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 5008a97..1dadd3c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -572,11 +572,12 @@ class vardbapi(dbapi):
 	def _aux_cache_init(self):
 		aux_cache = None
 		open_kwargs = {}
-		if sys.hexversion >= 0x3000000:
+		if sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000:
 			# Buffered io triggers extreme performance issues in
 			# Unpickler.load() (problem observed with python-3.0.1).
 			# Unfortunately, performance is still poor relative to
-			# python-2.x, but buffering makes it much worse.
+			# python-2.x, but buffering makes it much worse (problem
+			# appears to be solved in Python >=3.2 at least).
 			open_kwargs["buffering"] = 0
 		try:
 			f = open(_unicode_encode(self._aux_cache_filename,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     13abe0398fbe724218c8c9ac2597ebe15d7db7e1
Author:     W-Mark Kubacki <wmark <AT> hurrikane <DOT> de>
AuthorDate: Wed Aug  1 19:12:24 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Aug  2 00:54:44 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=13abe039

Fix index file's mtime, which can differ from TIMESTAMP.

This enables Portage to reliably query for remote indices with
HTTP-header If-Modified-Since.

Without this patch mtime is greater than TIMESTAMP for large
indices and slow storages - because writing a large file takes
time. If the difference spans a second (TIMESTAMP 08:00:00, mtime
08:00:01), then Portage will always fetch the remote index because
it will appear being modified (mtime is used there) after the copy
has been made (local copy's TIMESTAMP is used here).

---
 pym/portage/dbapi/bintree.py |    6 +++++-
 1 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 16ae8ec..0367503 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1186,9 +1186,13 @@ class binarytree(object):
 			pkgindex.packages.append(d)
 
 			self._update_pkgindex_header(pkgindex.header)
-			f = atomic_ofstream(os.path.join(self.pkgdir, "Packages"))
+			pkgindex_filename = os.path.join(self.pkgdir, "Packages")
+			f = atomic_ofstream(pkgindex_filename)
 			pkgindex.write(f)
 			f.close()
+			# some seconds might have elapsed since TIMESTAMP
+			atime = mtime = long(pkgindex.header["TIMESTAMP"])
+			os.utime(pkgindex_filename, (atime, mtime))
 		finally:
 			if pkgindex_lock:
 				unlockfile(pkgindex_lock)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Arfrever Frehtes Taifersar Arahesis]

commit:     907adbeeb89176c63ad7a04288e080f08f3eb401
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Tue Jul 31 23:07:31 2012 +0000
Commit:     Arfrever Frehtes Taifersar Arahesis <arfrever.fta <AT> gmail <DOT> com>
CommitDate: Tue Jul 31 23:07:31 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=907adbee

Use nanosecond precision in cache for category directory mtimes
and cache for cp_list results when using Python >=3.3.

---
 pym/portage/dbapi/vartree.py |   10 ++++++++--
 1 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 08580e8..5008a97 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -376,7 +376,10 @@ class vardbapi(dbapi):
 		if mysplit[0] == '*':
 			mysplit[0] = mysplit[0][1:]
 		try:
-			mystat = os.stat(self.getpath(mysplit[0])).st_mtime
+			if sys.hexversion >= 0x3030000:
+				mystat = os.stat(self.getpath(mysplit[0])).st_mtime_ns
+			else:
+				mystat = os.stat(self.getpath(mysplit[0])).st_mtime
 		except OSError:
 			mystat = 0
 		if use_cache and mycp in self.cpcache:
@@ -511,7 +514,10 @@ class vardbapi(dbapi):
 			return list(self._iter_match(mydep,
 				self.cp_list(mydep.cp, use_cache=use_cache)))
 		try:
-			curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime
+			if sys.hexversion >= 0x3030000:
+				curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime_ns
+			else:
+				curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime
 		except (IOError, OSError):
 			curmtime=0
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4f1558ec085d4d1716faf8558741aad2e542463f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 12 20:58:30 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jul 12 20:58:30 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4f1558ec

vardbapi: use float mtime for aux cache

---
 pym/portage/dbapi/vartree.py |   12 +++++++++---
 1 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index fbf2e74..dddc094 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -647,7 +647,8 @@ class vardbapi(dbapi):
 			if e.errno != errno.ENOENT:
 				raise
 			raise KeyError(mycpv)
-		mydir_mtime = mydir_stat[stat.ST_MTIME]
+		# Use float mtime when available.
+		mydir_mtime = mydir_stat.st_mtime
 		pkg_data = self._aux_cache["packages"].get(mycpv)
 		pull_me = cache_these.union(wants)
 		mydata = {"_mtime_" : mydir_mtime}
@@ -660,13 +661,18 @@ class vardbapi(dbapi):
 				pkg_data = None
 			else:
 				cache_mtime, metadata = pkg_data
-				if not isinstance(cache_mtime, (long, int)) or \
+				if not isinstance(cache_mtime, (float, long, int)) or \
 					not isinstance(metadata, dict):
 					pkg_data = None
 
 		if pkg_data:
 			cache_mtime, metadata = pkg_data
-			cache_valid = cache_mtime == mydir_mtime
+			if isinstance(cache_mtime, float):
+				cache_valid = cache_mtime == mydir_stat.st_mtime
+			else:
+				# Cache may contain integer mtime.
+				cache_valid = cache_mtime == mydir_stat[stat.ST_MTIME]
+
 		if cache_valid:
 			# Migrate old metadata to unicode.
 			for k, v in metadata.items():

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     89b1eabbbe99cf2d45a1f6d01ce1ffdadfae0ebf
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jul  5 03:28:42 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jul  5 03:28:42 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=89b1eabb

vartree.py: fix stutter in comment

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 289eaae..fbf2e74 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1847,7 +1847,7 @@ class dblink(object):
 				log_path = self.settings.get("PORTAGE_LOG_FILE")
 
 			# Do this before the following _prune_plib_registry call, since
-			# that removes preserved libraries from our CONTENTS, and we want
+			# that removes preserved libraries from our CONTENTS, and we
 			# may want to backup those libraries first.
 			if not caller_handles_backup:
 				retval = self._pre_unmerge_backup(background)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     cd6cc7631ac4930bfd89fd56d02e80c6a912ff96
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jul  2 22:27:28 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jul  2 22:27:28 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cd6cc763

Fix unmerge-backup/preserve-libs interaction.

---
 pym/portage/dbapi/vartree.py |    9 ++++++---
 1 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index d619754..289eaae 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1832,9 +1832,6 @@ class dblink(object):
 		except UnsupportedAPIException as e:
 			eapi_unsupported = e
 
-		self._prune_plib_registry(unmerge=True, needed=needed,
-			preserve_paths=preserve_paths)
-
 		builddir_lock = None
 		scheduler = self._scheduler
 		retval = os.EX_OK
@@ -1849,6 +1846,9 @@ class dblink(object):
 				prepare_build_dirs(settings=self.settings, cleanup=True)
 				log_path = self.settings.get("PORTAGE_LOG_FILE")
 
+			# Do this before the following _prune_plib_registry call, since
+			# that removes preserved libraries from our CONTENTS, and we want
+			# may want to backup those libraries first.
 			if not caller_handles_backup:
 				retval = self._pre_unmerge_backup(background)
 				if retval != os.EX_OK:
@@ -1856,6 +1856,9 @@ class dblink(object):
 						level=logging.ERROR, noiselevel=-1)
 					return retval
 
+			self._prune_plib_registry(unmerge=True, needed=needed,
+				preserve_paths=preserve_paths)
+
 			# Log the error after PORTAGE_LOG_FILE is initialized
 			# by prepare_build_dirs above.
 			if eapi_unsupported:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e4e04d20a81ed35d5732f683f2001ea0b3af77c0
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jul  2 21:41:19 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jul  2 21:41:19 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4e04d20

_pre_unmerge_backup: show failure message

---
 pym/portage/dbapi/vartree.py |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index dd37be0..d619754 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1852,6 +1852,8 @@ class dblink(object):
 			if not caller_handles_backup:
 				retval = self._pre_unmerge_backup(background)
 				if retval != os.EX_OK:
+					showMessage(_("!!! FAILED prerm: quickpkg: %s\n") % retval,
+						level=logging.ERROR, noiselevel=-1)
 					return retval
 
 			# Log the error after PORTAGE_LOG_FILE is initialized

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     77703d4f3660374068c275369670dcfe9c2f4131
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul  1 20:07:27 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul  1 20:07:27 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=77703d4f

_quickpkg_dblink: set PYTHONPATH for quickpkg

---
 pym/portage/dbapi/vartree.py |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index e5e1195..dd37be0 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4745,6 +4745,12 @@ class dblink(object):
 			env = dict(self.vartree.settings.items())
 			env["__PORTAGE_INHERIT_VARDB_LOCK"] = "1"
 
+			pythonpath = [x for x in env.get('PYTHONPATH', '').split(":") if x]
+			if not pythonpath or \
+				not os.path.samefile(pythonpath[0], portage._pym_path):
+				pythonpath.insert(0, portage._pym_path)
+			env['PYTHONPATH'] = ":".join(pythonpath)
+
 			quickpkg_proc = SpawnProcess(
 				args=[portage._python_interpreter, quickpkg_binary,
 					"=%s" % (backup_dblink.mycpv,)],

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e26ac28af7a6dc4cff2bf00737841d14dea669a5
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul  1 08:11:12 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul  1 08:11:12 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e26ac28a

Fix bugs in binarytree.invalids usage.

---
 pym/portage/dbapi/bintree.py |    4 +++-
 pym/portage/dbapi/vartree.py |    2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index fb31572..1048cc1 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1312,6 +1312,8 @@ class binarytree(object):
 		"""Returns the URI to the Packages file for a given package."""
 		return self._pkgindex_uri.get(pkgname)
 
+
+
 	def gettbz2(self, pkgname):
 		"""Fetches the package from a remote site, if necessary.  Attempts to
 		resume if the file appears to be partially downloaded."""
@@ -1319,7 +1321,7 @@ class binarytree(object):
 		tbz2name = os.path.basename(tbz2_path)
 		resume = False
 		if os.path.exists(tbz2_path):
-			if (tbz2name not in self.invalids):
+			if tbz2name[:-5] not in self.invalids:
 				return
 			else:
 				resume = True

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 0d7327a..e5e1195 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4727,7 +4727,7 @@ class dblink(object):
 		bintree = trees["bintree"]
 		binpkg_path = bintree.getname(backup_dblink.mycpv)
 		if os.path.exists(binpkg_path) and \
-			backup_dblink.mycpv not in bintree.invalids:
+			catsplit(backup_dblink.mycpv)[1] not in bintree.invalids:
 			return os.EX_OK
 
 		self.lockdb()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     324bf88228b7be64814e99dd96ad73a204a5df63
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 24 21:01:26 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun 24 21:01:26 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=324bf882

protected_symlinks: suggest course of action

See discussion in bug #423127.

---
 pym/portage/dbapi/vartree.py |    6 +++++-
 1 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 7866870..34098ea 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2370,7 +2370,11 @@ class dblink(object):
 		if protected_symlinks:
 			msg = "One or more symlinks to directories have been " + \
 				"preserved in order to ensure that files installed " + \
-				"via these symlinks remain accessible:"
+				"via these symlinks remain accessible. " + \
+				"This indicates that the mentioned symlink(s) may " + \
+				"be obsolete remnants of an old install, and it " + \
+				"may be appropriate to replace a given symlink " + \
+				"with the directory that it points to."
 			lines = textwrap.wrap(msg, 72)
 			lines.append("")
 			flat_list = set()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     542907e8e21b3a2a155d31095b63a1821354c1c7
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 24 19:16:34 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun 24 19:16:34 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=542907e8

protected_symlinks: change eerror to elog

Since bug #421165, this code will trigger more often (see bug #423127),
and eerror is too harsh because the symlinks will never be orphaned.

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 020240d..7866870 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2380,7 +2380,7 @@ class dblink(object):
 				lines.append("\t%s" % (os.path.join(real_root,
 					f.lstrip(os.sep))))
 			lines.append("")
-			self._elog("eerror", "postrm", lines)
+			self._elog("elog", "postrm", lines)
 
 		# Remove stale entries from config memory.
 		if stale_confmem:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e0fa8f389867d78438451ed4a8b1af1ea89b65b6
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 23 20:39:27 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun 23 20:39:27 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e0fa8f38

Skip unmerge of libdir symlinks for bug #423127.

---
 pym/portage/dbapi/vartree.py |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index e2b1b9b..020240d 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2181,6 +2181,12 @@ class dblink(object):
 						ignore = True
 						break
 
+				if not ignore:
+					if islink and f_match in \
+						("/lib", "/usr/lib", "/usr/local/lib"):
+						# Ignore libdir symlinks for bug #423127.
+						ignore = True
+
 				if ignore:
 					show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj)
 					continue

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f5eaf5de4dd342e939dbeda02c9b6e2c12dc1fa2
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 23 01:13:56 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun 23 01:13:56 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f5eaf5de

treewalk: fix USE=multislot breakage

It broke in commit e4ba8f36e6a4624f4fec61c7ce8bed0e3bd2fa01.

---
 pym/portage/dbapi/vartree.py |    9 ++++-----
 1 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 60bab73..e2b1b9b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3467,11 +3467,10 @@ class dblink(object):
 		if not os.path.exists(self.dbcatdir):
 			ensure_dirs(self.dbcatdir)
 
-		try:
-			slot = self.mycpv.slot
-		except AttributeError:
-			# discard the sub-slot if necesssary
-			slot = _pkg_str(self.mycpv, slot=slot).slot
+		# NOTE: We use SLOT obtained from the inforoot
+		#	directory, in order to support USE=multislot.
+		# Use _pkg_str discard the sub-slot part if necessary.
+		slot = _pkg_str(self.mycpv, slot=slot).slot
 		cp = self.mysplit[0]
 		slot_atom = "%s:%s" % (cp, slot)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0c29f1531fe9787cd1bd96fcd5c34ec5e6128aa4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 15 23:42:44 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun 15 23:42:44 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0c29f153

Unmerge symlink listed as dir for bug #421165.

---
 pym/portage/dbapi/vartree.py |    9 +++++----
 1 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index fbbf49d..5448bd4 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2180,8 +2180,9 @@ class dblink(object):
 							is_owned = True
 							break
 
-					if file_type == "sym" and is_owned and \
-						(islink and statobj and stat.S_ISDIR(statobj.st_mode)):
+					if is_owned and islink and \
+						file_type in ("sym", "dir") and \
+						statobj and stat.S_ISDIR(statobj.st_mode):
 						# A new instance of this package claims the file, so
 						# don't unmerge it. If the file is symlink to a
 						# directory and the unmerging package installed it as
@@ -2246,12 +2247,12 @@ class dblink(object):
 					show_unmerge("---", unmerge_desc["!mtime"], file_type, obj)
 					continue
 
-				if pkgfiles[objkey][0] == "dir":
+				if file_type == "dir" and not islink:
 					if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode):
 						show_unmerge("---", unmerge_desc["!dir"], file_type, obj)
 						continue
 					mydirs.add((obj, (lstatobj.st_dev, lstatobj.st_ino)))
-				elif pkgfiles[objkey][0] == "sym":
+				elif file_type == "sym" or (file_type == "dir" and islink):
 					if not islink:
 						show_unmerge("---", unmerge_desc["!sym"], file_type, obj)
 						continue

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     9a49a7c4189c55d2105ddf21b12321e7da3bf415
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jun 12 06:40:52 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jun 12 06:40:52 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9a49a7c4

dbapi: construct _pkg_str during match on-demand

---
 pym/portage/dbapi/__init__.py |   47 ++++++++++++++++++++++++++++++++--------
 pym/portage/dbapi/porttree.py |   17 ++++++++++----
 2 files changed, 49 insertions(+), 15 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index a90d59e..2ab7dc5 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -16,6 +16,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 
 from portage import os
 from portage import auxdbkeys
+from portage.exception import InvalidData
 from portage.localization import _
 
 class dbapi(object):
@@ -24,6 +25,8 @@ class dbapi(object):
 	_use_mutable = False
 	_known_keys = frozenset(x for x in auxdbkeys
 		if not x.startswith("UNUSED_0"))
+	_pkg_str_aux_keys = ("EAPI", "SLOT", "repository")
+
 	def __init__(self):
 		pass
 
@@ -125,29 +128,53 @@ class dbapi(object):
 
 	def _iter_match(self, atom, cpv_iter):
 		cpv_iter = iter(match_from_list(atom, cpv_iter))
+		if atom.repo:
+			cpv_iter = self._iter_match_repo(atom, cpv_iter)
 		if atom.slot:
 			cpv_iter = self._iter_match_slot(atom, cpv_iter)
 		if atom.unevaluated_atom.use:
 			cpv_iter = self._iter_match_use(atom, cpv_iter)
-		if atom.repo:
-			cpv_iter = self._iter_match_repo(atom, cpv_iter)
 		return cpv_iter
 
+	def _pkg_str(self, cpv, repo):
+		"""
+		This is used to contruct _pkg_str instances on-demand during
+		matching. If cpv is a _pkg_str instance with slot attribute,
+		then simply return it. Otherwise, fetch metadata and construct
+		a _pkg_str instance. This may raise KeyError or InvalidData.
+		"""
+		try:
+			cpv.slot
+		except AttributeError:
+			pass
+		else:
+			return cpv
+
+		metadata = dict(zip(self._pkg_str_aux_keys,
+			self.aux_get(cpv, self._pkg_str_aux_keys, myrepo=repo)))
+
+		return _pkg_str(cpv, slot=metadata["SLOT"],
+			repo=metadata["repository"], eapi=metadata["EAPI"])
+
 	def _iter_match_repo(self, atom, cpv_iter):
 		for cpv in cpv_iter:
 			try:
-				if self.aux_get(cpv, ["repository"], myrepo=atom.repo)[0] == atom.repo:
-					yield cpv
-			except KeyError:
-				continue
+				pkg_str = self._pkg_str(cpv, atom.repo)
+			except (KeyError, InvalidData):
+				pass
+			else:
+				if pkg_str.repo == atom.repo:
+					yield pkg_str
 
 	def _iter_match_slot(self, atom, cpv_iter):
 		for cpv in cpv_iter:
 			try:
-				if self.aux_get(cpv, ["SLOT"], myrepo=atom.repo)[0] == atom.slot:
-					yield cpv
-			except KeyError:
-				continue
+				pkg_str = self._pkg_str(cpv, atom.repo)
+			except (KeyError, InvalidData):
+				pass
+			else:
+				if pkg_str.slot == atom.slot:
+					yield pkg_str
 
 	def _iter_match_use(self, atom, cpv_iter):
 		"""

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index c5ee770..df68115 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -22,7 +22,8 @@ from portage.cache.cache_errors import CacheError
 from portage.cache.mappings import Mapping
 from portage.dbapi import dbapi
 from portage.exception import PortageException, \
-	FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName
+	FileNotFound, InvalidAtom, InvalidData, \
+	InvalidDependString, InvalidPackageName
 from portage.localization import _
 
 from portage import eclass_cache, \
@@ -825,18 +826,24 @@ class portdbapi(dbapi):
 						# ebuild not in this repo, or masked by corruption
 						continue
 
-					if visibility_filter and not self._visible(cpv, metadata):
+					try:
+						pkg_str = _pkg_str(cpv, slot=metadata["SLOT"],
+							repo=metadata["repository"], eapi=metadata["EAPI"])
+					except InvalidData:
+						continue
+
+					if visibility_filter and not self._visible(pkg_str, metadata):
 						continue
 
 					if mydep.slot is not None and \
-						mydep.slot != metadata["SLOT"]:
+						mydep.slot != pkg_str.slot:
 						continue
 
 					if mydep.unevaluated_atom.use is not None and \
-						not self._match_use(mydep, cpv, metadata):
+						not self._match_use(mydep, pkg_str, metadata):
 						continue
 
-					myval.append(cpv)
+					myval.append(pkg_str)
 					# only yield a given cpv once
 					break
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f2f4b2e3b6d7865ccef11a254055b5d213f4bac3
Author:     László Szalma <dblaci <AT> dblaci <DOT> hu>
AuthorDate: Thu May 17 20:07:52 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 17 20:07:52 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f2f4b2e3

merge: fix abs symlink adjustment, bug #416431

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 399bf9f..517c873 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4287,7 +4287,7 @@ class dblink(object):
 				myabsto = myabsto.lstrip(sep)
 				if self.settings and self.settings["D"]:
 					if myto.startswith(self.settings["D"]):
-						myto = myto[len(self.settings["D"]):]
+						myto = myto[len(self.settings["D"])-1:]
 				# myrealto contains the path of the real file to which this symlink points.
 				# we can simply test for existence of this file to see if the target has been merged yet
 				myrealto = normalize_path(os.path.join(destroot, myabsto))

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     999907a64249d9bfd2e42e477bfb9dd9a3cb535a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 14 23:56:20 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 14 23:56:20 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=999907a6

bintree: validate remove cpv more

---
 pym/portage/dbapi/bintree.py |   16 ++++++++++++++--
 1 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 51c0936..a8027ee 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -24,7 +24,7 @@ from portage.cache.mappings import slot_dict_class
 from portage.const import CACHE_PATH
 from portage.dbapi.virtual import fakedbapi
 from portage.dep import Atom, use_reduce, paren_enclose
-from portage.exception import AlarmSignal, InvalidPackageName, \
+from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
 	PermissionDenied, PortageException
 from portage.localization import _
 from portage import _movefile
@@ -219,6 +219,13 @@ def _pkgindex_cpv_map_latest_build(pkgindex):
 	for d in pkgindex.packages:
 		cpv = d["CPV"]
 
+		try:
+			cpv = _pkg_str(cpv)
+		except InvalidData:
+			writemsg(_("!!! Invalid remote binary package: %s\n") % cpv,
+				noiselevel=-1)
+			continue
+
 		btime = d.get('BUILD_TIME', '')
 		try:
 			btime = int(btime)
@@ -1003,7 +1010,12 @@ class binarytree(object):
 						noiselevel=-1)
 					continue
 				mycat = mycat.strip()
-				fullpkg = mycat+"/"+mypkg[:-5]
+				try:
+					fullpkg = _pkg_str(mycat+"/"+mypkg[:-5])
+				except InvalidData:
+					writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg,
+						noiselevel=-1)
+					continue
 
 				if fullpkg in metadata:
 					# When using this old protocol, comparison with the remote

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a1196c12131c43e8d09005f288bbd71d1df770e0
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 14 00:46:44 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 14 00:46:44 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1196c12

portdbapi: don't convert _pkg_str to str

---
 pym/portage/dbapi/porttree.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 663b62d..c5ee770 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -869,7 +869,8 @@ class portdbapi(dbapi):
 			xcache_this_level = self.xcache.get(level)
 			if xcache_this_level is not None:
 				xcache_this_level[cache_key] = myval
-				myval = myval[:]
+				if not isinstance(myval, _pkg_str):
+					myval = myval[:]
 
 		return myval
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     640207f6083ffd3be9a3ad01ad85769974db7165
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 14 00:29:36 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 14 00:29:36 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=640207f6

dblink: use _pkg_str + vercmp, not pkgcmp

---
 pym/portage/dbapi/vartree.py |   14 +++++++++-----
 1 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index fe352c5..399bf9f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -30,7 +30,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util.movefile:movefile',
 	'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry',
 	'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap',
-	'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,pkgcmp,' + \
+	'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \
 		'_pkgsplit@pkgsplit,_pkg_str',
 	'subprocess',
 	'tarfile',
@@ -1436,8 +1436,13 @@ class dblink(object):
 		self.cat = cat
 		self.pkg = pkg
 		self.mycpv = self.cat + "/" + self.pkg
-		self.mysplit = list(catpkgsplit(self.mycpv)[1:])
-		self.mysplit[0] = "%s/%s" % (self.cat, self.mysplit[0])
+		if self.mycpv == settings.mycpv and \
+			isinstance(settings.mycpv, _pkg_str):
+			self.mycpv = settings.mycpv
+		else:
+			self.mycpv = _pkg_str(self.mycpv)
+		self.mysplit = list(self.mycpv.cpv_split[1:])
+		self.mysplit[0] = self.mycpv.cp
 		self.treetype = treetype
 		if vartree is None:
 			vartree = portage.db[self._eroot]["vartree"]
@@ -3851,9 +3856,8 @@ class dblink(object):
 			# Always behave like --noconfmem is enabled for downgrades
 			# so that people who don't know about this option are less
 			# likely to get confused when doing upgrade/downgrade cycles.
-			pv_split = catpkgsplit(self.mycpv)[1:]
 			for other in others_in_slot:
-				if pkgcmp(pv_split, catpkgsplit(other.mycpv)[1:]) < 0:
+				if vercmp(self.mycpv.version, other.mycpv.version) < 0:
 					cfgfiledict["IGNORE"] = 1
 					break
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     b59bd3348d709aa53c6883006313bdbdc9c737ea
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May  9 22:28:54 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May  9 22:28:54 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b59bd334

porttree.py: remove unused imports

---
 pym/portage/dbapi/porttree.py |    2 --
 1 files changed, 0 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 16962b5..2bf17e1 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -36,8 +36,6 @@ from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
 from _emerge.PollScheduler import PollScheduler
 
 import os as _os
-import io
-import stat
 import sys
 import traceback
 import warnings

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d448e1f1844fe1fe8b865f8f9080206b84c6f4e3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May  5 16:54:14 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May  5 16:54:14 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d448e1f1

COLLISION_IGNORE: handle prefix

---
 pym/portage/dbapi/vartree.py |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 71b5d80..ee77fac 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3183,17 +3183,18 @@ class dblink(object):
 				if not isowned and self.isprotected(full_path):
 					isowned = True
 				if not isowned:
+					f_match = full_path[len(self._eroot)-1:]
 					stopmerge = True
 					if collision_ignore:
-						if f in collision_ignore:
+						if f_match in collision_ignore:
 							stopmerge = False
 						else:
 							for myignore in collision_ignore:
-								if f.startswith(myignore + os.path.sep):
+								if f_match.startswith(myignore + os.path.sep):
 									stopmerge = False
 									break
 					for pattern in unowned_ignore_patterns:
-						if fnmatch.fnmatch(f, pattern):
+						if fnmatch.fnmatch(f_match, pattern):
 							stopmerge = False
 							break
 					if stopmerge:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e4ac7ed0d47b68b95da7854881a29e40d77f6e2f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 22 20:34:18 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 22 20:34:18 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4ac7ed0

fakedbapi.cp_list: remove obsolete virtual code

This should have been included in commit
d30db49bcaecbdeb20f6d4e180979b60b6260f69.

---
 pym/portage/dbapi/virtual.py |    3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py
index 8a35d0c..84b6b93 100644
--- a/pym/portage/dbapi/virtual.py
+++ b/pym/portage/dbapi/virtual.py
@@ -49,8 +49,7 @@ class fakedbapi(dbapi):
 		# match cache uses the result from dep_expand for the cache_key.
 		cache_key = (mycp, mycp)
 		cachelist = self._match_cache.get(cache_key)
-		# cp_list() doesn't expand old-style virtuals
-		if cachelist and cachelist[0].startswith(mycp):
+		if cachelist is not None:
 			return cachelist[:]
 		cpv_list = self.cpdict.get(mycp)
 		if cpv_list is None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5bd8b8a41326d16d4e8347bcddfee0d4ff8f6bd3
Author:     Greg Turner <gmturner007 <AT> ameritech <DOT> net>
AuthorDate: Wed Apr 18 01:48:32 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Apr 18 01:48:32 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5bd8b8a4

mergeme: cleanup hardlink_candidates, bug #412453

---
 pym/portage/dbapi/vartree.py |    9 ++++-----
 1 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 25ea4c1..a3a6c76 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1452,7 +1452,7 @@ class dblink(object):
 		self._contents_inodes = None
 		self._contents_basenames = None
 		self._linkmap_broken = False
-		self._md5_merge_map = {}
+		self._hardlink_merge_map = {}
 		self._hash_key = (self._eroot, self.mycpv)
 		self._protect_obj = None
 		self._pipe = pipe
@@ -4500,10 +4500,10 @@ class dblink(object):
 					# as hardlinks (having identical st_dev and st_ino).
 					hardlink_key = (mystat.st_dev, mystat.st_ino)
 
-					hardlink_candidates = self._md5_merge_map.get(hardlink_key)
+					hardlink_candidates = self._hardlink_merge_map.get(hardlink_key)
 					if hardlink_candidates is None:
 						hardlink_candidates = []
-						self._md5_merge_map[hardlink_key] = hardlink_candidates
+						self._hardlink_merge_map[hardlink_key] = hardlink_candidates
 
 					mymtime = movefile(mysrc, mydest, newmtime=thismtime,
 						sstat=mystat, mysettings=self.settings,
@@ -4511,8 +4511,7 @@ class dblink(object):
 						encoding=_encodings['merge'])
 					if mymtime is None:
 						return 1
-					if hardlink_candidates is not None:
-						hardlink_candidates.append(mydest)
+					hardlink_candidates.append(mydest)
 					zing = ">>>"
 
 				if mymtime != None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7c58705c30fed6b11f978c4880225ced8a45c70d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 23 17:28:21 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 23 17:28:21 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7c58705c

Clarify symlink/dir collision for bug #409359.

---
 pym/portage/dbapi/vartree.py |   31 ++++++++++++++++++++++++-------
 1 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 0bcc947..3e603e2 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3661,6 +3661,21 @@ class dblink(object):
 			self._collision_protect(srcroot, destroot,
 			others_in_slot + blockers, myfilelist, mylinklist)
 
+		if symlink_collisions:
+			# Symlink collisions need to be distinguished from other types
+			# of collisions, in order to avoid confusion (see bug #409359).
+			msg = _("Package '%s' has one or more collisions "
+				"between symlinks and directories, which is explicitly "
+				"forbidden by PMS section 13.4 (see bug #326685):") % \
+				(self.settings.mycpv,)
+			msg = textwrap.wrap(msg, 70)
+			msg.append("")
+			for f in symlink_collisions:
+				msg.append("\t%s" % os.path.join(destroot,
+					f.lstrip(os.path.sep)))
+			msg.append("")
+			self._elog("eerror", "preinst", msg)
+
 		if collisions:
 			collision_protect = "collision-protect" in self.settings.features
 			protect_owned = "protect-owned" in self.settings.features
@@ -3742,12 +3757,20 @@ class dblink(object):
 					eerror([_("None of the installed"
 						" packages claim the file(s)."), ""])
 
+			symlink_abort_msg =_("Package '%s' NOT merged since it has "
+				"one or more collisions between symlinks and directories, "
+				"which is explicitly forbidden by PMS section 13.4 "
+				"(see bug #326685).")
+
 			# The explanation about the collision and how to solve
 			# it may not be visible via a scrollback buffer, especially
 			# if the number of file collisions is large. Therefore,
 			# show a summary at the end.
 			abort = False
-			if collision_protect:
+			if symlink_collisions:
+				abort = True
+				msg = symlink_abort_msg % (self.settings.mycpv,)
+			elif collision_protect:
 				abort = True
 				msg = _("Package '%s' NOT merged due to file collisions.") % \
 					self.settings.mycpv
@@ -3755,12 +3778,6 @@ class dblink(object):
 				abort = True
 				msg = _("Package '%s' NOT merged due to file collisions.") % \
 					self.settings.mycpv
-			elif symlink_collisions:
-				abort = True
-				msg = _("Package '%s' NOT merged due to collision " + \
-				"between a symlink and a directory which is explicitly " + \
-				"forbidden by PMS (see bug #326685).") % \
-				(self.settings.mycpv,)
 			else:
 				msg = _("Package '%s' merged despite file collisions.") % \
 					self.settings.mycpv

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     aa7a0a91390accd84c04302e1e738f945aee333a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 18 22:39:40 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar 18 22:39:40 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=aa7a0a91

unmerge: remove unused preserve-libs

This fixes a case were one or more packages are unmerged without being
replaced, and some preserved libraries need to be removed because
they all consumers have just been unmerged.

---
 pym/portage/dbapi/vartree.py |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 4811e60..0bcc947 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1966,6 +1966,15 @@ class dblink(object):
 			contents=contents, env=self.settings,
 			writemsg_level=self._display_merge, vardbapi=self.vartree.dbapi)
 
+		unmerge_with_replacement = preserve_paths is not None
+		if not unmerge_with_replacement:
+			# When there's a replacement package which calls us via treewalk,
+			# treewalk will automatically call _prune_plib_registry for us.
+			# Otherwise, we need to call _prune_plib_registry ourselves.
+			# Don't pass in the "unmerge=True" flag here, since that flag
+			# is intended to be used _prior_ to unmerge, not after.
+			self._prune_plib_registry()
+
 		return os.EX_OK
 
 	def _display_merge(self, msg, level=0, noiselevel=0):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     411a8461d08b88c5636e900d6297c2b4873af549
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 23 05:30:19 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 23 05:30:19 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=411a8461

Unmerge orphan directory symlinks for bug 384397.

---
 pym/portage/dbapi/vartree.py |   88 ++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 4b1ede1..6d354af 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2027,9 +2027,6 @@ class dblink(object):
 					settings=self.settings,
 					vartree=self.vartree, treetype="vartree", pipe=self._pipe))
 
-		dest_root = self._eroot
-		dest_root_len = len(dest_root) - 1
-
 		cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file)
 		stale_confmem = []
 		protected_symlinks = {}
@@ -2333,6 +2330,11 @@ class dblink(object):
 
 			self._unmerge_dirs(mydirs, infodirs_inodes,
 				protected_symlinks, unmerge_desc, unlink, os)
+			mydirs.clear()
+
+		if protected_symlinks:
+			self._unmerge_protected_symlinks(others_in_slot, infodirs_inodes,
+				protected_symlinks, unmerge_desc, unlink, os)
 
 		if protected_symlinks:
 			msg = "One or more symlinks to directories have been " + \
@@ -2358,6 +2360,86 @@ class dblink(object):
 		#remove self from vartree database so that our own virtual gets zapped if we're the last node
 		self.vartree.zap(self.mycpv)
 
+	def _unmerge_protected_symlinks(self, others_in_slot, infodirs_inodes,
+		protected_symlinks, unmerge_desc, unlink, os):
+
+		real_root = self.settings['ROOT']
+		show_unmerge = self._show_unmerge
+		ignored_unlink_errnos = self._ignored_unlink_errnos
+
+		flat_list = set()
+		flat_list.update(*protected_symlinks.values())
+		flat_list = sorted(flat_list)
+
+		for f in flat_list:
+			for dblnk in others_in_slot:
+				if dblnk.isowner(f):
+					# If another package in the same slot installed
+					# a file via a protected symlink, return early
+					# and don't bother searching for any other owners.
+					return
+
+		msg = []
+		msg.append("")
+		msg.append(_("Directory symlink(s) may need protection:"))
+		msg.append("")
+
+		for f in flat_list:
+			msg.append("\t%s" % \
+				os.path.join(real_root, f.lstrip(os.path.sep)))
+
+		msg.append("")
+		msg.append(_("Searching all installed"
+			" packages for files installed via above symlink(s)..."))
+		msg.append("")
+		self._elog("elog", "postrm", msg)
+
+		self.lockdb()
+		try:
+			owners = self.vartree.dbapi._owners.get_owners(flat_list)
+			self.vartree.dbapi.flush_cache()
+		finally:
+			self.unlockdb()
+
+		for owner in list(owners):
+			if owner.mycpv == self.mycpv:
+				owners.pop(owner, None)
+
+		if not owners:
+			msg = []
+			msg.append(_("The above directory symlink(s) are all "
+				"safe to remove. Removing them now..."))
+			msg.append("")
+			self._elog("elog", "postrm", msg)
+			dirs = set()
+			for unmerge_syms in protected_symlinks.values():
+				for relative_path in unmerge_syms:
+					obj = os.path.join(real_root,
+						relative_path.lstrip(os.sep))
+					parent = os.path.dirname(obj)
+					while len(parent) > len(self._eroot):
+						try:
+							lstatobj = os.lstat(parent)
+						except OSError:
+							break
+						else:
+							dirs.add((parent,
+								(lstatobj.st_dev, lstatobj.st_ino)))
+							parent = os.path.dirname(parent)
+					try:
+						unlink(obj, os.lstat(obj))
+						show_unmerge("<<<", "", "sym", obj)
+					except (OSError, IOError) as e:
+						if e.errno not in ignored_unlink_errnos:
+							raise
+						del e
+						show_unmerge("!!!", "", "sym", obj)
+
+			protected_symlinks.clear()
+			self._unmerge_dirs(dirs, infodirs_inodes,
+				protected_symlinks, unmerge_desc, unlink, os)
+			dirs.clear()
+
 	def _unmerge_dirs(self, dirs, infodirs_inodes,
 		protected_symlinks, unmerge_desc, unlink, os):
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0839bb5891f5dfee6e7930de9a2e044859734c04
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 23 03:07:11 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 23 03:07:11 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0839bb58

dblink: split out _unmerge_dirs method

This code will need to be called twice when safely unmerging symlinks
to directories (bug #384397).

---
 pym/portage/dbapi/vartree.py |  194 +++++++++++++++++++++++-------------------
 1 files changed, 106 insertions(+), 88 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 96c2530..4b1ede1 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2011 Gentoo Foundation
+# Copyright 1998-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = [
@@ -1388,6 +1388,20 @@ class dblink(object):
 		r')$'
 	)
 
+	# These files are generated by emerge, so we need to remove
+	# them when they are the only thing left in a directory.
+	_infodir_cleanup = frozenset(["dir", "dir.old"])
+
+	_ignored_unlink_errnos = (
+		errno.EBUSY, errno.ENOENT,
+		errno.ENOTDIR, errno.EISDIR)
+
+	_ignored_rmdir_errnos = (
+		errno.EEXIST, errno.ENOTEMPTY,
+		errno.EBUSY, errno.ENOENT,
+		errno.ENOTDIR, errno.EISDIR,
+		errno.EPERM)
+
 	def __init__(self, cat, pkg, myroot=None, settings=None, treetype=None,
 		vartree=None, blockers=None, scheduler=None, pipe=None):
 		"""
@@ -1973,6 +1987,10 @@ class dblink(object):
 					log_path=log_path, background=background,
 					level=level, noiselevel=noiselevel)
 
+	def _show_unmerge(self, zing, desc, file_type, file_name):
+		self._display_merge("%s %s %s %s\n" % \
+			(zing, desc.ljust(8), file_type, file_name))
+
 	def _unmerge_pkgfiles(self, pkgfiles, others_in_slot):
 		"""
 		
@@ -1989,6 +2007,9 @@ class dblink(object):
 		os = _os_merge
 		perf_md5 = perform_md5
 		showMessage = self._display_merge
+		show_unmerge = self._show_unmerge
+		ignored_unlink_errnos = self._ignored_unlink_errnos
+		ignored_rmdir_errnos = self._ignored_rmdir_errnos
 
 		if not pkgfiles:
 			showMessage(_("No package files given... Grabbing a set.\n"))
@@ -2024,14 +2045,6 @@ class dblink(object):
 
 			#process symlinks second-to-last, directories last.
 			mydirs = set()
-			ignored_unlink_errnos = (
-				errno.EBUSY, errno.ENOENT,
-				errno.ENOTDIR, errno.EISDIR)
-			ignored_rmdir_errnos = (
-				errno.EEXIST, errno.ENOTEMPTY,
-				errno.EBUSY, errno.ENOENT,
-				errno.ENOTDIR, errno.EISDIR,
-				errno.EPERM)
 			modprotect = os.path.join(self._eroot, "lib/modules/")
 
 			def unlink(file_name, lstatobj):
@@ -2067,10 +2080,6 @@ class dblink(object):
 						# Restore the parent flags we saved before unlinking
 						bsd_chflags.chflags(parent_name, pflags)
 
-			def show_unmerge(zing, desc, file_type, file_name):
-					showMessage("%s %s %s %s\n" % \
-						(zing, desc.ljust(8), file_type, file_name))
-
 			unmerge_desc = {}
 			unmerge_desc["cfgpro"] = _("cfgpro")
 			unmerge_desc["replaced"] = _("replaced")
@@ -2088,9 +2097,6 @@ class dblink(object):
 			real_root_len = len(real_root) - 1
 			eroot = self.settings["EROOT"]
 
-			# These files are generated by emerge, so we need to remove
-			# them when they are the only thing left in a directory.
-			infodir_cleanup = frozenset(["dir", "dir.old"])
 			infodirs = frozenset(infodir for infodir in chain(
 				self.settings.get("INFOPATH", "").split(":"),
 				self.settings.get("INFODIR", "").split(":")) if infodir)
@@ -2325,78 +2331,8 @@ class dblink(object):
 				elif pkgfiles[objkey][0] == "dev":
 					show_unmerge("---", "", file_type, obj)
 
-			mydirs = sorted(mydirs)
-			mydirs.reverse()
-
-			for obj, inode_key in mydirs:
-				# Treat any directory named "info" as a candidate here,
-				# since it might have been in INFOPATH previously even
-				# though it may not be there now.
-				if inode_key in infodirs_inodes or \
-					os.path.basename(obj) == "info":
-					try:
-						remaining = os.listdir(obj)
-					except OSError:
-						pass
-					else:
-						cleanup_info_dir = ()
-						if remaining and \
-							len(remaining) <= len(infodir_cleanup):
-							if not set(remaining).difference(infodir_cleanup):
-								cleanup_info_dir = remaining
-
-						for child in cleanup_info_dir:
-							child = os.path.join(obj, child)
-							try:
-								lstatobj = os.lstat(child)
-								if stat.S_ISREG(lstatobj.st_mode):
-									unlink(child, lstatobj)
-									show_unmerge("<<<", "", "obj", child)
-							except EnvironmentError as e:
-								if e.errno not in ignored_unlink_errnos:
-									raise
-								del e
-								show_unmerge("!!!", "", "obj", child)
-				try:
-					if bsd_chflags:
-						lstatobj = os.lstat(obj)
-						if lstatobj.st_flags != 0:
-							bsd_chflags.lchflags(obj, 0)
-						parent_name = os.path.dirname(obj)
-						# Use normal stat/chflags for the parent since we want to
-						# follow any symlinks to the real parent directory.
-						pflags = os.stat(parent_name).st_flags
-						if pflags != 0:
-							bsd_chflags.chflags(parent_name, 0)
-					try:
-						os.rmdir(obj)
-					finally:
-						if bsd_chflags and pflags != 0:
-							# Restore the parent flags we saved before unlinking
-							bsd_chflags.chflags(parent_name, pflags)
-					show_unmerge("<<<", "", "dir", obj)
-				except EnvironmentError as e:
-					if e.errno not in ignored_rmdir_errnos:
-						raise
-					if e.errno != errno.ENOENT:
-						show_unmerge("---", unmerge_desc["!empty"], "dir", obj)
-					del e
-				else:
-					# When a directory is successfully removed, there's
-					# no need to protect symlinks that point to it.
-					unmerge_syms = protected_symlinks.pop(inode_key, None)
-					if unmerge_syms is not None:
-						for relative_path in unmerge_syms:
-							obj = os.path.join(real_root,
-								relative_path.lstrip(os.sep))
-							try:
-								unlink(obj, os.lstat(obj))
-								show_unmerge("<<<", "", "sym", obj)
-							except (OSError, IOError) as e:
-								if e.errno not in ignored_unlink_errnos:
-									raise
-								del e
-								show_unmerge("!!!", "", "sym", obj)
+			self._unmerge_dirs(mydirs, infodirs_inodes,
+				protected_symlinks, unmerge_desc, unlink, os)
 
 		if protected_symlinks:
 			msg = "One or more symlinks to directories have been " + \
@@ -2422,6 +2358,88 @@ class dblink(object):
 		#remove self from vartree database so that our own virtual gets zapped if we're the last node
 		self.vartree.zap(self.mycpv)
 
+	def _unmerge_dirs(self, dirs, infodirs_inodes,
+		protected_symlinks, unmerge_desc, unlink, os):
+
+		show_unmerge = self._show_unmerge
+		infodir_cleanup = self._infodir_cleanup
+		ignored_unlink_errnos = self._ignored_unlink_errnos
+		ignored_rmdir_errnos = self._ignored_rmdir_errnos
+		real_root = self.settings['ROOT']
+
+		dirs = sorted(dirs)
+		dirs.reverse()
+
+		for obj, inode_key in dirs:
+			# Treat any directory named "info" as a candidate here,
+			# since it might have been in INFOPATH previously even
+			# though it may not be there now.
+			if inode_key in infodirs_inodes or \
+				os.path.basename(obj) == "info":
+				try:
+					remaining = os.listdir(obj)
+				except OSError:
+					pass
+				else:
+					cleanup_info_dir = ()
+					if remaining and \
+						len(remaining) <= len(infodir_cleanup):
+						if not set(remaining).difference(infodir_cleanup):
+							cleanup_info_dir = remaining
+
+					for child in cleanup_info_dir:
+						child = os.path.join(obj, child)
+						try:
+							lstatobj = os.lstat(child)
+							if stat.S_ISREG(lstatobj.st_mode):
+								unlink(child, lstatobj)
+								show_unmerge("<<<", "", "obj", child)
+						except EnvironmentError as e:
+							if e.errno not in ignored_unlink_errnos:
+								raise
+							del e
+							show_unmerge("!!!", "", "obj", child)
+			try:
+				if bsd_chflags:
+					lstatobj = os.lstat(obj)
+					if lstatobj.st_flags != 0:
+						bsd_chflags.lchflags(obj, 0)
+					parent_name = os.path.dirname(obj)
+					# Use normal stat/chflags for the parent since we want to
+					# follow any symlinks to the real parent directory.
+					pflags = os.stat(parent_name).st_flags
+					if pflags != 0:
+						bsd_chflags.chflags(parent_name, 0)
+				try:
+					os.rmdir(obj)
+				finally:
+					if bsd_chflags and pflags != 0:
+						# Restore the parent flags we saved before unlinking
+						bsd_chflags.chflags(parent_name, pflags)
+				show_unmerge("<<<", "", "dir", obj)
+			except EnvironmentError as e:
+				if e.errno not in ignored_rmdir_errnos:
+					raise
+				if e.errno != errno.ENOENT:
+					show_unmerge("---", unmerge_desc["!empty"], "dir", obj)
+				del e
+			else:
+				# When a directory is successfully removed, there's
+				# no need to protect symlinks that point to it.
+				unmerge_syms = protected_symlinks.pop(inode_key, None)
+				if unmerge_syms is not None:
+					for relative_path in unmerge_syms:
+						obj = os.path.join(real_root,
+							relative_path.lstrip(os.sep))
+						try:
+							unlink(obj, os.lstat(obj))
+							show_unmerge("<<<", "", "sym", obj)
+						except (OSError, IOError) as e:
+							if e.errno not in ignored_unlink_errnos:
+								raise
+							del e
+							show_unmerge("!!!", "", "sym", obj)
+
 	def isowner(self, filename, destroot=None):
 		""" 
 		Check if a file belongs to this package. This may

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     b39ff144bd47952ee06bba0b549069c635d31a31
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 15 22:28:41 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 15 22:28:41 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b39ff144

MergeProcess: inherit stdin for use with pdb

---
 pym/portage/dbapi/_MergeProcess.py |    7 +++++++
 1 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 1747a09..21d43dd 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -4,6 +4,7 @@
 import io
 import platform
 import signal
+import sys
 import traceback
 
 import errno
@@ -40,6 +41,12 @@ class MergeProcess(SpawnProcess):
 			settings.reset()
 			settings.setcpv(cpv, mydb=self.mydbapi)
 
+		# Inherit stdin by default, so that the pdb SIGUSR1
+		# handler is usable for the subprocess.
+		if self.fd_pipes is None:
+			self.fd_pipes = {}
+		self.fd_pipes.setdefault(0, sys.stdin.fileno())
+
 		super(MergeProcess, self)._start()
 
 	def _lock_vdb(self):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     fb89e5a829dfe0083b921c8c3958a3341d055cf8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 15 22:17:38 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 15 22:17:38 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fb89e5a8

treewalk: handle EAGAIN from listdir for PyPy 1.8

---
 pym/portage/dbapi/vartree.py |   17 +++++++++++++++--
 1 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a86170c..96c2530 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3402,6 +3402,7 @@ class dblink(object):
 		while True:
 
 			unicode_error = False
+			eagain_error = False
 
 			myfilelist = []
 			mylinklist = []
@@ -3409,7 +3410,19 @@ class dblink(object):
 			srcroot_len = len(srcroot)
 			def onerror(e):
 				raise
-			for parent, dirs, files in os.walk(srcroot, onerror=onerror):
+			walk_iter = os.walk(srcroot, onerror=onerror)
+			while True:
+				try:
+					parent, dirs, files = next(walk_iter)
+				except StopIteration:
+					break
+				except OSError as e:
+					if e.errno != errno.EAGAIN:
+						raise
+					# Observed with PyPy 1.8.
+					eagain_error = True
+					break
+
 				try:
 					parent = _unicode_decode(parent,
 						encoding=_encodings['merge'], errors='strict')
@@ -3464,7 +3477,7 @@ class dblink(object):
 				if unicode_error:
 					break
 
-			if not unicode_error:
+			if not (unicode_error or eagain_error):
 				break
 
 		if unicode_errors:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a7071bd82676ffdf3020b2f71effa2512ed803d4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 11 18:46:00 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 11 18:46:00 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a7071bd8

Fix FEATURES=no{doc,info,man} for bug #403181

This has been broken since commit
93b654199a32fc3df1299b030317720b9294b0c3.

---
 pym/portage/dbapi/vartree.py |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c504e5c..a86170c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3381,7 +3381,10 @@ class dblink(object):
 					max_dblnk = dblnk
 			self._installed_instance = max_dblnk
 
-		if self.settings.get("INSTALL_MASK"):
+		if self.settings.get("INSTALL_MASK") or \
+			"nodoc" in self.settings.features or \
+			"noinfo" in self.settings.features or \
+			"noman" in self.settings.features:
 			# Apply INSTALL_MASK before collision-protect, since it may
 			# be useful to avoid collisions in some scenarios.
 			phase = MiscFunctionsProcess(background=False,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7092cf7fcbd1c51bd739110baaea9da011c5ebf5
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb  9 05:17:18 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb  9 05:17:18 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7092cf7f

MergeProcess: handle POLLHUP from elog pipe

---
 pym/portage/dbapi/_MergeProcess.py |    7 +++++++
 1 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index eed7bd4..da278ea 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -83,6 +83,13 @@ class MergeProcess(SpawnProcess):
 					reporter = getattr(portage.elog.messages, funcname)
 					reporter(msg, phase=phase, key=key, out=out)
 
+		if event & PollConstants.POLLHUP:
+			self.scheduler.unregister(self._elog_reg_id)
+			self._elog_reg_id = None
+			os.close(self._elog_reader_fd)
+			self._elog_reader_fd = None
+			return False
+
 		return True
 
 	def _spawn(self, args, fd_pipes, **kwargs):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e3f2b5a95fbe028d32aca8e5b268237a87260917
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb  9 05:04:16 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb  9 05:04:16 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e3f2b5a9

MergeProcess: handle POLLHUP from elog pipe

---
 pym/portage/dbapi/_MergeProcess.py |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index eed7bd4..d03049a 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -83,6 +83,8 @@ class MergeProcess(SpawnProcess):
 					reporter = getattr(portage.elog.messages, funcname)
 					reporter(msg, phase=phase, key=key, out=out)
 
+		self._unregister_if_appropriate(event)
+
 		return True
 
 	def _spawn(self, args, fd_pipes, **kwargs):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     02522e5755ab369c2d9a9f712bb0c1e7b882ad98
Author:     Brian Harring <ferringb <AT> chromium <DOT> org>
AuthorDate: Fri Jan 27 18:23:02 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jan 27 22:01:34 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=02522e57

Apply INSTALL_MASK prior to reading the tree

If this isn't done, collision-protect uses the old cached filelist,
leading to it complaining about things that INSTALL_MASK sorted already.

BUGS=chromium-os:25060
TEST=emerge sudo && emerge chromeos-base

---
 pym/portage/dbapi/vartree.py |   18 +++++++++---------
 1 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 5bb4fa4..c504e5c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3381,6 +3381,15 @@ class dblink(object):
 					max_dblnk = dblnk
 			self._installed_instance = max_dblnk
 
+		if self.settings.get("INSTALL_MASK"):
+			# Apply INSTALL_MASK before collision-protect, since it may
+			# be useful to avoid collisions in some scenarios.
+			phase = MiscFunctionsProcess(background=False,
+				commands=["preinst_mask"], phase="preinst",
+				scheduler=self._scheduler, settings=self.settings)
+			phase.start()
+			phase.wait()
+
 		# We check for unicode encoding issues after src_install. However,
 		# the check must be repeated here for binary packages (it's
 		# inexpensive since we call os.walk() here anyway).
@@ -3517,15 +3526,6 @@ class dblink(object):
 			for other in others_in_slot])
 		prepare_build_dirs(settings=self.settings, cleanup=cleanup)
 
-		if self.settings.get("INSTALL_MASK"):
-			# Apply INSTALL_MASK before collision-protect, since it may
-			# be useful to avoid collisions in some scenarios.
-			phase = MiscFunctionsProcess(background=False,
-				commands=["preinst_mask"], phase="preinst",
-				scheduler=self._scheduler, settings=self.settings)
-			phase.start()
-			phase.wait()
-
 		# check for package collisions
 		blockers = self._blockers
 		if blockers is None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0dd5593f6754f92657c8c7331f66f812d10eeb44
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 11 16:02:26 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jan 11 16:02:26 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0dd5593f

quickpkg: add leading ./ to names for bug 398543

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a66316b..af70ec3 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4629,7 +4629,7 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None):
 			continue
 		contents_type = contents[path][0]
 		if path.startswith(root):
-			arcname = path[len(root):]
+			arcname = "./" + path[len(root):]
 		else:
 			raise ValueError("invalid root argument: '%s'" % root)
 		live_path = path

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     08e4be7fa43c99b9acbfe30982d51ef6f881c000
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 19:43:11 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 19:43:11 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=08e4be7f

_aux_env_search: decode unicode more

---
 pym/portage/dbapi/vartree.py |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index d93d3c2..b9ef583 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -798,6 +798,9 @@ class vardbapi(dbapi):
 					else:
 						value = [var_assign_match.group(4)]
 						for line in proc.stdout:
+							line = _unicode_decode(line,
+								encoding=_encodings['content'],
+								errors='replace')
 							value.append(line)
 							if have_end_quote(quote, line):
 								break

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     6b5ffa8fc0d01647d123fe62686c724f3c832b20
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 00:32:48 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 00:32:48 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6b5ffa8f

vardbapi._aux_get: unicode EAPI key

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c2e90d8..d93d3c2 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -743,7 +743,7 @@ class vardbapi(dbapi):
 				results[k] = v
 
 		if results.get("EAPI") == "":
-			results["EAPI"] = _unicode_decode('0')
+			results[_unicode_decode("EAPI")] = _unicode_decode('0')
 
 		return results
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e8ea1bcb31db5929e19dac0392922994a9577bdd
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 00:29:30 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 00:29:30 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e8ea1bcb

vardbapi._aux_get: search environment in one pass

---
 pym/portage/dbapi/vartree.py |   96 +++++++++++++++++++++++-------------------
 1 files changed, 53 insertions(+), 43 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 7184550..c2e90d8 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -625,7 +625,8 @@ class vardbapi(dbapi):
 				cache_these_wants.add(x)
 
 		if not cache_these_wants:
-			return self._aux_get(mycpv, wants)
+			mydata = self._aux_get(mycpv, wants)
+			return [mydata[x] for x in wants]
 
 		cache_these = set(self._aux_cache_keys)
 		cache_these.update(cache_these_wants)
@@ -670,9 +671,7 @@ class vardbapi(dbapi):
 		if pull_me:
 			# pull any needed data and cache it
 			aux_keys = list(pull_me)
-			for k, v in zip(aux_keys,
-				self._aux_get(mycpv, aux_keys, st=mydir_stat)):
-				mydata[k] = v
+			mydata.update(self._aux_get(mycpv, aux_keys, st=mydir_stat))
 			if not cache_valid or cache_these.difference(metadata):
 				cache_data = {}
 				if cache_valid and metadata:
@@ -703,10 +702,11 @@ class vardbapi(dbapi):
 					raise
 		if not stat.S_ISDIR(st.st_mode):
 			raise KeyError(mycpv)
-		results = []
+		results = {}
+		env_keys = []
 		for x in wants:
 			if x == "_mtime_":
-				results.append(st[stat.ST_MTIME])
+				results[x] = st[stat.ST_MTIME]
 				continue
 			try:
 				myf = io.open(
@@ -719,35 +719,46 @@ class vardbapi(dbapi):
 				finally:
 					myf.close()
 			except IOError:
-				myd = None
 				if x not in self._aux_cache_keys and \
 					self._aux_cache_keys_re.match(x) is None:
-					myd = self._aux_env_search(mycpv, x)
-				if myd is None:
-					myd = _unicode_decode('')
+					env_keys.append(x)
+					continue
+				myd = _unicode_decode('')
 
 			# Preserve \n for metadata that is known to
 			# contain multiple lines.
 			if self._aux_multi_line_re.match(x) is None:
 				myd = " ".join(myd.split())
 
-			if x == "EAPI" and not myd:
-				results.append(_unicode_decode('0'))
-			else:
-				results.append(myd)
+			results[x] = myd
+
+		if env_keys:
+			env_results = self._aux_env_search(mycpv, env_keys)
+			for k in env_keys:
+				v = env_results.get(k)
+				if v is None:
+					v = _unicode_decode('')
+				if self._aux_multi_line_re.match(k) is None:
+					v = " ".join(v.split())
+				results[k] = v
+
+		if results.get("EAPI") == "":
+			results["EAPI"] = _unicode_decode('0')
+
 		return results
 
-	def _aux_env_search(self, cpv, variable):
+	def _aux_env_search(self, cpv, variables):
 		"""
-		Search environment.bz2 of the specified variable. Returns
-		the value if found, otherwise None. This is useful for
-		querying variables like ${SRC_URI} and ${A}, which are not
-		saved in separate files but are available in environment.bz2
-		(see bug #395463).
+		Search environment.bz2 for the specified variables. Returns
+		a dict mapping variables to values, and any variables not
+		found in the environment will not be included in the dict.
+		This is useful for querying variables like ${SRC_URI} and
+		${A}, which are not saved in separate files but are available
+		in environment.bz2 (see bug #395463).
 		"""
 		env_file = self.getpath(cpv, filename="environment.bz2")
 		if not os.path.isfile(env_file):
-			return None
+			return {}
 		bunzip2_cmd = portage.util.shlex_split(
 			self.settings.get("PORTAGE_BUNZIP2_COMMAND", ""))
 		if not bunzip2_cmd:
@@ -771,38 +782,37 @@ class vardbapi(dbapi):
 			return close_quote_match is not None and \
 				close_quote_match.group(1) == quote
 
-		value = None
+		variables = frozenset(variables)
+		results = {}
 		for line in proc.stdout:
 			line = _unicode_decode(line,
 				encoding=_encodings['content'], errors='replace')
 			var_assign_match = var_assign_re.match(line)
 			if var_assign_match is not None:
-				if var_assign_match.group(2) == variable:
-					quote = var_assign_match.group(3)
-					if quote is not None:
-						if have_end_quote(quote,
-							line[var_assign_match.end(2)+2:]):
-							value = var_assign_match.group(4)
-						else:
-							value = [var_assign_match.group(4)]
-							for line in proc.stdout:
-								value.append(line)
-								if have_end_quote(quote, line):
-									break
-							value = ''.join(value)
-						# remove trailing quote and whitespace
-						value = value.rstrip()[:-1]
+				key = var_assign_match.group(2)
+				quote = var_assign_match.group(3)
+				if quote is not None:
+					if have_end_quote(quote,
+						line[var_assign_match.end(2)+2:]):
+						value = var_assign_match.group(4)
 					else:
-						value = var_assign_match.group(4).rstrip()
+						value = [var_assign_match.group(4)]
+						for line in proc.stdout:
+							value.append(line)
+							if have_end_quote(quote, line):
+								break
+						value = ''.join(value)
+					# remove trailing quote and whitespace
+					value = value.rstrip()[:-1]
+				else:
+					value = var_assign_match.group(4).rstrip()
 
-					# ignore remainder of file
-					for line in proc.stdout:
-						pass
-					break
+				if key in variables:
+					results[key] = value
 
 		proc.wait()
 		proc.stdout.close()
-		return value
+		return results
 
 	def aux_update(self, cpv, values):
 		mylink = self._dblink(cpv)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0472c8083ea647de8973504f60563aa08e4ef576
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Dec 21 23:16:13 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Dec 21 23:16:13 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0472c808

vardbapi._aux_get: account for _aux_cache_keys_re

---
 pym/portage/dbapi/vartree.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 6f8ee5f..7184550 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -720,7 +720,8 @@ class vardbapi(dbapi):
 					myf.close()
 			except IOError:
 				myd = None
-				if x not in self._aux_cache_keys:
+				if x not in self._aux_cache_keys and \
+					self._aux_cache_keys_re.match(x) is None:
 					myd = self._aux_env_search(mycpv, x)
 				if myd is None:
 					myd = _unicode_decode('')

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     9ec7e9fd222849e1fbe6311bb6973867957a9d81
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Dec 21 22:58:41 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Dec 21 22:58:41 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9ec7e9fd

vardbapi.aux_get: search environment, bug 395463

---
 pym/portage/dbapi/vartree.py |   84 +++++++++++++++++++++++++++++++++++++++--
 1 files changed, 79 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 6e71848..6f8ee5f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -32,6 +32,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap',
 	'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,pkgcmp,' + \
 		'_pkgsplit@pkgsplit',
+	'subprocess',
 	'tarfile',
 )
 
@@ -717,18 +718,91 @@ class vardbapi(dbapi):
 					myd = myf.read()
 				finally:
 					myf.close()
-				# Preserve \n for metadata that is known to
-				# contain multiple lines.
-				if self._aux_multi_line_re.match(x) is None:
-					myd = " ".join(myd.split())
 			except IOError:
-				myd = _unicode_decode('')
+				myd = None
+				if x not in self._aux_cache_keys:
+					myd = self._aux_env_search(mycpv, x)
+				if myd is None:
+					myd = _unicode_decode('')
+
+			# Preserve \n for metadata that is known to
+			# contain multiple lines.
+			if self._aux_multi_line_re.match(x) is None:
+				myd = " ".join(myd.split())
+
 			if x == "EAPI" and not myd:
 				results.append(_unicode_decode('0'))
 			else:
 				results.append(myd)
 		return results
 
+	def _aux_env_search(self, cpv, variable):
+		"""
+		Search environment.bz2 of the specified variable. Returns
+		the value if found, otherwise None. This is useful for
+		querying variables like ${SRC_URI} and ${A}, which are not
+		saved in separate files but are available in environment.bz2
+		(see bug #395463).
+		"""
+		env_file = self.getpath(cpv, filename="environment.bz2")
+		if not os.path.isfile(env_file):
+			return None
+		bunzip2_cmd = portage.util.shlex_split(
+			self.settings.get("PORTAGE_BUNZIP2_COMMAND", ""))
+		if not bunzip2_cmd:
+			bunzip2_cmd = portage.util.shlex_split(
+				self.settings["PORTAGE_BZIP2_COMMAND"])
+			bunzip2_cmd.append("-d")
+		args = bunzip2_cmd + ["-c", env_file]
+		try:
+			proc = subprocess.Popen(args, stdout=subprocess.PIPE)
+		except EnvironmentError as e:
+			if e.errno != errno.ENOENT:
+				raise
+			raise portage.exception.CommandNotFound(args[0])
+
+		# Parts of the following code are borrowed from
+		# filter-bash-environment.py (keep them in sync).
+		var_assign_re = re.compile(r'(^|^declare\s+-\S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?(.*)$')
+		close_quote_re = re.compile(r'(\\"|"|\')\s*$')
+		def have_end_quote(quote, line):
+			close_quote_match = close_quote_re.search(line)
+			return close_quote_match is not None and \
+				close_quote_match.group(1) == quote
+
+		value = None
+		for line in proc.stdout:
+			line = _unicode_decode(line,
+				encoding=_encodings['content'], errors='replace')
+			var_assign_match = var_assign_re.match(line)
+			if var_assign_match is not None:
+				if var_assign_match.group(2) == variable:
+					quote = var_assign_match.group(3)
+					if quote is not None:
+						if have_end_quote(quote,
+							line[var_assign_match.end(2)+2:]):
+							value = var_assign_match.group(4)
+						else:
+							value = [var_assign_match.group(4)]
+							for line in proc.stdout:
+								value.append(line)
+								if have_end_quote(quote, line):
+									break
+							value = ''.join(value)
+						# remove trailing quote and whitespace
+						value = value.rstrip()[:-1]
+					else:
+						value = var_assign_match.group(4).rstrip()
+
+					# ignore remainder of file
+					for line in proc.stdout:
+						pass
+					break
+
+		proc.wait()
+		proc.stdout.close()
+		return value
+
 	def aux_update(self, cpv, values):
 		mylink = self._dblink(cpv)
 		if not mylink.exists():

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3c36110f7c30fe866764dd1d20a65a273e0ecc6e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 20:26:48 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 20:26:48 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3c36110f

Silently allow missing /var/cache/edb/counter.

Files under /var/cache/ are allowed to disappear.

---
 pym/portage/dbapi/vartree.py |   12 ++++--------
 1 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a9a147a..6e71848 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -772,8 +772,7 @@ class vardbapi(dbapi):
 
 		@param myroot: ignored, self._eroot is used instead
 		"""
-		myroot = None
-		new_vdb = False
+		del myroot
 		counter = -1
 		try:
 			cfile = io.open(
@@ -782,8 +781,9 @@ class vardbapi(dbapi):
 				mode='r', encoding=_encodings['repo.content'],
 				errors='replace')
 		except EnvironmentError as e:
-			new_vdb = not bool(self.cpv_all())
-			if not new_vdb:
+			# Silently allow ENOENT since files under
+			# /var/cache/ are allowed to disappear.
+			if e.errno != errno.ENOENT:
 				writemsg(_("!!! Unable to read COUNTER file: '%s'\n") % \
 					self._counter_path, noiselevel=-1)
 				writemsg("!!! %s\n" % str(e), noiselevel=-1)
@@ -820,10 +820,6 @@ class vardbapi(dbapi):
 				if pkg_counter > max_counter:
 					max_counter = pkg_counter
 
-		if counter < 0 and not new_vdb:
-			writemsg(_("!!! Initializing COUNTER to " \
-				"value of %d\n") % max_counter, noiselevel=-1)
-
 		return max_counter + 1
 
 	def counter_tick_core(self, myroot=None, incrementing=1, mycpv=None):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4735e8a0ef6e3182b05c9e9d0e27202d19db619a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 10 19:41:18 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Dec 10 19:41:18 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4735e8a0

vartree.py: lazy movefile import

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index dd74c10..9af3a87 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -27,6 +27,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util.digraph:digraph',
 	'portage.util.env_update:env_update',
 	'portage.util.listdir:dircache,listdir',
+	'portage.util.movefile:movefile',
 	'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry',
 	'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap',
 	'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,pkgcmp,' + \
@@ -42,7 +43,6 @@ from portage.exception import CommandNotFound, \
 	InvalidData, InvalidLocation, InvalidPackageName, \
 	FileNotFound, PermissionDenied, UnsupportedAPIException
 from portage.localization import _
-from portage.util.movefile import movefile
 
 from portage import abssymlink, _movefile, bsd_chflags
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     716ad204d2107092704e4188f3a73736ec836d44
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 10 07:04:24 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Dec 10 07:04:24 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=716ad204

Make portdbapi.eclassdb a property.

---
 pym/portage/dbapi/porttree.py |    8 +++++++-
 1 files changed, 7 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index e483edb..233a2c1 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -59,6 +59,13 @@ class portdbapi(dbapi):
 	def porttree_root(self):
 		return self.settings.repositories.mainRepoLocation()
 
+	@property
+	def eclassdb(self):
+		main_repo = self.repositories.mainRepo()
+		if main_repo is None:
+			return None
+		return main_repo.eclass_db
+
 	def __init__(self, _unused_param=None, mysettings=None):
 		"""
 		@param _unused_param: deprecated, use mysettings['PORTDIR'] instead
@@ -102,7 +109,6 @@ class portdbapi(dbapi):
 					":".join(filter(None, sandbox_write))
 
 		self.porttrees = list(self.settings.repositories.repoLocationList())
-		self.eclassdb = self.repositories.mainRepo().eclass_db
 
 		# This is used as sanity check for aux_get(). If there is no
 		# root eclass dir, we assume that PORTDIR is invalid or

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d365eb3a6187ffcfccd1e68e637698d52e02070b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec  1 20:34:52 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Dec  1 20:34:52 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d365eb3a

MergeProcess: check if _elog_reader_fd is None

It seems saner to check for None, given that _elog_reader_fd is an int,
even though it will probably never be zero.

---
 pym/portage/dbapi/_MergeProcess.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index c9b6288..df501be 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -212,7 +212,7 @@ class MergeProcess(SpawnProcess):
 		if self._elog_reg_id is not None:
 			self.scheduler.unregister(self._elog_reg_id)
 			self._elog_reg_id = None
-		if self._elog_reader_fd:
+		if self._elog_reader_fd is not None:
 			os.close(self._elog_reader_fd)
 			self._elog_reader_fd = None
 		if self._elog_keys is not None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7c5b170d47ab054bc3f8a7778dd3f8139c1239c6
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Nov 27 20:53:55 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Nov 27 20:53:55 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7c5b170d

Revert "dblink.mergeme(): Merge files in alphabetic order."

This reverts commit 1d4ac47c28706094230cb2c4e6ee1c1c71629aa0.
As discussed on the gentoo-portage-dev mailing list, if we don't have
a compelling reason to sort here then it's just a waste:

http://archives.gentoo.org/gentoo-portage-dev/msg_67b08fcd7a1b832b347caf1b19362cbb.xml

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 099164a..dd74c10 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3981,7 +3981,7 @@ class dblink(object):
 			mergelist = stufftomerge
 			offset = ""
 
-		for i, x in enumerate(sorted(mergelist)):
+		for i, x in enumerate(mergelist):
 
 			mysrc = join(srcroot, offset, x)
 			mydest = join(destroot, offset, x)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Arfrever Frehtes Taifersar Arahesis]

commit:     1d4ac47c28706094230cb2c4e6ee1c1c71629aa0
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Gentoo <DOT> Org>
AuthorDate: Sat Nov 26 01:52:49 2011 +0000
Commit:     Arfrever Frehtes Taifersar Arahesis <arfrever <AT> gentoo <DOT> org>
CommitDate: Sat Nov 26 01:52:49 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1d4ac47c

dblink.mergeme(): Merge files in alphabetic order.

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index dd74c10..099164a 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3981,7 +3981,7 @@ class dblink(object):
 			mergelist = stufftomerge
 			offset = ""
 
-		for i, x in enumerate(mergelist):
+		for i, x in enumerate(sorted(mergelist)):
 
 			mysrc = join(srcroot, offset, x)
 			mydest = join(destroot, offset, x)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0744ebedfda56f446f83630084a8532edd68e363
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 29 23:52:06 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Oct 29 23:52:06 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0744ebed

portagetree: remove broken depcheck method

It called dep_check without the required third argument.

---
 pym/portage/dbapi/porttree.py |    5 +----
 1 files changed, 1 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 9ad434f..15f1faf 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -26,7 +26,7 @@ from portage.exception import PortageException, \
 from portage.localization import _
 
 from portage import eclass_cache, \
-	eapi_is_supported, dep_check, \
+	eapi_is_supported, \
 	_eapi_is_deprecated
 from portage import os
 from portage import _encodings
@@ -1138,9 +1138,6 @@ class portagetree(object):
 		psplit = pkgsplit(mysplit[1])
 		return "/".join([self.portroot, mysplit[0], psplit[0], mysplit[1]])+".ebuild"
 
-	def depcheck(self, mycheck, use="yes", myusesplit=None):
-		return dep_check(mycheck, self.dbapi, use=use, myuse=myusesplit)
-
 	def getslot(self,mycatpkg):
 		"Get a slot for a catpkg; assume it exists."
 		myslot = ""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     290990af18d2c56c26bb4b33f24e641948879522
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 29 05:35:01 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Oct 29 05:35:01 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=290990af

quickpkg: fix regression in hardlink support

Hardlink support has been broken since commit
4198da0184aaec30c41f2e5d2c7af71c4d35b662, which omitted the hardlink
logic from TarFile.gettarinfo().

---
 pym/portage/dbapi/vartree.py |   13 +++++++++++--
 1 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index ee0db6f..73772b0 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4560,11 +4560,20 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None):
 		tarinfo.mode = lst.st_mode
 		tarinfo.uid = lst.st_uid
 		tarinfo.gid = lst.st_gid
-		tarinfo.size = lst.st_size
+		tarinfo.size = 0
 		tarinfo.mtime = lst.st_mtime
 		tarinfo.linkname = ""
 		if stat.S_ISREG(lst.st_mode):
-			tarinfo.type = tarfile.REGTYPE
+			inode = (lst.st_ino, lst.st_dev)
+			if (lst.st_nlink > 1 and
+				inode in tar.inodes and
+				arcname != tar.inodes[inode]):
+				tarinfo.type = tarfile.LNKTYPE
+				tarinfo.linkname = tar.inodes[inode]
+			else:
+				tar.inodes[inode] = arcname
+				tarinfo.type = tarfile.REGTYPE
+				tarinfo.size = lst.st_size
 		elif stat.S_ISDIR(lst.st_mode):
 			tarinfo.type = tarfile.DIRTYPE
 		elif stat.S_ISLNK(lst.st_mode):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3cfb2bc711d820fcef376aee170de560d8195819
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 29 04:17:11 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Oct 29 04:17:11 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3cfb2bc7

Deprecate unused 'virtual' constructor parameters

---
 pym/portage/dbapi/bintree.py  |    9 ++++++++-
 pym/portage/dbapi/porttree.py |   18 ++++++++++++++++--
 pym/portage/dbapi/vartree.py  |    8 +++++++-
 3 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 0fb8be9..4c24929 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -243,7 +243,8 @@ def _pkgindex_cpv_map_latest_build(pkgindex):
 
 class binarytree(object):
 	"this tree scans for a list of all packages available in PKGDIR"
-	def __init__(self, _unused=None, pkgdir=None, virtual=None, settings=None):
+	def __init__(self, _unused=None, pkgdir=None,
+		virtual=DeprecationWarning, settings=None):
 
 		if pkgdir is None:
 			raise TypeError("pkgdir parameter is required")
@@ -258,6 +259,12 @@ class binarytree(object):
 				"settings['ROOT'] instead.",
 				DeprecationWarning, stacklevel=2)
 
+		if virtual is not DeprecationWarning:
+			warnings.warn("The 'virtual' parameter of the "
+				"portage.dbapi.bintree.binarytree"
+				" constructor is unused",
+				DeprecationWarning, stacklevel=2)
+
 		if True:
 			self.pkgdir = normalize_path(pkgdir)
 			self.dbapi = bindbapi(self, settings=settings)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index fc7feb3..5c45d97 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1064,7 +1064,7 @@ def close_portdbapi_caches():
 portage.process.atexit_register(portage.portageexit)
 
 class portagetree(object):
-	def __init__(self, root=None, virtual=None, settings=None):
+	def __init__(self, root=None, virtual=DeprecationWarning, settings=None):
 		"""
 		Constructor for a PortageTree
 		
@@ -1087,8 +1087,14 @@ class portagetree(object):
 				"settings['ROOT'] instead.",
 				DeprecationWarning, stacklevel=2)
 
+		if virtual is not DeprecationWarning:
+			warnings.warn("The 'virtual' parameter of the "
+				"portage.dbapi.porttree.portagetree"
+				" constructor is unused",
+				DeprecationWarning, stacklevel=2)
+
 		self.portroot = settings["PORTDIR"]
-		self.virtual = virtual
+		self.__virtual = virtual
 		self.dbapi = portdbapi(mysettings=settings)
 
 	@property
@@ -1100,6 +1106,14 @@ class portagetree(object):
 			DeprecationWarning, stacklevel=3)
 		return self.settings['ROOT']
 
+	@property
+	def virtual(self):
+		warnings.warn("The 'virtual' attribute of " + \
+			"portage.dbapi.porttree.portagetree" + \
+			" is deprecated.",
+			DeprecationWarning, stacklevel=3)
+		return self.__virtual
+
 	def dep_bestmatch(self,mydep):
 		"compatibility method"
 		mymatch = self.dbapi.xmatch("bestmatch-visible",mydep)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index e719062..ee0db6f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1163,7 +1163,7 @@ class vardbapi(dbapi):
 
 class vartree(object):
 	"this tree will scan a var/db/pkg database located at root (passed to init)"
-	def __init__(self, root=None, virtual=None, categories=None,
+	def __init__(self, root=None, virtual=DeprecationWarning, categories=None,
 		settings=None):
 
 		if settings is None:
@@ -1176,6 +1176,12 @@ class vartree(object):
 				"settings['ROOT'] instead.",
 				DeprecationWarning, stacklevel=2)
 
+		if virtual is not DeprecationWarning:
+			warnings.warn("The 'virtual' parameter of the "
+				"portage.dbapi.vartree.vartree"
+				" constructor is unused",
+				DeprecationWarning, stacklevel=2)
+
 		self.settings = settings
 		self.dbapi = vardbapi(settings=settings, vartree=self)
 		self.populated = 1

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4198da0184aaec30c41f2e5d2c7af71c4d35b662
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 28 18:03:02 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Oct 28 18:03:02 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4198da01

quickpkg: fix unicode for bug #388773

---
 pym/portage/dbapi/vartree.py |   41 ++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 40 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 7f94cc5..e719062 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -31,6 +31,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap',
 	'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,pkgcmp,' + \
 		'_pkgsplit@pkgsplit',
+	'tarfile',
 )
 
 from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \
@@ -61,10 +62,12 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
 
 import errno
 import gc
+import grp
 import io
 from itertools import chain
 import logging
 import os as _os
+import pwd
 import re
 import shutil
 import stat
@@ -4501,6 +4504,7 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None):
 			os = portage.os
 			encoding = _encodings['fs']
 
+	tar.encoding = encoding
 	root = normalize_path(root).rstrip(os.path.sep) + os.path.sep
 	id_strings = {}
 	maxval = len(contents)
@@ -4534,7 +4538,42 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None):
 			# recorded as a real directory in the tar file to ensure that tar
 			# can properly extract it's children.
 			live_path = os.path.realpath(live_path)
-		tarinfo = tar.gettarinfo(live_path, arcname)
+			lst = os.lstat(live_path)
+
+		# Since os.lstat() inside TarFile.gettarinfo() can trigger a
+		# UnicodeEncodeError when python has something other than utf_8
+		# return from sys.getfilesystemencoding() (as in bug #388773),
+		# we implement the needed functionality here, using the result
+		# of our successful lstat call. An alternative to this would be
+		# to pass in the fileobj argument to TarFile.gettarinfo(), so
+		# that it could use fstat instead of lstat. However, that would
+		# have the unwanted effect of dereferencing symlinks.
+
+		tarinfo = tar.tarinfo()
+		tarinfo.name = arcname
+		tarinfo.mode = lst.st_mode
+		tarinfo.uid = lst.st_uid
+		tarinfo.gid = lst.st_gid
+		tarinfo.size = lst.st_size
+		tarinfo.mtime = lst.st_mtime
+		tarinfo.linkname = ""
+		if stat.S_ISREG(lst.st_mode):
+			tarinfo.type = tarfile.REGTYPE
+		elif stat.S_ISDIR(lst.st_mode):
+			tarinfo.type = tarfile.DIRTYPE
+		elif stat.S_ISLNK(lst.st_mode):
+			tarinfo.type = tarfile.SYMTYPE
+			tarinfo.linkname = os.readlink(live_path)
+		else:
+			continue
+		try:
+			tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
+		except KeyError:
+			pass
+		try:
+			tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
+		except KeyError:
+			pass
 
 		if stat.S_ISREG(lst.st_mode):
 			if protect and protect(path):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1a7e2ade1a907a3695e91d4be1e1ff3254085b0a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 25 14:51:26 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct 25 14:51:26 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1a7e2ade

expand_new_virt: fix vardbapi.root reference

---
 pym/portage/dbapi/_expand_new_virt.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/_expand_new_virt.py b/pym/portage/dbapi/_expand_new_virt.py
index 6d6a27d..d379b4c 100644
--- a/pym/portage/dbapi/_expand_new_virt.py
+++ b/pym/portage/dbapi/_expand_new_virt.py
@@ -63,7 +63,8 @@ def expand_new_virt(vardb, atom):
 
 		success, atoms = portage.dep_check(rdepend,
 			None, vardb.settings, myuse=valid_use,
-			myroot=vardb.root, trees={vardb.root:{"porttree":vardb.vartree,
+			myroot=vardb.settings['EROOT'],
+			trees={vardb.settings['EROOT']:{"porttree":vardb.vartree,
 			"vartree":vardb.vartree}})
 
 		if success:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     de807d14fb994535f1ea529ecef8de0ee9db4136
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 19 21:19:09 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Oct 19 21:19:09 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=de807d14

Add bindbapi.getfetchsizes() like pordbapi has.

---
 pym/portage/dbapi/bintree.py |   28 ++++++++++++++++++++++++++++
 1 files changed, 28 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 62fc623..5c091bd 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -177,6 +177,34 @@ class bindbapi(fakedbapi):
 			self.bintree.populate()
 		return fakedbapi.cpv_all(self)
 
+	def getfetchsizes(self, pkg):
+		"""
+		This will raise MissingSignature if SIZE signature is not available,
+		or InvalidSignature if SIZE signature is invalid.
+		"""
+
+		if not self.bintree.populated:
+			self.bintree.populate()
+
+		pkg = getattr(pkg, 'cpv', pkg)
+
+		filesdict = {}
+		if not self.bintree.isremote(pkg):
+			pass
+		else:
+			metadata = self.bintree._remotepkgs[pkg]
+			try:
+				size = int(metadata["SIZE"])
+			except KeyError:
+				raise portage.exception.MissingSignature("SIZE")
+			except ValueError:
+				raise portage.exception.InvalidSignature(
+					"SIZE: %s" % metadata["SIZE"])
+			else:
+				filesdict[os.path.basename(self.bintree.getname(pkg))] = size
+
+		return filesdict
+
 def _pkgindex_cpv_map_latest_build(pkgindex):
 	"""
 	Given a PackageIndex instance, create a dict of cpv -> metadata map.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7ddb7d3068f53025e064e5f23bcfaf068401b9bd
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 17 05:20:10 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Oct 17 05:20:10 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7ddb7d30

portdbapi.aux_get: remove redunant EAPI code

This code only needs to exist in _metadata_callback.

---
 pym/portage/dbapi/porttree.py |   19 ++++++++-----------
 1 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index a0b7d7e..6ad7a6b 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -405,8 +405,11 @@ class portdbapi(dbapi):
 			eapi = "0"
 			metadata["EAPI"] = eapi
 		if not eapi_is_supported(eapi):
-			for k in set(metadata).difference(("_mtime_", "_eclasses_")):
-				metadata[k] = ""
+			keys = set(metadata)
+			keys.discard('_eclasses_')
+			keys.discard('_mtime_')
+			keys.discard('_%s_' % chf)
+			metadata.update((k, '') for k in keys)
 			metadata["EAPI"] = "-" + eapi.lstrip("-")
 
 		if cache is not None:
@@ -455,6 +458,7 @@ class portdbapi(dbapi):
 			eapi = metadata.get('EAPI', '').strip()
 			if not eapi:
 				eapi = '0'
+				metadata['EAPI'] = eapi
 			if eapi[:1] == '-' and eapi_is_supported(eapi[1:]):
 				continue
 			if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
@@ -544,21 +548,14 @@ class portdbapi(dbapi):
 
 				mydata = proc.metadata
 
-		# do we have a origin repository name for the current package
 		mydata["repository"] = self.repositories.get_name_for_location(mylocation)
-		mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", []))
 		mydata["_mtime_"] = ebuild_hash.mtime
-
 		eapi = mydata.get("EAPI")
 		if not eapi:
 			eapi = "0"
 			mydata["EAPI"] = eapi
-		if not eapi_is_supported(eapi):
-			keys = set(mydata)
-			keys.discard("_eclasses_")
-			keys.discard("_mtime_")
-			mydata.update((k, '') for k in keys)
-			mydata["EAPI"] = "-" + eapi.lstrip("-")
+		if eapi_is_supported(eapi):
+			mydata["INHERITED"] = " ".join(mydata.get("_eclasses_", []))
 
 		#finally, we look at our internal cache entry and return the requested data.
 		returnme = [mydata.get(x, "") for x in mylist]

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     28115d7d72cf0f0c42c636039ed2a08ee2412dd3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 16 05:58:56 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 16 05:58:56 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=28115d7d

_pull_valid_cache: pass unicode to hashed_path

This removes a conversion to bytes that should have been removed in
commit 2ed1cb53cc4158af08c22d466b15b9a9a7767212. The hashed_path
location attribute should always be unicode, for consistency will path
handling throughout portage. The hashed_path class internally uses the
portage portage.os wrapper to ensure that paths are encoded as utf_8
bytes when they are passed to the real os module for stat calls.

---
 pym/portage/dbapi/porttree.py |    5 +----
 1 files changed, 1 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index a1308a7..13c27e9 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -420,10 +420,7 @@ class portdbapi(dbapi):
 
 	def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
 		try:
-			# Don't use unicode-wrapped os module, for better performance.
-			path = _unicode_encode(ebuild_path,
-				encoding=_encodings['fs'], errors='strict')
-			ebuild_hash = eclass_cache.hashed_path(path)
+			ebuild_hash = eclass_cache.hashed_path(ebuild_path)
 			# snag mtime since we use it later, and to trigger stat failure
 			# if it doesn't exist
 			ebuild_hash.mtime

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e3f0d11c8615010506052793689207b1c3b3bd86
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 15 20:28:44 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Oct 15 20:28:44 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e3f0d11c

portdbapi.aux_get: fix broken _metadata_callback

This fixes a regression in FEATURES=parse-eapi-ebuild-head support
since commit 2ed1cb53cc4158af08c22d466b15b9a9a7767212.

---
 pym/portage/dbapi/porttree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 36b326c..a1308a7 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -529,7 +529,7 @@ class portdbapi(dbapi):
 
 			if eapi is not None and not portage.eapi_is_supported(eapi):
 				mydata = self._metadata_callback(
-					mycpv, ebuild_hash, mylocation, {'EAPI':eapi}, emtime)
+					mycpv, mylocation, {'EAPI':eapi}, ebuild_hash)
 			else:
 				proc = EbuildMetadataPhase(cpv=mycpv,
 					ebuild_hash=ebuild_hash,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     382e74bb63d6f1eed2b3d88daee95fc7530df3da
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 14 00:23:28 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Oct 14 00:23:28 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=382e74bb

_pull_valid_cache: fix readonly delete logic

---
 pym/portage/dbapi/porttree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index f48741b..dd627e3 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -429,7 +429,7 @@ class portdbapi(dbapi):
 			except KeyError:
 				pass
 			except CacheError:
-				if auxdb is not pregen_auxdb:
+				if not auxdb.readonly:
 					try:
 						del auxdb[cpv]
 					except KeyError:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1c8ff00a26b0e2e3627e9c552374f71235ee6c39
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct  5 04:27:03 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Oct  5 04:27:03 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1c8ff00a

dblink._elog_process: preserve empty lines

---
 pym/portage/dbapi/vartree.py |    2 --
 1 files changed, 0 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 50fa6f5..1fc95a3 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3126,8 +3126,6 @@ class dblink(object):
 						lines = [lines]
 					for line in lines:
 						for line in line.split('\n'):
-							if not line:
-								continue
 							fields = (funcname, phase, cpv, line)
 							str_buffer.append(' '.join(fields))
 							str_buffer.append('\n')

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     37e01fd5095a40b6a57b1365add1c64f29671211
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  2 19:52:57 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct  2 19:52:57 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=37e01fd5

dblink._elog_process: handle newlines in messages

This will fix bug #385341.

---
 pym/portage/dbapi/vartree.py |    9 ++++++---
 1 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a4c54bd..50fa6f5 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3125,9 +3125,12 @@ class dblink(object):
 					if isinstance(lines, basestring):
 						lines = [lines]
 					for line in lines:
-						fields = (funcname, phase, cpv, line.rstrip('\n'))
-						str_buffer.append(' '.join(fields))
-						str_buffer.append('\n')
+						for line in line.split('\n'):
+							if not line:
+								continue
+							fields = (funcname, phase, cpv, line)
+							str_buffer.append(' '.join(fields))
+							str_buffer.append('\n')
 			if str_buffer:
 				os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     60c7ca4030839a728a5975ee01e28c0330cf33ae
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 27 16:46:02 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 27 16:46:02 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=60c7ca40

merge: handle \r in file names for bug #384597

---
 pym/portage/dbapi/vartree.py |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index f146ade..a4c54bd 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3279,6 +3279,7 @@ class dblink(object):
 		# the check must be repeated here for binary packages (it's
 		# inexpensive since we call os.walk() here anyway).
 		unicode_errors = []
+		line_ending_re = re.compile('[\n\r]')
 
 		while True:
 
@@ -3330,7 +3331,7 @@ class dblink(object):
 
 					relative_path = fpath[srcroot_len:]
 
-					if "\n" in relative_path:
+					if line_ending_re.search(relative_path) is not None:
 						paths_with_newlines.append(relative_path)
 
 					file_mode = os.lstat(fpath).st_mode
@@ -3353,11 +3354,11 @@ class dblink(object):
 
 		if paths_with_newlines:
 			msg = []
-			msg.append(_("This package installs one or more files containing a newline (\\n) character:"))
+			msg.append(_("This package installs one or more files containing line ending characters:"))
 			msg.append("")
 			paths_with_newlines.sort()
 			for f in paths_with_newlines:
-				msg.append("\t/%s" % (f.replace("\n", "\\n")))
+				msg.append("\t/%s" % (f.replace("\n", "\\n").replace("\r", "\\r")))
 			msg.append("")
 			msg.append(_("package %s NOT merged") % self.mycpv)
 			msg.append("")

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     da782a6de61a3c0d9cd079c14d920852d2a4db9b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 27 14:58:26 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 27 14:58:26 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=da782a6d

porttree.py: remove unused variables

---
 pym/portage/dbapi/porttree.py |   12 +++++-------
 1 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index d710d64..e679f00 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -235,13 +235,12 @@ class portdbapi(dbapi):
 		group."""
 
 		dirmode  = 0o2070
-		filemode =   0o60
 		modemask =    0o2
 
 		try:
 			ensure_dirs(self.depcachedir, gid=portage_gid,
 				mode=dirmode, mask=modemask)
-		except PortageException as e:
+		except PortageException:
 			pass
 
 	def close_caches(self):
@@ -626,7 +625,7 @@ class portdbapi(dbapi):
 			mystat = None
 			try:
 				mystat = os.stat(file_path)
-			except OSError as e:
+			except OSError:
 				pass
 			if mystat is None:
 				existing_size = 0
@@ -1130,9 +1129,9 @@ class portagetree(object):
 		myslot = ""
 		try:
 			myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0]
-		except SystemExit as e:
+		except SystemExit:
 			raise
-		except Exception as e:
+		except Exception:
 			pass
 		return myslot
 
@@ -1193,7 +1192,7 @@ def _parse_uri_map(cpv, metadata, use=None):
 	while myuris:
 		uri = myuris.pop()
 		if myuris and myuris[-1] == "->":
-			operator = myuris.pop()
+			myuris.pop()
 			distfile = myuris.pop()
 		else:
 			distfile = os.path.basename(uri)
@@ -1208,6 +1207,5 @@ def _parse_uri_map(cpv, metadata, use=None):
 			uri_map[distfile] = uri_set
 		uri_set.add(uri)
 		uri = None
-		operator = None
 
 	return uri_map

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     77b63ed6f9fc2f1c30dfb100ca12aef02f5f0240
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 22:23:29 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 22:23:29 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=77b63ed6

portdbapi.xmatch: fix unknown repo / return empty

We want to return an empty list for the match-* methods, otherwise an
empty string.

---
 pym/portage/dbapi/porttree.py |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 9a8f508..d710d64 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -849,7 +849,10 @@ class portdbapi(dbapi):
 		if mydep.repo is not None:
 			mytree = self.treemap.get(mydep.repo)
 			if mytree is None:
-				myval = []
+				if level.startswith("match-"):
+					myval = []
+				else:
+					myval = ""
 
 		if myval is not None:
 			# Unknown repo, empty result.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     c495b938fd10295928f455db6391bcdac2031acc
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 22:03:07 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 22:03:07 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c495b938

portdbapi: deprecate unused 'visible' method

---
 pym/portage/dbapi/porttree.py |   13 ++++++-------
 1 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 62a1e8f..9a8f508 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -962,18 +962,17 @@ class portdbapi(dbapi):
 	def gvisible(self, mylist):
 		warnings.warn("The 'gvisible' method of "
 			"portage.dbapi.porttree.portdbapi "
-			"is deprecated, and the functionality "
-			"has been combined into the 'visible' method",
+			"is deprecated",
 			DeprecationWarning, stacklevel=2)
-		return self.visible(mylist)
+		return list(self._iter_visible(iter(mylist)))
 
 	def visible(self, cpv_iter):
-		"""
-		Return a list containing only visible packages.
-		"""
+		warnings.warn("The 'visible' method of "
+			"portage.dbapi.porttree.portdbapi "
+			"is deprecated",
+			DeprecationWarning, stacklevel=2)
 		if cpv_iter is None:
 			return []
-
 		return list(self._iter_visible(iter(cpv_iter)))
 
 	def _iter_visible(self, cpv_iter, myrepo=None):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     145d69342301e3a6372a172cfd2eb8069a6a0b3e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 21:44:36 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 21:44:36 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=145d6934

portdbapi.xmatch: combine match-all with others

This eliminates redundant code.

---
 pym/portage/dbapi/porttree.py |   25 +++----------------------
 1 files changed, 3 insertions(+), 22 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 94b25e8..62a1e8f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -868,26 +868,7 @@ class portdbapi(dbapi):
 				myval = match_from_list(mydep,
 					self.cp_list(mykey, mytree=mytree))
 
-		elif level == "match-all":
-			# match *all* visible *and* masked packages
-			if mydep == mykey:
-				myval = self.cp_list(mykey, mytree=mytree)
-			elif mydep.repo is not None:
-				myval = list(self._iter_match(mydep,
-					self.cp_list(mykey, mytree=mytree)))
-			else:
-				myval = set()
-				# We iterate over self.porttrees, since it's common to
-				# tweak this attribute in order to adjust match behavior.
-				for tree in self.porttrees:
-					repo = self.repositories.get_name_for_location(tree)
-					myval.update(self._iter_match(mydep.with_repo(repo),
-					self.cp_list(mykey, mytree=tree)))
-				myval = list(myval)
-				if len(myval) > 1:
-					self._cpv_sort_ascending(myval)
-
-		elif level in ("bestmatch-visible", "match-visible",
+		elif level in ("bestmatch-visible", "match-all", "match-visible",
 			"minimum-all", "minimum-visible"):
 			# Find the minimum matching visible version. This is optimized to
 			# minimize the number of metadata accesses (improves performance
@@ -898,8 +879,8 @@ class portdbapi(dbapi):
 				mylist = match_from_list(mydep,
 					self.cp_list(mykey, mytree=mytree))
 
-			visibility_filter = level != "minimum-all"
-			single_match = level != "match-visible"
+			visibility_filter = level not in ("match-all", "minimum-all")
+			single_match = level not in ("match-all", "match-visible")
 			myval = []
 			aux_keys = list(self._aux_cache_keys)
 			if level == "bestmatch-visible":

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     c0e897575e41ec4b8367e05d5d53f658ed0fcf8a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 21:31:48 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 21:31:48 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c0e89757

portdbapi.xmatch: combine minimum-all/*-visible

This eliminates redundant code.

---
 pym/portage/dbapi/porttree.py |   29 ++++-------------------------
 1 files changed, 4 insertions(+), 25 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index d5068ab..94b25e8 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -887,30 +887,8 @@ class portdbapi(dbapi):
 				if len(myval) > 1:
 					self._cpv_sort_ascending(myval)
 
-		elif level == "minimum-all":
-			# Find the minimum matching version. This is optimized to
-			# minimize the number of metadata accesses (improves performance
-			# especially in cases where metadata needs to be generated).
-			myval = ""
-			if mydep == mykey:
-				for myval in self.cp_list(mykey, mytree=mytree):
-					break
-			elif mydep.repo is not None:
-				for myval in self._iter_match(mydep,
-					self.cp_list(mykey, mytree=mytree)):
-					break
-			else:
-				# We iterate over self.porttrees, since it's common to
-				# tweak this attribute in order to adjust match behavior.
-				for tree in self.porttrees:
-					repo = self.repositories.get_name_for_location(tree)
-					for myval in self._iter_match(mydep.with_repo(repo),
-						self.cp_list(mykey, mytree=tree)):
-						break
-					if myval:
-						break
-
-		elif level in ("match-visible", "minimum-visible", "bestmatch-visible"):
+		elif level in ("bestmatch-visible", "match-visible",
+			"minimum-all", "minimum-visible"):
 			# Find the minimum matching visible version. This is optimized to
 			# minimize the number of metadata accesses (improves performance
 			# especially in cases where metadata needs to be generated).
@@ -920,6 +898,7 @@ class portdbapi(dbapi):
 				mylist = match_from_list(mydep,
 					self.cp_list(mykey, mytree=mytree))
 
+			visibility_filter = level != "minimum-all"
 			single_match = level != "match-visible"
 			myval = []
 			aux_keys = list(self._aux_cache_keys)
@@ -946,7 +925,7 @@ class portdbapi(dbapi):
 						# ebuild not in this repo, or masked by corruption
 						continue
 
-					if not self._visible(cpv, metadata):
+					if visibility_filter and not self._visible(cpv, metadata):
 						continue
 
 					if mydep.slot is not None and \

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     03adf83487276541b619396bdf79812b38ee8422
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 21:18:41 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 21:18:41 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=03adf834

portdbapi.xmatch: combine *-visible code

This allows match-visible to avoid calling match-all, which allows it
to avoid an extra loop over all repos.

---
 pym/portage/dbapi/porttree.py |   30 +++++++++++++++++-------------
 1 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 782cecc..d5068ab 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -887,12 +887,6 @@ class portdbapi(dbapi):
 				if len(myval) > 1:
 					self._cpv_sort_ascending(myval)
 
-		elif level == "match-visible":
-			# find all visible matches
-			myval = self.xmatch("match-all", mydep)
-			if myval:
-				myval = list(self._iter_visible(myval, myrepo=mydep.repo))
-
 		elif level == "minimum-all":
 			# Find the minimum matching version. This is optimized to
 			# minimize the number of metadata accesses (improves performance
@@ -916,7 +910,7 @@ class portdbapi(dbapi):
 					if myval:
 						break
 
-		elif level in ("minimum-visible", "bestmatch-visible"):
+		elif level in ("match-visible", "minimum-visible", "bestmatch-visible"):
 			# Find the minimum matching visible version. This is optimized to
 			# minimize the number of metadata accesses (improves performance
 			# especially in cases where metadata needs to be generated).
@@ -925,12 +919,14 @@ class portdbapi(dbapi):
 			else:
 				mylist = match_from_list(mydep,
 					self.cp_list(mykey, mytree=mytree))
-			myval = ""
+
+			single_match = level != "match-visible"
+			myval = []
 			aux_keys = list(self._aux_cache_keys)
-			if level == "minimum-visible":
-				iterfunc = iter
-			else:
+			if level == "bestmatch-visible":
 				iterfunc = reversed
+			else:
+				iterfunc = iter
 
 			if mydep.repo is not None:
 				repos = [mydep.repo]
@@ -961,11 +957,19 @@ class portdbapi(dbapi):
 						not self._match_use(mydep, cpv, metadata):
 						continue
 
-					myval = cpv
+					myval.append(cpv)
+					# only yield a given cpv once
 					break
-				if myval:
+
+				if myval and single_match:
 					break
 
+			if single_match:
+				if myval:
+					myval = myval[0]
+				else:
+					myval = ""
+
 		elif level == "bestmatch-list":
 			#dep match -- find best match but restrict search to sublist
 			warnings.warn("The 'bestmatch-list' mode of "

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5e587e1db10e38f003462f8171bd4d838cc095be
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 20:47:34 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 20:47:34 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5e587e1d

dbapi: split _match_use from _iter_match_use

---
 pym/portage/dbapi/__init__.py |  106 +++++++++++++++++++++-------------------
 1 files changed, 56 insertions(+), 50 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index e386faa..96e85ce 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -155,64 +155,70 @@ class dbapi(object):
 		2) Check enabled/disabled flag states.
 		"""
 
-		iuse_implicit_match = self.settings._iuse_implicit_match
+		aux_keys = ["IUSE", "SLOT", "USE"]
 		for cpv in cpv_iter:
 			try:
-				iuse, slot, use = self.aux_get(cpv, ["IUSE", "SLOT", "USE"], myrepo=atom.repo)
+				metadata = dict(zip(aux_keys,
+					self.aux_get(cpv, aux_keys, myrepo=atom.repo)))
 			except KeyError:
 				continue
-			iuse = frozenset(x.lstrip('+-') for x in iuse.split())
-			missing_iuse = False
-			for x in atom.unevaluated_atom.use.required:
-				if x not in iuse and not iuse_implicit_match(x):
-					missing_iuse = True
-					break
-			if missing_iuse:
+
+			if not self._match_use(atom, cpv, metadata):
 				continue
-			if not atom.use:
-				pass
-			elif not self._use_mutable:
-				# Use IUSE to validate USE settings for built packages,
-				# in case the package manager that built this package
-				# failed to do that for some reason (or in case of
-				# data corruption).
-				use = frozenset(x for x in use.split() if x in iuse or \
-					iuse_implicit_match(x))
-				missing_enabled = atom.use.missing_enabled.difference(iuse)
-				missing_disabled = atom.use.missing_disabled.difference(iuse)
-
-				if atom.use.enabled:
-					if atom.use.enabled.intersection(missing_disabled):
-						continue
-					need_enabled = atom.use.enabled.difference(use)
+
+			yield cpv
+
+	def _match_use(self, atom, cpv, metadata):
+		iuse_implicit_match = self.settings._iuse_implicit_match
+		iuse = frozenset(x.lstrip('+-') for x in metadata["IUSE"].split())
+
+		for x in atom.unevaluated_atom.use.required:
+			if x not in iuse and not iuse_implicit_match(x):
+				return False
+
+		if atom.use is None:
+			pass
+
+		elif not self._use_mutable:
+			# Use IUSE to validate USE settings for built packages,
+			# in case the package manager that built this package
+			# failed to do that for some reason (or in case of
+			# data corruption).
+			use = frozenset(x for x in metadata["USE"].split()
+				if x in iuse or iuse_implicit_match(x))
+			missing_enabled = atom.use.missing_enabled.difference(iuse)
+			missing_disabled = atom.use.missing_disabled.difference(iuse)
+
+			if atom.use.enabled:
+				if atom.use.enabled.intersection(missing_disabled):
+					return False
+				need_enabled = atom.use.enabled.difference(use)
+				if need_enabled:
+					need_enabled = need_enabled.difference(missing_enabled)
 					if need_enabled:
-						need_enabled = need_enabled.difference(missing_enabled)
-						if need_enabled:
-							continue
+						return False
 
-				if atom.use.disabled:
-					if atom.use.disabled.intersection(missing_enabled):
-						continue
-					need_disabled = atom.use.disabled.intersection(use)
+			if atom.use.disabled:
+				if atom.use.disabled.intersection(missing_enabled):
+					return False
+				need_disabled = atom.use.disabled.intersection(use)
+				if need_disabled:
+					need_disabled = need_disabled.difference(missing_disabled)
 					if need_disabled:
-						need_disabled = need_disabled.difference(missing_disabled)
-						if need_disabled:
-							continue
-			else:
-				# Check masked and forced flags for repoman.
-				mysettings = getattr(self, 'settings', None)
-				if mysettings is not None and not mysettings.local_config:
+						return False
 
-					pkg = "%s:%s" % (cpv, slot)
-					usemask = mysettings._getUseMask(pkg)
-					if usemask.intersection(atom.use.enabled):
-						continue
+		elif not self.settings.local_config:
+			# Check masked and forced flags for repoman.
+			pkg = "%s:%s" % (cpv, metadata["SLOT"])
+			usemask = self.settings._getUseMask(pkg)
+			if usemask.intersection(atom.use.enabled):
+				return False
 
-					useforce = mysettings._getUseForce(pkg).difference(usemask)
-					if useforce.intersection(atom.use.disabled):
-						continue
+			useforce = self.settings._getUseForce(pkg).difference(usemask)
+			if useforce.intersection(atom.use.disabled):
+				return False
 
-			yield cpv
+		return True
 
 	def invalidentry(self, mypath):
 		if '/-MERGING-' in mypath:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     97d2dac2cb7ee7ad33a31786eed74e1c204576dd
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 20:14:59 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 20:14:59 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=97d2dac2

portdbapi: tweak single-tree optimization

This optimizes aux_get and cp_list for cases where we only have a
single tree, which is common for repoman and ebuild(1).

---
 pym/portage/dbapi/porttree.py |   29 +++++++++++++++++++++--------
 1 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index d3cca00..42e5ca1 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -465,9 +465,16 @@ class portdbapi(dbapi):
 			if mytree is None:
 				raise KeyError(myrepo)
 
-		if not mytree:
+		if mytree is not None and len(self.porttrees) == 1 \
+			and mytree == self.porttrees[0]:
+			# mytree matches our only tree, so it's safe to
+			# ignore mytree and cache the result
+			mytree = None
+			myrepo = None
+
+		if mytree is None:
 			cache_me = True
-		if not mytree and not self._known_keys.intersection(
+		if mytree is None and not self._known_keys.intersection(
 			mylist).difference(self._aux_cache_keys):
 			aux_cache = self._aux_cache.get(mycpv)
 			if aux_cache is not None:
@@ -730,6 +737,14 @@ class portdbapi(dbapi):
 		return l
 
 	def cp_list(self, mycp, use_cache=1, mytree=None):
+
+		if self.frozen and mytree is not None \
+			and len(self.porttrees) == 1 \
+			and mytree == self.porttrees[0]:
+			# mytree matches our only tree, so it's safe to
+			# ignore mytree and cache the result
+			mytree = None
+
 		if self.frozen and mytree is None:
 			cachelist = self.xcache["cp-list"].get(mycp)
 			if cachelist is not None:
@@ -857,7 +872,7 @@ class portdbapi(dbapi):
 			# match *all* visible *and* masked packages
 			if mydep == mykey:
 				myval = self.cp_list(mykey, mytree=mytree)
-			elif mydep.repo is not None or len(self.porttrees) == 1:
+			elif mydep.repo is not None:
 				myval = list(self._iter_match(mydep,
 					self.cp_list(mykey, mytree=mytree)))
 			else:
@@ -886,7 +901,7 @@ class portdbapi(dbapi):
 			if mydep == mykey:
 				for myval in self.cp_list(mykey, mytree=mytree):
 					break
-			elif mydep.repo is not None or len(self.porttrees) == 1:
+			elif mydep.repo is not None:
 				for myval in self._iter_match(mydep,
 					self.cp_list(mykey, mytree=mytree)):
 					break
@@ -917,7 +932,7 @@ class portdbapi(dbapi):
 			else:
 				iterfunc = reversed
 
-			if mydep.repo is not None or len(self.porttrees) == 1:
+			if mydep.repo is not None:
 				repos = [mydep.repo]
 			else:
 				# We iterate over self.porttrees, since it's common to
@@ -1010,9 +1025,7 @@ class portdbapi(dbapi):
 		aux_keys = list(self._aux_cache_keys)
 		metadata = {}
 
-		if len(self.porttrees) == 1:
-			repos = [None]
-		elif myrepo is not None:
+		if myrepo is not None:
 			repos = [myrepo]
 		else:
 			# We iterate over self.porttrees, since it's common to

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d3ad44b102f44aecdc0dc6ef951e853a9dd8dbbe
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 19:50:06 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 19:50:06 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d3ad44b1

portdbapi.xmatch: use _visible more

---
 pym/portage/dbapi/porttree.py |   25 ++++---------------------
 1 files changed, 4 insertions(+), 21 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 0967714..d3cca00 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -911,8 +911,6 @@ class portdbapi(dbapi):
 				mylist = match_from_list(mydep,
 					self.cp_list(mykey, mytree=mytree))
 			myval = ""
-			settings = self.settings
-			local_config = settings.local_config
 			aux_keys = list(self._aux_cache_keys)
 			if level == "minimum-visible":
 				iterfunc = iter
@@ -936,29 +934,14 @@ class portdbapi(dbapi):
 					except KeyError:
 						# ebuild not in this repo, or masked by corruption
 						continue
-					if not eapi_is_supported(metadata["EAPI"]):
+
+					if not self._visible(cpv, metadata):
 						continue
+
 					if mydep.slot is not None and \
 						mydep.slot != metadata["SLOT"]:
 						continue
-					if settings._getMissingKeywords(cpv, metadata):
-						continue
-					if settings._getMaskAtom(cpv, metadata):
-						continue
-					if local_config:
-						metadata["USE"] = ""
-						if "?" in metadata["LICENSE"] or \
-							"?" in metadata["PROPERTIES"]:
-							self.doebuild_settings.setcpv(cpv, mydb=metadata)
-							metadata["USE"] = \
-								self.doebuild_settings.get("PORTAGE_USE", "")
-						try:
-							if settings._getMissingLicenses(cpv, metadata):
-								continue
-							if settings._getMissingProperties(cpv, metadata):
-								continue
-						except InvalidDependString:
-							continue
+
 					if mydep.use is not None:
 						mydep_with_repo = mydep
 						if repo is not None and mydep.repo is None:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     13729a95fe68a51a0a18194a913fee4d3c17d160
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 19:27:39 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 19:27:39 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=13729a95

portdbapi: split _visible from _iter_visible

---
 pym/portage/dbapi/porttree.py |   68 +++++++++++++++++++++--------------------
 1 files changed, 35 insertions(+), 33 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index aaa69fa..0967714 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1026,10 +1026,6 @@ class portdbapi(dbapi):
 		"""
 		aux_keys = list(self._aux_cache_keys)
 		metadata = {}
-		local_config = self.settings.local_config
-		chost = self.settings.get('CHOST', '')
-		accept_chost = self.settings._accept_chost
-		getMaskAtom = self.settings._getMaskAtom
 
 		if len(self.porttrees) == 1:
 			repos = [None]
@@ -1056,41 +1052,47 @@ class portdbapi(dbapi):
 					writemsg("!!! %s\n" % (e,), noiselevel=-1)
 					del e
 					continue
-				eapi = metadata["EAPI"]
-				if not eapi_is_supported(eapi):
-					continue
-				if _eapi_is_deprecated(eapi):
-					continue
-				if not metadata["SLOT"]:
-					continue
-				if getMaskAtom(mycpv, metadata):
-					continue
-				if self.settings._getMissingKeywords(mycpv, metadata):
+
+				if not self._visible(mycpv, metadata):
 					continue
-				if local_config:
-					metadata['CHOST'] = chost
-					if not accept_chost(mycpv, metadata):
-						continue
-					metadata["USE"] = ""
-					if "?" in metadata["LICENSE"] or \
-						"?" in metadata["PROPERTIES"]:
-						self.doebuild_settings.setcpv(mycpv, mydb=metadata)
-						metadata['USE'] = \
-							self.doebuild_settings['PORTAGE_USE']
-					try:
-						if self.settings._getMissingLicenses(
-							mycpv, metadata):
-							continue
-						if self.settings._getMissingProperties(
-							mycpv, metadata):
-							continue
-					except InvalidDependString:
-						continue
 
 				yield mycpv
 				# only yield a given cpv once
 				break
 
+	def _visible(self, cpv, metadata):
+		eapi = metadata["EAPI"]
+		if not eapi_is_supported(eapi):
+			return False
+		if _eapi_is_deprecated(eapi):
+			return False
+		if not metadata["SLOT"]:
+			return False
+
+		settings = self.settings
+		if settings._getMaskAtom(cpv, metadata):
+			return False
+		if settings._getMissingKeywords(cpv, metadata):
+			return False
+		if settings.local_config:
+			metadata['CHOST'] = settings.get('CHOST', '')
+			if not settings._accept_chost(cpv, metadata):
+				return False
+			metadata["USE"] = ""
+			if "?" in metadata["LICENSE"] or \
+				"?" in metadata["PROPERTIES"]:
+				self.doebuild_settings.setcpv(cpv, mydb=metadata)
+				metadata['USE'] = self.doebuild_settings['PORTAGE_USE']
+			try:
+				if settings._getMissingLicenses(cpv, metadata):
+					return False
+				if settings._getMissingProperties(cpv, metadata):
+					return False
+			except InvalidDependString:
+				return False
+
+		return True
+
 def close_portdbapi_caches():
 	for i in portdbapi.portdbapi_instances:
 		i.close_caches()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5076fa35cdcba5a092f16c2004e90d4d84c01044
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 19:05:00 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 19:05:00 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5076fa35

portdbapi: move repo loop to _iter_visible

This allows use to avoid using a set to eliminate duplicates, since we
can break out of the repo loop as soon as a given cpv is found to be
visible.

---
 pym/portage/dbapi/porttree.py |  123 +++++++++++++++++++++-------------------
 1 files changed, 65 insertions(+), 58 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index e4cf9a6..aaa69fa 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -874,22 +874,9 @@ class portdbapi(dbapi):
 
 		elif level == "match-visible":
 			# find all visible matches
-			if mydep.repo is not None or len(self.porttrees) == 1:
-				myval = list(self._iter_visible(
-					self.xmatch("match-all", mydep),
-					mytree=mytree))
-			else:
-				myval = set()
-				# We iterate over self.porttrees, since it's common to
-				# tweak this attribute in order to adjust match behavior.
-				for tree in self.porttrees:
-					repo = self.repositories.get_name_for_location(tree)
-					myval.update(self._iter_visible(
-						self.xmatch("match-all", mydep.with_repo(repo)),
-						mytree=tree))
-				myval = list(myval)
-				if len(myval) > 1:
-					self._cpv_sort_ascending(myval)
+			myval = self.xmatch("match-all", mydep)
+			if myval:
+				myval = list(self._iter_visible(myval, myrepo=mydep.repo))
 
 		elif level == "minimum-all":
 			# Find the minimum matching version. This is optimized to
@@ -1016,24 +1003,24 @@ class portdbapi(dbapi):
 	def match(self, mydep, use_cache=1):
 		return self.xmatch("match-visible", mydep)
 
-	def gvisible(self, mylist, mytree=None):
+	def gvisible(self, mylist):
 		warnings.warn("The 'gvisible' method of "
 			"portage.dbapi.porttree.portdbapi "
 			"is deprecated, and the functionality "
 			"has been combined into the 'visible' method",
 			DeprecationWarning, stacklevel=2)
-		return self.visible(mylist, mytree=mytree)
+		return self.visible(mylist)
 
-	def visible(self, cpv_iter, mytree=None):
+	def visible(self, cpv_iter):
 		"""
 		Return a list containing only visible packages.
 		"""
-		if mylist is None:
+		if cpv_iter is None:
 			return []
 
-		return list(self._iter_visible(iter(cpv_iter), mytree=mytree))
+		return list(self._iter_visible(iter(cpv_iter)))
 
-	def _iter_visible(self, cpv_iter, mytree=None):
+	def _iter_visible(self, cpv_iter, myrepo=None):
 		"""
 		Return a new list containing only visible packages.
 		"""
@@ -1043,46 +1030,66 @@ class portdbapi(dbapi):
 		chost = self.settings.get('CHOST', '')
 		accept_chost = self.settings._accept_chost
 		getMaskAtom = self.settings._getMaskAtom
+
+		if len(self.porttrees) == 1:
+			repos = [None]
+		elif myrepo is not None:
+			repos = [myrepo]
+		else:
+			# We iterate over self.porttrees, since it's common to
+			# tweak this attribute in order to adjust match behavior.
+			repos = []
+			for tree in reversed(self.porttrees):
+				repos.append(self.repositories.get_name_for_location(tree))
+
 		for mycpv in cpv_iter:
-			metadata.clear()
-			try:
-				metadata.update(zip(aux_keys,
-					self.aux_get(mycpv, aux_keys, mytree=mytree)))
-			except KeyError:
-				continue
-			except PortageException as e:
-				writemsg("!!! Error: aux_get('%s', %s)\n" % (mycpv, aux_keys),
-					noiselevel=-1)
-				writemsg("!!! %s\n" % (e,), noiselevel=-1)
-				del e
-				continue
-			eapi = metadata["EAPI"]
-			if not eapi_is_supported(eapi):
-				continue
-			if _eapi_is_deprecated(eapi):
-				continue
-			if not metadata["SLOT"]:
-				continue
-			if getMaskAtom(mycpv, metadata):
-				continue
-			if self.settings._getMissingKeywords(mycpv, metadata):
-				continue
-			if local_config:
-				metadata['CHOST'] = chost
-				if not accept_chost(mycpv, metadata):
-					continue
-				metadata["USE"] = ""
-				if "?" in metadata["LICENSE"] or "?" in metadata["PROPERTIES"]:
-					self.doebuild_settings.setcpv(mycpv, mydb=metadata)
-					metadata['USE'] = self.doebuild_settings['PORTAGE_USE']
+			for repo in repos:
+				metadata.clear()
 				try:
-					if self.settings._getMissingLicenses(mycpv, metadata):
+					metadata.update(zip(aux_keys,
+						self.aux_get(mycpv, aux_keys, myrepo=repo)))
+				except KeyError:
+					continue
+				except PortageException as e:
+					writemsg("!!! Error: aux_get('%s', %s)\n" %
+						(mycpv, aux_keys), noiselevel=-1)
+					writemsg("!!! %s\n" % (e,), noiselevel=-1)
+					del e
+					continue
+				eapi = metadata["EAPI"]
+				if not eapi_is_supported(eapi):
+					continue
+				if _eapi_is_deprecated(eapi):
+					continue
+				if not metadata["SLOT"]:
+					continue
+				if getMaskAtom(mycpv, metadata):
+					continue
+				if self.settings._getMissingKeywords(mycpv, metadata):
+					continue
+				if local_config:
+					metadata['CHOST'] = chost
+					if not accept_chost(mycpv, metadata):
 						continue
-					if self.settings._getMissingProperties(mycpv, metadata):
+					metadata["USE"] = ""
+					if "?" in metadata["LICENSE"] or \
+						"?" in metadata["PROPERTIES"]:
+						self.doebuild_settings.setcpv(mycpv, mydb=metadata)
+						metadata['USE'] = \
+							self.doebuild_settings['PORTAGE_USE']
+					try:
+						if self.settings._getMissingLicenses(
+							mycpv, metadata):
+							continue
+						if self.settings._getMissingProperties(
+							mycpv, metadata):
+							continue
+					except InvalidDependString:
 						continue
-				except InvalidDependString:
-					continue
-			yield mycpv
+
+				yield mycpv
+				# only yield a given cpv once
+				break
 
 def close_portdbapi_caches():
 	for i in portdbapi.portdbapi_instances:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f28353bdfb47855f14268edb6361f58577799136
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 18:29:35 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 18:29:35 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f28353bd

pordbapi: split _iter_visible method from visible

---
 pym/portage/dbapi/porttree.py |   24 ++++++++++++++----------
 1 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index ba0da34..e4cf9a6 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -875,16 +875,16 @@ class portdbapi(dbapi):
 		elif level == "match-visible":
 			# find all visible matches
 			if mydep.repo is not None or len(self.porttrees) == 1:
-				myval = self.visible(
+				myval = list(self._iter_visible(
 					self.xmatch("match-all", mydep),
-					mytree=mytree)
+					mytree=mytree))
 			else:
 				myval = set()
 				# We iterate over self.porttrees, since it's common to
 				# tweak this attribute in order to adjust match behavior.
 				for tree in self.porttrees:
 					repo = self.repositories.get_name_for_location(tree)
-					myval.update(self.visible(
+					myval.update(self._iter_visible(
 						self.xmatch("match-all", mydep.with_repo(repo)),
 						mytree=tree))
 				myval = list(myval)
@@ -1024,21 +1024,26 @@ class portdbapi(dbapi):
 			DeprecationWarning, stacklevel=2)
 		return self.visible(mylist, mytree=mytree)
 
-	def visible(self, mylist, mytree=None):
+	def visible(self, cpv_iter, mytree=None):
 		"""
-		Return a new list containing only visible packages.
+		Return a list containing only visible packages.
 		"""
-
 		if mylist is None:
 			return []
-		newlist=[]
+
+		return list(self._iter_visible(iter(cpv_iter), mytree=mytree))
+
+	def _iter_visible(self, cpv_iter, mytree=None):
+		"""
+		Return a new list containing only visible packages.
+		"""
 		aux_keys = list(self._aux_cache_keys)
 		metadata = {}
 		local_config = self.settings.local_config
 		chost = self.settings.get('CHOST', '')
 		accept_chost = self.settings._accept_chost
 		getMaskAtom = self.settings._getMaskAtom
-		for mycpv in mylist:
+		for mycpv in cpv_iter:
 			metadata.clear()
 			try:
 				metadata.update(zip(aux_keys,
@@ -1077,8 +1082,7 @@ class portdbapi(dbapi):
 						continue
 				except InvalidDependString:
 					continue
-			newlist.append(mycpv)
-		return newlist
+			yield mycpv
 
 def close_portdbapi_caches():
 	for i in portdbapi.portdbapi_instances:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     64e7f6e3364cb5eceb9ccc1a1d915fe1f5cb810d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 24 18:13:14 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Sep 24 18:13:14 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=64e7f6e3

portdbapi: combine gvisible and visible methods

There's no need to have separate methods, and this allows us to avoid
using separate aux_get calls in each method.

---
 pym/portage/dbapi/porttree.py |   49 ++++++++++++++++------------------------
 1 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index ac58375..ba0da34 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -875,18 +875,18 @@ class portdbapi(dbapi):
 		elif level == "match-visible":
 			# find all visible matches
 			if mydep.repo is not None or len(self.porttrees) == 1:
-				myval = self.gvisible(self.visible(
+				myval = self.visible(
 					self.xmatch("match-all", mydep),
-					mytree=mytree), mytree=mytree)
+					mytree=mytree)
 			else:
 				myval = set()
 				# We iterate over self.porttrees, since it's common to
 				# tweak this attribute in order to adjust match behavior.
 				for tree in self.porttrees:
 					repo = self.repositories.get_name_for_location(tree)
-					myval.update(self.gvisible(self.visible(
+					myval.update(self.visible(
 						self.xmatch("match-all", mydep.with_repo(repo)),
-						mytree=tree), mytree=tree))
+						mytree=tree))
 				myval = list(myval)
 				if len(myval) > 1:
 					self._cpv_sort_ascending(myval)
@@ -1016,32 +1016,18 @@ class portdbapi(dbapi):
 	def match(self, mydep, use_cache=1):
 		return self.xmatch("match-visible", mydep)
 
-	def visible(self, mylist, mytree=None):
-		"""two functions in one.  Accepts a list of cpv values and uses the package.mask *and*
-		packages file to remove invisible entries, returning remaining items.  This function assumes
-		that all entries in mylist have the same category and package name."""
-		if not mylist:
-			return []
-
-		db_keys = ["repository", "SLOT"]
-		visible = []
-		getMaskAtom = self.settings._getMaskAtom
-		for cpv in mylist:
-			try:
-				metadata = dict(zip(db_keys,
-					self.aux_get(cpv, db_keys, mytree=mytree)))
-			except KeyError:
-				# masked by corruption
-				continue
-			if not metadata["SLOT"]:
-				continue
-			if getMaskAtom(cpv, metadata):
-				continue
-			visible.append(cpv)
-		return visible
-
 	def gvisible(self, mylist, mytree=None):
-		"strip out group-masked (not in current group) entries"
+		warnings.warn("The 'gvisible' method of "
+			"portage.dbapi.porttree.portdbapi "
+			"is deprecated, and the functionality "
+			"has been combined into the 'visible' method",
+			DeprecationWarning, stacklevel=2)
+		return self.visible(mylist, mytree=mytree)
+
+	def visible(self, mylist, mytree=None):
+		"""
+		Return a new list containing only visible packages.
+		"""
 
 		if mylist is None:
 			return []
@@ -1051,6 +1037,7 @@ class portdbapi(dbapi):
 		local_config = self.settings.local_config
 		chost = self.settings.get('CHOST', '')
 		accept_chost = self.settings._accept_chost
+		getMaskAtom = self.settings._getMaskAtom
 		for mycpv in mylist:
 			metadata.clear()
 			try:
@@ -1069,6 +1056,10 @@ class portdbapi(dbapi):
 				continue
 			if _eapi_is_deprecated(eapi):
 				continue
+			if not metadata["SLOT"]:
+				continue
+			if getMaskAtom(mycpv, metadata):
+				continue
 			if self.settings._getMissingKeywords(mycpv, metadata):
 				continue
 			if local_config:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0d375f1105ad67c8b7e3b16b57cdbb367f99cd69
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Sep 23 20:50:45 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Sep 23 20:50:45 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0d375f11

portdbapi.xmatch: deprecate *match-list

These don't mesh well with multi-repo support.

---
 pym/portage/dbapi/porttree.py |   24 +++++++++++++++---------
 1 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index d42e41e..ac58375 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -989,23 +989,29 @@ class portdbapi(dbapi):
 
 		elif level == "bestmatch-list":
 			#dep match -- find best match but restrict search to sublist
-			#no point in calling xmatch again since we're not caching list deps
-
+			warnings.warn("The 'bestmatch-list' mode of "
+				"portage.dbapi.porttree.portdbapi.xmatch is deprecated",
+				DeprecationWarning, stacklevel=2)
 			myval = best(list(self._iter_match(mydep, mylist)))
 		elif level == "match-list":
 			#dep match -- find all matches but restrict search to sublist (used in 2nd half of visible())
-
+			warnings.warn("The 'match-list' mode of "
+				"portage.dbapi.porttree.portdbapi.xmatch is deprecated",
+				DeprecationWarning, stacklevel=2)
 			myval = list(self._iter_match(mydep, mylist))
-
 		else:
 			raise AssertionError(
 				"Invalid level argument: '%s'" % level)
 
-		if self.frozen and (level not in ["match-list", "bestmatch-list"]):
-			self.xcache[level][mydep] = myval
-			if origdep and origdep != mydep:
-				self.xcache[level][origdep] = myval
-		return myval[:]
+		if self.frozen:
+			xcache_this_level = self.xcache.get(level)
+			if xcache_this_level is not None:
+				xcache_this_level[mydep] = myval
+				if origdep and origdep != mydep:
+					xcache_this_level[origdep] = myval
+				myval = myval[:]
+
+		return myval
 
 	def match(self, mydep, use_cache=1):
 		return self.xmatch("match-visible", mydep)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8be9e967be8e0c038245f55ec927da5b1caa5d54
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Sep 23 20:00:57 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Sep 23 20:00:57 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8be9e967

portdbapi.xmatch: fix minimum-all for multi-repo

---
 pym/portage/dbapi/porttree.py |   23 ++++++++++++++++-------
 1 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 1354124..d42e41e 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -895,15 +895,24 @@ class portdbapi(dbapi):
 			# Find the minimum matching version. This is optimized to
 			# minimize the number of metadata accesses (improves performance
 			# especially in cases where metadata needs to be generated).
+			myval = ""
 			if mydep == mykey:
-				cpv_iter = iter(self.cp_list(mykey, mytree=mytree))
+				for myval in self.cp_list(mykey, mytree=mytree):
+					break
+			elif mydep.repo is not None or len(self.porttrees) == 1:
+				for myval in self._iter_match(mydep,
+					self.cp_list(mykey, mytree=mytree)):
+					break
 			else:
-				cpv_iter = self._iter_match(mydep,
-					self.cp_list(mykey, mytree=mytree))
-			try:
-				myval = next(cpv_iter)
-			except StopIteration:
-				myval = ""
+				# We iterate over self.porttrees, since it's common to
+				# tweak this attribute in order to adjust match behavior.
+				for tree in self.porttrees:
+					repo = self.repositories.get_name_for_location(tree)
+					for myval in self._iter_match(mydep.with_repo(repo),
+						self.cp_list(mykey, mytree=tree)):
+						break
+					if myval:
+						break
 
 		elif level in ("minimum-visible", "bestmatch-visible"):
 			# Find the minimum matching visible version. This is optimized to

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0666aa9e6c88cdeaf5806dda27e40eb96df9db86
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Sep 23 19:52:58 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Sep 23 19:52:58 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0666aa9e

Fix cp_list mytree argument for match-all.

This is a regression from commit
3d07f74a2fb2e0480e126335f6981218f60da824.

---
 pym/portage/dbapi/porttree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index e3fe9a9..1354124 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -867,7 +867,7 @@ class portdbapi(dbapi):
 				for tree in self.porttrees:
 					repo = self.repositories.get_name_for_location(tree)
 					myval.update(self._iter_match(mydep.with_repo(repo),
-					self.cp_list(mykey, mytree=mytree)))
+					self.cp_list(mykey, mytree=tree)))
 				myval = list(myval)
 				if len(myval) > 1:
 					self._cpv_sort_ascending(myval)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3d07f74a2fb2e0480e126335f6981218f60da824
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Sep 23 18:19:13 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Sep 23 18:19:13 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3d07f74a

portdbapi.xmatch: fix match-all for multi-repo

---
 pym/portage/dbapi/porttree.py |   26 +++++++++++++++++++-------
 1 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index eb7e36a..e3fe9a9 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -853,6 +853,25 @@ class portdbapi(dbapi):
 				myval = match_from_list(mydep,
 					self.cp_list(mykey, mytree=mytree))
 
+		elif level == "match-all":
+			# match *all* visible *and* masked packages
+			if mydep == mykey:
+				myval = self.cp_list(mykey, mytree=mytree)
+			elif mydep.repo is not None or len(self.porttrees) == 1:
+				myval = list(self._iter_match(mydep,
+					self.cp_list(mykey, mytree=mytree)))
+			else:
+				myval = set()
+				# We iterate over self.porttrees, since it's common to
+				# tweak this attribute in order to adjust match behavior.
+				for tree in self.porttrees:
+					repo = self.repositories.get_name_for_location(tree)
+					myval.update(self._iter_match(mydep.with_repo(repo),
+					self.cp_list(mykey, mytree=mytree)))
+				myval = list(myval)
+				if len(myval) > 1:
+					self._cpv_sort_ascending(myval)
+
 		elif level == "match-visible":
 			# find all visible matches
 			if mydep.repo is not None or len(self.porttrees) == 1:
@@ -969,13 +988,6 @@ class portdbapi(dbapi):
 
 			myval = list(self._iter_match(mydep, mylist))
 
-		elif level == "match-all":
-			#match *all* visible *and* masked packages
-			if mydep == mykey:
-				myval = self.cp_list(mykey, mytree=mytree)
-			else:
-				myval = list(self._iter_match(mydep,
-					self.cp_list(mykey, mytree=mytree)))
 		else:
 			raise AssertionError(
 				"Invalid level argument: '%s'" % level)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1b4ee4f4c5c01c49b91f1618f4b2ab6305f28eca
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Sep 23 01:47:04 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Sep 23 01:47:04 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1b4ee4f4

Fix minimum/bestmatch-visible for multi-repo.

---
 pym/portage/dbapi/porttree.py |   83 +++++++++++++++++++++++++---------------
 1 files changed, 52 insertions(+), 31 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 7f1bbf0..4f5dc78 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -898,41 +898,62 @@ class portdbapi(dbapi):
 				iterfunc = iter
 			else:
 				iterfunc = reversed
+
+			if mydep.repo is not None or len(self.porttrees) == 1:
+				repos = [mydep.repo]
+			else:
+				# We iterate over self.porttrees, since it's common to
+				# tweak this attribute in order to adjust match behavior.
+				repos = []
+				for tree in reversed(self.porttrees):
+					repos.append(self.repositories.get_name_for_location(tree))
+
 			for cpv in iterfunc(mylist):
-				try:
-					metadata = dict(zip(aux_keys,
-						self.aux_get(cpv, aux_keys)))
-				except KeyError:
-					# ebuild masked by corruption
-					continue
-				if not eapi_is_supported(metadata["EAPI"]):
-					continue
-				if mydep.slot and mydep.slot != metadata["SLOT"]:
-					continue
-				if settings._getMissingKeywords(cpv, metadata):
-					continue
-				if settings._getMaskAtom(cpv, metadata):
-					continue
-				if local_config:
-					metadata["USE"] = ""
-					if "?" in metadata["LICENSE"] or "?" in metadata["PROPERTIES"]:
-						self.doebuild_settings.setcpv(cpv, mydb=metadata)
-						metadata["USE"] = self.doebuild_settings.get("USE", "")
+				for repo in repos:
 					try:
-						if settings._getMissingLicenses(cpv, metadata):
-							continue
-						if settings._getMissingProperties(cpv, metadata):
-							continue
-					except InvalidDependString:
+						metadata = dict(zip(aux_keys,
+							self.aux_get(cpv, aux_keys, myrepo=repo)))
+					except KeyError:
+						# ebuild not in this repo, or masked by corruption
 						continue
-				if mydep.use:
-					has_iuse = False
-					for has_iuse in self._iter_match_use(mydep, [cpv]):
-						break
-					if not has_iuse:
+					if not eapi_is_supported(metadata["EAPI"]):
 						continue
-				myval = cpv
-				break
+					if mydep.slot is not None and \
+						mydep.slot != metadata["SLOT"]:
+						continue
+					if settings._getMissingKeywords(cpv, metadata):
+						continue
+					if settings._getMaskAtom(cpv, metadata):
+						continue
+					if local_config:
+						metadata["USE"] = ""
+						if "?" in metadata["LICENSE"] or \
+							"?" in metadata["PROPERTIES"]:
+							self.doebuild_settings.setcpv(cpv, mydb=metadata)
+							metadata["USE"] = \
+								self.doebuild_settings.get("PORTAGE_USE", "")
+						try:
+							if settings._getMissingLicenses(cpv, metadata):
+								continue
+							if settings._getMissingProperties(cpv, metadata):
+								continue
+						except InvalidDependString:
+							continue
+					if mydep.use is not None:
+						mydep_with_repo = mydep
+						if repo is not None and mydep.repo is None:
+							mydep_with_repo = mydep.with_repo(repo)
+						has_iuse = False
+						for has_iuse in self._iter_match_use(
+							mydep_with_repo, [cpv]):
+							break
+						if has_iuse is False:
+							continue
+					myval = cpv
+					break
+				if myval:
+					break
+
 		elif level == "bestmatch-list":
 			#dep match -- find best match but restrict search to sublist
 			#no point in calling xmatch again since we're not caching list deps

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f4f78a6a7c90fb25f2937dba3422774e5f2b535c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Sep 23 00:53:45 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Sep 23 00:53:45 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f4f78a6a

portdbapi: handle multi-repo visiblity

This will fix bug #384063. The xmatch list-visible mode was previously
used to cache a list of all visible cpvs for a given cp, but this is
not useful when there can be duplicate cpvs with different visibility
from multiple repos.

---
 pym/portage/dbapi/porttree.py |   45 ++++++++++++++++++++++++++--------------
 1 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index af2bca1..7f1bbf0 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -797,7 +797,7 @@ class portdbapi(dbapi):
 		return mylist
 
 	def freeze(self):
-		for x in "bestmatch-visible", "cp-list", "list-visible", "match-all", \
+		for x in "bestmatch-visible", "cp-list", "match-all", \
 			"match-all-cpv-only", "match-visible", "minimum-all", \
 			"minimum-visible":
 			self.xcache[x]={}
@@ -809,6 +809,8 @@ class portdbapi(dbapi):
 
 	def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):
 		"caching match function; very trick stuff"
+		if level == "list-visible":
+			level = "match-visible"
 		#if no updates are being made to the tree, we can consult our xcache...
 		if self.frozen:
 			try:
@@ -846,12 +848,25 @@ class portdbapi(dbapi):
 				myval = match_from_list(mydep,
 					self.cp_list(mykey, mytree=mytree))
 
-		elif level == "list-visible":
-			#a list of all visible packages, not called directly (just by xmatch())
-			#myval = self.visible(self.cp_list(mykey))
+		elif level == "match-visible":
+			# find all visible matches
+			if mydep.repo is not None or len(self.porttrees) == 1:
+				myval = self.gvisible(self.visible(
+					self.xmatch("match-all", mydep),
+					mytree=mytree), mytree=mytree)
+			else:
+				myval = set()
+				# We iterate over self.porttrees, since it's common to
+				# tweak this attribute in order to adjust match behavior.
+				for tree in self.porttrees:
+					repo = self.repositories.get_name_for_location(tree)
+					myval.update(self.gvisible(self.visible(
+						self.xmatch("match-all", mydep.with_repo(repo)),
+						mytree=tree), mytree=tree))
+				myval = list(myval)
+				if len(myval) > 1:
+					self._cpv_sort_ascending(myval)
 
-			myval = self.gvisible(self.visible(
-				self.cp_list(mykey, mytree=mytree)))
 		elif level == "minimum-all":
 			# Find the minimum matching version. This is optimized to
 			# minimize the number of metadata accesses (improves performance
@@ -927,11 +942,7 @@ class portdbapi(dbapi):
 			#dep match -- find all matches but restrict search to sublist (used in 2nd half of visible())
 
 			myval = list(self._iter_match(mydep, mylist))
-		elif level == "match-visible":
-			#dep match -- find all visible matches
-			#get all visible packages, then get the matching ones
-			myval = list(self._iter_match(mydep,
-				self.xmatch("list-visible", mykey, mydep=Atom(mykey), mykey=mykey)))
+
 		elif level == "match-all":
 			#match *all* visible *and* masked packages
 			if mydep == mykey:
@@ -952,19 +963,20 @@ class portdbapi(dbapi):
 	def match(self, mydep, use_cache=1):
 		return self.xmatch("match-visible", mydep)
 
-	def visible(self, mylist):
+	def visible(self, mylist, mytree=None):
 		"""two functions in one.  Accepts a list of cpv values and uses the package.mask *and*
 		packages file to remove invisible entries, returning remaining items.  This function assumes
 		that all entries in mylist have the same category and package name."""
 		if not mylist:
 			return []
 
-		db_keys = ["SLOT"]
+		db_keys = ["repository", "SLOT"]
 		visible = []
 		getMaskAtom = self.settings._getMaskAtom
 		for cpv in mylist:
 			try:
-				metadata = dict(zip(db_keys, self.aux_get(cpv, db_keys)))
+				metadata = dict(zip(db_keys,
+					self.aux_get(cpv, db_keys, mytree=mytree)))
 			except KeyError:
 				# masked by corruption
 				continue
@@ -975,7 +987,7 @@ class portdbapi(dbapi):
 			visible.append(cpv)
 		return visible
 
-	def gvisible(self,mylist):
+	def gvisible(self, mylist, mytree=None):
 		"strip out group-masked (not in current group) entries"
 
 		if mylist is None:
@@ -989,7 +1001,8 @@ class portdbapi(dbapi):
 		for mycpv in mylist:
 			metadata.clear()
 			try:
-				metadata.update(zip(aux_keys, self.aux_get(mycpv, aux_keys)))
+				metadata.update(zip(aux_keys,
+					self.aux_get(mycpv, aux_keys, mytree=mytree)))
 			except KeyError:
 				continue
 			except PortageException as e:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     37d9c3df6b25f10a78ff216875bb481e0171fae0
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Sep  7 17:36:37 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Sep  7 17:36:37 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=37d9c3df

pordbapi: add unshared cache mode for egencache

---
 pym/portage/dbapi/porttree.py |   33 +++++++++++++++++++++++++++------
 1 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..ac972c4 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -154,11 +154,32 @@ class portdbapi(dbapi):
 		self.auxdb = {}
 		self._pregen_auxdb = {}
 		self._init_cache_dirs()
-		depcachedir_w_ok = os.access(self.depcachedir, os.W_OK)
-		cache_kwargs = {
-			'gid'     : portage_gid,
-			'perms'   : 0o664
-		}
+		try:
+			depcachedir_st = os.stat(self.depcachedir)
+			depcachedir_w_ok = os.access(self.depcachedir, os.W_OK)
+		except OSError:
+			depcachedir_st = None
+			depcachedir_w_ok = False
+
+		cache_kwargs = {}
+
+		depcachedir_unshared = False
+		if portage.data.secpass < 1 and \
+			depcachedir_w_ok and \
+			depcachedir_st is not None and \
+			os.getuid() == depcachedir_st.st_uid and \
+			os.getgid() == depcachedir_st.st_gid:
+			# If this user owns depcachedir and is not in the
+			# portage group, then don't bother to set permissions
+			# on cache entries. This makes it possible to run
+			# egencache without any need to be a member of the
+			# portage group.
+			depcachedir_unshared = True
+		else:
+			cache_kwargs.update({
+				'gid'     : portage_gid,
+				'perms'   : 0o664
+			})
 
 		# XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
 		# ~harring
@@ -167,7 +188,7 @@ class portdbapi(dbapi):
 		# If secpass < 1, we don't want to write to the cache
 		# since then we won't be able to apply group permissions
 		# to the cache entries/directories.
-		if secpass < 1 or not depcachedir_w_ok:
+		if (secpass < 1 and not depcachedir_unshared) or not depcachedir_w_ok:
 			for x in self.porttrees:
 				try:
 					db_ro = self.auxdbmodule(self.depcachedir, x,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d318bcce356b55d481e78b8cf511d6698f67eee5
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Sep  2 15:06:58 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Sep  2 15:06:58 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d318bcce

vartree: add missing _merge_unicode_error import

---
 pym/portage/dbapi/vartree.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 70816b1..ba2cc08 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -18,7 +18,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.locks:lockdir,unlockdir,lockfile,unlockfile',
 	'portage.output:bold,colorize',
 	'portage.package.ebuild.doebuild:doebuild_environment,' + \
-		'_spawn_phase',
+		'_merge_unicode_error', '_spawn_phase',
 	'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
 	'portage.update:fixdbentries',
 	'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \
@@ -3352,7 +3352,7 @@ class dblink(object):
 				break
 
 		if unicode_errors:
-			eerror(portage._merge_unicode_error(unicode_errors))
+			eerror(_merge_unicode_error(unicode_errors))
 
 		if paths_with_newlines:
 			msg = []

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5e15e5830a915c614cd81f6a87b2e73ac116368d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Sep  1 08:43:03 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Sep  1 08:43:03 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5e15e583

merge: start at correct EPREFIX offset

---
 pym/portage/dbapi/vartree.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 0c0ed6a..70816b1 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3872,7 +3872,8 @@ class dblink(object):
 
 		# we do a first merge; this will recurse through all files in our srcroot but also build up a
 		# "second hand" of symlinks to merge later
-		if self.mergeme(srcroot, destroot, outfile, secondhand, "", cfgfiledict, mymtime):
+		if self.mergeme(srcroot, destroot, outfile, secondhand,
+			self.settings["EPREFIX"].lstrip(os.sep), cfgfiledict, mymtime):
 			return 1
 
 		# now, it's time for dealing our second hand; we'll loop until we can't merge anymore.	The rest are

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0d79d130c27af4ba9676c9297639e064d18d9528
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Sep  1 06:45:01 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Sep  1 06:45:01 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0d79d130

unmerge: never unmerge outside of EROOT

---
 pym/portage/dbapi/vartree.py |    9 ++++++++-
 1 files changed, 8 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 1a38d42..0c0ed6a 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1980,10 +1980,11 @@ class dblink(object):
 			unmerge_desc["!mtime"] = _("!mtime")
 			unmerge_desc["!obj"] = _("!obj")
 			unmerge_desc["!sym"] = _("!sym")
+			unmerge_desc["!prefix"] = _("!prefix")
 
 			real_root = self.settings['ROOT']
 			real_root_len = len(real_root) - 1
-			eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1
+			eroot = self.settings["EROOT"]
 
 			# These files are generated by emerge, so we need to remove
 			# them when they are the only thing left in a directory.
@@ -2023,6 +2024,12 @@ class dblink(object):
 
 				file_data = pkgfiles[objkey]
 				file_type = file_data[0]
+
+				# don't try to unmerge the prefix offset itself
+				if len(obj) <= len(eroot) or not obj.startswith(eroot):
+					show_unmerge("---", unmerge_desc["!prefix"], file_type, obj)
+					continue
+
 				statobj = None
 				try:
 					statobj = os.stat(obj)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     290108eeb6ddf01efcafcca9bdb9e109ce9af009
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Aug 30 02:09:13 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Aug 30 02:09:13 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=290108ee

dblink.treewalk: remove dowgrade check

This code is unused since commit
a37eb8ebd2fad3f8074490a061f067e2c637f05d.

---
 pym/portage/dbapi/vartree.py |    9 ---------
 1 files changed, 0 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index afcbceb..1a38d42 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3231,10 +3231,6 @@ class dblink(object):
 		if not os.path.exists(self.dbcatdir):
 			ensure_dirs(self.dbcatdir)
 
-		otherversions = []
-		for v in self.vartree.dbapi.cp_list(self.mysplit[0]):
-			otherversions.append(v.split("/")[1])
-
 		cp = self.mysplit[0]
 		slot_atom = "%s:%s" % (cp, slot)
 
@@ -3805,11 +3801,6 @@ class dblink(object):
 			showMessage(_("!!! FAILED postinst: ")+str(a)+"\n",
 				level=logging.ERROR, noiselevel=-1)
 
-		downgrade = False
-		for v in otherversions:
-			if pkgcmp(catpkgsplit(self.pkg)[1:], catpkgsplit(v)[1:]) < 0:
-				downgrade = True
-
 		# Lock the config memory file to prevent symlink creation
 		# in merge_contents from overlapping with env-update.
 		self.vartree.dbapi._fs_lock()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     2c92ae3c7ddb903bcffe96046cebe8106439e86b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Aug  6 11:09:51 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Aug  6 11:09:51 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2c92ae3c

unmerge: remove more info dir files for #323213

Treat any directory named "info" as a candidate here, since it might
have been in INFOPATH previously even though it may not be there now.

---
 pym/portage/dbapi/vartree.py |    6 +++++-
 1 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 94b7936..7f7873b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2220,7 +2220,11 @@ class dblink(object):
 			mydirs.reverse()
 
 			for obj, inode_key in mydirs:
-				if inode_key in infodirs_inodes:
+				# Treat any directory named "info" as a candidate here,
+				# since it might have been in INFOPATH previously even
+				# though it may not be there now.
+				if inode_key in infodirs_inodes or \
+					os.path.basename(obj) == "info":
 					try:
 						remaining = os.listdir(obj)
 					except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3acce846c43eed7feefd3ab9c47dd172a83ae1db
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Aug  4 23:09:36 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Aug  4 23:09:36 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3acce846

unmerge: remove generated info "dir" files

These files are generated by emerge, so we need to remove them
when they are the only thing left in the directory. This will fix
bug #323213.

---
 pym/portage/dbapi/vartree.py |   40 ++++++++++++++++++++++++++++++++++++++++
 1 files changed, 40 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index db44491..94b7936 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1985,6 +1985,22 @@ class dblink(object):
 			real_root_len = len(real_root) - 1
 			eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1
 
+			# These files are generated by emerge, so we need to remove
+			# them when they are the only thing left in a directory.
+			infodir_cleanup = frozenset(["dir", "dir.old"])
+			infodirs = frozenset(infodir for infodir in chain(
+				self.settings.get("INFOPATH", "").split(":"),
+				self.settings.get("INFODIR", "").split(":")) if infodir)
+			infodirs_inodes = set()
+			for infodir in infodirs:
+				infodir = os.path.join(real_root, infodir.lstrip(os.sep))
+				try:
+					statobj = os.stat(infodir)
+				except OSError:
+					pass
+				else:
+					infodirs_inodes.add((statobj.st_dev, statobj.st_ino))
+
 			for i, objkey in enumerate(mykeys):
 
 				obj = normalize_path(objkey)
@@ -2204,6 +2220,30 @@ class dblink(object):
 			mydirs.reverse()
 
 			for obj, inode_key in mydirs:
+				if inode_key in infodirs_inodes:
+					try:
+						remaining = os.listdir(obj)
+					except OSError:
+						pass
+					else:
+						cleanup_info_dir = ()
+						if remaining and \
+							len(remaining) <= len(infodir_cleanup):
+							if not set(remaining).difference(infodir_cleanup):
+								cleanup_info_dir = remaining
+
+						for child in cleanup_info_dir:
+							child = os.path.join(obj, child)
+							try:
+								lstatobj = os.lstat(child)
+								if stat.S_ISREG(lstatobj.st_mode):
+									unlink(child, lstatobj)
+									show_unmerge("<<<", "", "obj", child)
+							except EnvironmentError as e:
+								if e.errno not in ignored_unlink_errnos:
+									raise
+								del e
+								show_unmerge("!!!", "", "obj", child)
 				try:
 					if bsd_chflags:
 						lstatobj = os.lstat(obj)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     ac3721418569682b4b3d48a212a475a8ec452b9b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Aug  2 08:56:11 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Aug  2 08:56:11 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ac372141

unmerge: display preserved syms in ascending ord

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 58584d7..db44491 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2253,7 +2253,7 @@ class dblink(object):
 			lines.append("")
 			flat_list = set()
 			flat_list.update(*protected_symlinks.values())
-			flat_list = sorted(flat_list, reverse=True)
+			flat_list = sorted(flat_list)
 			for f in flat_list:
 				lines.append("\t%s" % (os.path.join(real_root,
 					f.lstrip(os.sep))))

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     40f45506b5971073c8a60f172c3171eaf9cad3a6
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  1 12:32:00 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  1 12:32:00 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=40f45506

unmerge: unmerge symlinks broken by rmdir

This prevents some more false alarms from commit
4bb08136f073024c5d31dceb1618b6f4e7246369.

---
 pym/portage/dbapi/vartree.py |   36 +++++++++++++++++++++++++++++-------
 1 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 6317af7..58584d7 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1909,7 +1909,7 @@ class dblink(object):
 
 		cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file)
 		stale_confmem = []
-		protected_symlinks = []
+		protected_symlinks = {}
 
 		unmerge_orphans = "unmerge-orphans" in self.settings.features
 		calc_prelink = "prelink-checksums" in self.settings.features
@@ -2054,7 +2054,9 @@ class dblink(object):
 								break
 
 						if symlink_orphan:
-							protected_symlinks.append(relative_path)
+							protected_symlinks.setdefault(
+								(statobj.st_dev, statobj.st_ino),
+								[]).append(relative_path)
 
 					if is_owned:
 						show_unmerge("---", unmerge_desc["replaced"], file_type, obj)
@@ -2098,7 +2100,7 @@ class dblink(object):
 					if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode):
 						show_unmerge("---", unmerge_desc["!dir"], file_type, obj)
 						continue
-					mydirs.add(obj)
+					mydirs.add((obj, (lstatobj.st_dev, lstatobj.st_ino)))
 				elif pkgfiles[objkey][0] == "sym":
 					if not islink:
 						show_unmerge("---", unmerge_desc["!sym"], file_type, obj)
@@ -2144,7 +2146,9 @@ class dblink(object):
 										break
 
 								if not all_owned:
-									protected_symlinks.append(relative_path)
+									protected_symlinks.setdefault(
+										(statobj.st_dev, statobj.st_ino),
+										[]).append(relative_path)
 									show_unmerge("---", unmerge_desc["!empty"],
 										file_type, obj)
 									continue
@@ -2199,7 +2203,7 @@ class dblink(object):
 			mydirs = sorted(mydirs)
 			mydirs.reverse()
 
-			for obj in mydirs:
+			for obj, inode_key in mydirs:
 				try:
 					if bsd_chflags:
 						lstatobj = os.lstat(obj)
@@ -2224,6 +2228,22 @@ class dblink(object):
 					if e.errno != errno.ENOENT:
 						show_unmerge("---", unmerge_desc["!empty"], "dir", obj)
 					del e
+				else:
+					# When a directory is successfully removed, there's
+					# no need to protect symlinks that point to it.
+					unmerge_syms = protected_symlinks.pop(inode_key, None)
+					if unmerge_syms is not None:
+						for relative_path in unmerge_syms:
+							obj = os.path.join(real_root,
+								relative_path.lstrip(os.sep))
+							try:
+								unlink(obj, os.lstat(obj))
+								show_unmerge("<<<", "", "sym", obj)
+							except (OSError, IOError) as e:
+								if e.errno not in ignored_unlink_errnos:
+									raise
+								del e
+								show_unmerge("!!!", "", "sym", obj)
 
 		if protected_symlinks:
 			msg = "One or more symlinks to directories have been " + \
@@ -2231,8 +2251,10 @@ class dblink(object):
 				"via these symlinks remain accessible:"
 			lines = textwrap.wrap(msg, 72)
 			lines.append("")
-			protected_symlinks.reverse()
-			for f in protected_symlinks:
+			flat_list = set()
+			flat_list.update(*protected_symlinks.values())
+			flat_list = sorted(flat_list, reverse=True)
+			for f in flat_list:
 				lines.append("\t%s" % (os.path.join(real_root,
 					f.lstrip(os.sep))))
 			lines.append("")

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     38ab9edcb86df7c2ab057013e40c7389c6c51b70
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 29 20:16:34 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul 29 20:16:34 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=38ab9edc

unmerge: fix false alarm about protected symlinks

The error message about preserved symlinks, from commit
4bb08136f073024c5d31dceb1618b6f4e7246369, should only be triggered in
cases when the package being unmerged has the given path listed as a
symlink in its CONTENTS.

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index e85e0b8..6317af7 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2030,7 +2030,7 @@ class dblink(object):
 							is_owned = True
 							break
 
-					if is_owned and \
+					if file_type == "sym" and is_owned and \
 						(islink and statobj and stat.S_ISDIR(statobj.st_mode)):
 						# A new instance of this package claims the file, so
 						# don't unmerge it. If the file is symlink to a

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     bca981ba89773059498c44e68896785160b010a4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 29 07:22:19 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul 29 07:22:19 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=bca981ba

Fix a typo in a comment.

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 6bb68d3..e85e0b8 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2104,7 +2104,7 @@ class dblink(object):
 						show_unmerge("---", unmerge_desc["!sym"], file_type, obj)
 						continue
 
-					# If this symlink points to a direcory then we don't want
+					# If this symlink points to a directory then we don't want
 					# to unmerge it if there are any other packages that
 					# installed files into the directory via this symlink
 					# (see bug #326685).

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4bb08136f073024c5d31dceb1618b6f4e7246369
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 28 11:29:25 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jul 28 11:29:25 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4bb08136

emerge: protect symlinks to directories sometimes

Before, it was possible to unmerge a symlink to a directory, such that
files installed via the path of the symlink could become inaccessible
via that path (and also making it impossible to unmerge them via that
path).

Now, the symlink will only be unmerged if the directory that it points
to only contains regular files which are all being unmerged. In any
other case, the symlink will be preserved and an eerror log message
will record the event. This will give the user an opportunity to take
further action if they deem it necessary, and such symlink preservation
will not be silent as it was reported in bug #326685, comment #3.

---
 pym/portage/dbapi/vartree.py |   90 +++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 88 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 47f5eb2..6bb68d3 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -70,6 +70,7 @@ import shutil
 import stat
 import sys
 import tempfile
+import textwrap
 import time
 import warnings
 
@@ -1908,6 +1909,7 @@ class dblink(object):
 
 		cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file)
 		stale_confmem = []
+		protected_symlinks = []
 
 		unmerge_orphans = "unmerge-orphans" in self.settings.features
 		calc_prelink = "prelink-checksums" in self.settings.features
@@ -2027,9 +2029,34 @@ class dblink(object):
 						if dblnk.isowner(relative_path):
 							is_owned = True
 							break
-					if is_owned:
+
+					if is_owned and \
+						(islink and statobj and stat.S_ISDIR(statobj.st_mode)):
 						# A new instance of this package claims the file, so
-						# don't unmerge it.
+						# don't unmerge it. If the file is symlink to a
+						# directory and the unmerging package installed it as
+						# a symlink, but the new owner has it listed as a
+						# directory, then we'll produce a warning since the
+						# symlink is a sort of orphan in this case (see
+						# bug #326685).
+						symlink_orphan = False
+						for dblnk in others_in_slot:
+							parent_contents_key = \
+								dblnk._match_contents(relative_path)
+							if not parent_contents_key:
+								continue
+							if not parent_contents_key.startswith(
+								real_root):
+								continue
+							if dblnk.getcontents()[
+								parent_contents_key][0] == "dir":
+								symlink_orphan = True
+								break
+
+						if symlink_orphan:
+							protected_symlinks.append(relative_path)
+
+					if is_owned:
 						show_unmerge("---", unmerge_desc["replaced"], file_type, obj)
 						continue
 					elif relative_path in cfgfiledict:
@@ -2076,6 +2103,52 @@ class dblink(object):
 					if not islink:
 						show_unmerge("---", unmerge_desc["!sym"], file_type, obj)
 						continue
+
+					# If this symlink points to a direcory then we don't want
+					# to unmerge it if there are any other packages that
+					# installed files into the directory via this symlink
+					# (see bug #326685).
+					# TODO: Resolving a symlink to a directory will require
+					# simulation if $ROOT != / and the link is not relative.
+					if islink and statobj and stat.S_ISDIR(statobj.st_mode) \
+						and obj.startswith(real_root):
+
+						relative_path = obj[real_root_len:]
+						try:
+							target_dir_contents = os.listdir(obj)
+						except OSError:
+							pass
+						else:
+							if target_dir_contents:
+								# If all the children are regular files owned
+								# by this package, then the symlink should be
+								# safe to unmerge.
+								all_owned = True
+								for child in target_dir_contents:
+									child = os.path.join(relative_path, child)
+									if not self.isowner(child):
+										all_owned = False
+										break
+									try:
+										child_lstat = os.lstat(os.path.join(
+											real_root, child.lstrip(os.sep)))
+									except OSError:
+										continue
+
+									if not stat.S_ISREG(child_lstat.st_mode):
+										# Nested symlinks or directories make
+										# the issue very complex, so just
+										# preserve the symlink in order to be
+										# on the safe side.
+										all_owned = False
+										break
+
+								if not all_owned:
+									protected_symlinks.append(relative_path)
+									show_unmerge("---", unmerge_desc["!empty"],
+										file_type, obj)
+									continue
+
 					# Go ahead and unlink symlinks to directories here when
 					# they're actually recorded as symlinks in the contents.
 					# Normally, symlinks such as /lib -> lib64 are not recorded
@@ -2152,6 +2225,19 @@ class dblink(object):
 						show_unmerge("---", unmerge_desc["!empty"], "dir", obj)
 					del e
 
+		if protected_symlinks:
+			msg = "One or more symlinks to directories have been " + \
+				"preserved in order to ensure that files installed " + \
+				"via these symlinks remain accessible:"
+			lines = textwrap.wrap(msg, 72)
+			lines.append("")
+			protected_symlinks.reverse()
+			for f in protected_symlinks:
+				lines.append("\t%s" % (os.path.join(real_root,
+					f.lstrip(os.sep))))
+			lines.append("")
+			self._elog("eerror", "postrm", lines)
+
 		# Remove stale entries from config memory.
 		if stale_confmem:
 			for filename in stale_confmem:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a4a4d87786e6c0c34370870af2071904d6184a9d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 27 01:51:25 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jul 27 01:51:25 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a4a4d877

merge: abort if symlink replacing dir

This is required for compliance with PMS section 13.4 as discussed in
bug #326685.

---
 pym/portage/dbapi/vartree.py |   38 ++++++++++++++++++++++++++++++--------
 1 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c703415..47f5eb2 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -62,6 +62,7 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
 import errno
 import gc
 import io
+from itertools import chain
 import logging
 import os as _os
 import re
@@ -2699,7 +2700,8 @@ class dblink(object):
 
 		self.vartree.dbapi._plib_registry.pruneNonExisting()
 
-	def _collision_protect(self, srcroot, destroot, mypkglist, mycontents):
+	def _collision_protect(self, srcroot, destroot, mypkglist,
+		file_list, symlink_list):
 
 			os = _os_merge
 
@@ -2729,10 +2731,13 @@ class dblink(object):
 			showMessage = self._display_merge
 			stopmerge = False
 			collisions = []
+			symlink_collisions = []
 			destroot = self.settings['ROOT']
 			showMessage(_(" %s checking %d files for package collisions\n") % \
-				(colorize("GOOD", "*"), len(mycontents)))
-			for i, f in enumerate(mycontents):
+				(colorize("GOOD", "*"), len(file_list) + len(symlink_list)))
+			for i, (f, f_type) in enumerate(chain(
+				((f, "reg") for f in file_list),
+				((f, "sym") for f in symlink_list))):
 				if i % 1000 == 0 and i != 0:
 					showMessage(_("%d files checked ...\n") % i)
 
@@ -2772,6 +2777,14 @@ class dblink(object):
 				if f[0] != "/":
 					f="/"+f
 
+				if stat.S_ISDIR(dest_lstat.st_mode):
+					if f_type == "sym":
+						# This case is explicitly banned
+						# by PMS (see bug #326685).
+						symlink_collisions.append(f)
+						collisions.append(f)
+						continue
+
 				plibs = plib_inodes.get((dest_lstat.st_dev, dest_lstat.st_ino))
 				if plibs:
 					for path in plibs:
@@ -2806,7 +2819,7 @@ class dblink(object):
 									break
 					if stopmerge:
 						collisions.append(f)
-			return collisions, plib_collisions
+			return collisions, symlink_collisions, plib_collisions
 
 	def _lstat_inode_map(self, path_iter):
 		"""
@@ -3233,9 +3246,9 @@ class dblink(object):
 		blockers = self._blockers
 		if blockers is None:
 			blockers = []
-		collisions, plib_collisions = \
+		collisions, symlink_collisions, plib_collisions = \
 			self._collision_protect(srcroot, destroot,
-			others_in_slot + blockers, myfilelist + mylinklist)
+			others_in_slot + blockers, myfilelist, mylinklist)
 
 		# Make sure the ebuild environment is initialized and that ${T}/elog
 		# exists for logging of collision-protect eerror messages.
@@ -3294,7 +3307,7 @@ class dblink(object):
 			eerror(msg)
 
 			owners = None
-			if collision_protect or protect_owned:
+			if collision_protect or protect_owned or symlink_collisions:
 				msg = []
 				msg.append("")
 				msg.append(_("Searching all installed"
@@ -3333,12 +3346,21 @@ class dblink(object):
 			# it may not be visible via a scrollback buffer, especially
 			# if the number of file collisions is large. Therefore,
 			# show a summary at the end.
+			abort = False
 			if collision_protect:
+				abort = True
 				msg = _("Package '%s' NOT merged due to file collisions.") % \
 					self.settings.mycpv
 			elif protect_owned and owners:
+				abort = True
 				msg = _("Package '%s' NOT merged due to file collisions.") % \
 					self.settings.mycpv
+			elif symlink_collisions:
+				abort = True
+				msg = _("Package '%s' NOT merged due to collision " + \
+				"between a symlink and a directory which is explicitly " + \
+				"forbidden by PMS (see bug #326685).") % \
+				(self.settings.mycpv,)
 			else:
 				msg = _("Package '%s' merged despite file collisions.") % \
 					self.settings.mycpv
@@ -3346,7 +3368,7 @@ class dblink(object):
 				"messages for the whole content of the above message.")
 			eerror(wrap(msg, 70))
 
-			if collision_protect or (protect_owned and owners):
+			if abort:
 				return 1
 
 		# The merge process may move files out of the image directory,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     87df7332c631b96a2d238abd2526eb5bb1a6e4f6
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 23 18:46:41 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jul 23 18:46:41 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=87df7332

merge: handle symlink replacing dir

Previously, these symlinks would trigger a file collision message and
would then be silently dropped in cases when the file collsion did not
cause emerge to bail out due to FEATURES=protect-owned or
collision-protect.

Now, if emerge doesn't bail out due to a file collision, it will
trigger an eerror message and merge the symlink with a .backup.XXXX
extension appended to the file name, similar to handling of regular
files in commit 740f71301ed3daf44c0e77df5d5de39fe1438fb1. This will
fix bug 326685.

---
 pym/portage/dbapi/vartree.py |   20 ++++++++++++++------
 1 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index d61f739..c703415 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3824,12 +3824,20 @@ class dblink(object):
 				myrealto = normalize_path(os.path.join(destroot, myabsto))
 				if mydmode!=None:
 					#destination exists
-					if not stat.S_ISLNK(mydmode):
-						if stat.S_ISDIR(mydmode):
-							# directory in the way: we can't merge a symlink over a directory
-							# we won't merge this, continue with next file...
-							continue
+					if stat.S_ISDIR(mydmode):
+						# we can't merge a symlink over a directory
+						newdest = self._new_backup_path(mydest)
+						msg = []
+						msg.append("")
+						msg.append(_("Installation of a symlink is blocked by a directory:"))
+						msg.append("  '%s'" % mydest)
+						msg.append(_("This symlink will be merged with a different name:"))
+						msg.append("  '%s'" % newdest)
+						msg.append("")
+						self._eerror("preinst", msg)
+						mydest = newdest
 
+					elif not stat.S_ISLNK(mydmode):
 						if os.path.exists(mysrc) and stat.S_ISDIR(os.stat(mysrc)[stat.ST_MODE]):
 							# Kill file blocking installation of symlink to dir #71787
 							pass

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e38c55ac49dcbd004ebbc65fd28053198b521ce8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 12 19:48:45 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jul 12 19:48:45 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e38c55ac

Use explicit 'mode' keyword arg with io.open().

---
 pym/portage/dbapi/vartree.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index bec3d66..9efc47f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3385,8 +3385,8 @@ class dblink(object):
 			counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
 		io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
 			encoding=_encodings['fs'], errors='strict'),
-			'w', encoding=_encodings['repo.content'], errors='backslashreplace'
-			).write(_unicode_decode(str(counter)))
+			mode='w', encoding=_encodings['repo.content'],
+			errors='backslashreplace').write(_unicode_decode(str(counter)))
 
 		self.updateprotect()
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f2770d06d66074e704454403515e516727a496cc
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul  8 06:46:28 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul  8 06:46:28 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f2770d06

cpv_expand: fix KeyError when settings is None

---
 pym/portage/dbapi/cpv_expand.py |    6 +++++-
 1 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/cpv_expand.py b/pym/portage/dbapi/cpv_expand.py
index 7077b1f..947194c 100644
--- a/pym/portage/dbapi/cpv_expand.py
+++ b/pym/portage/dbapi/cpv_expand.py
@@ -3,6 +3,7 @@
 
 __all__ = ["cpv_expand"]
 
+import portage
 from portage.exception import AmbiguousPackageName
 from portage.localization import _
 from portage.util import writemsg
@@ -16,7 +17,10 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
 	myslash=mycpv.split("/")
 	mysplit = _pkgsplit(myslash[-1])
 	if settings is None:
-		settings = globals()["settings"]
+		try:
+			settings = mydb.settings
+		except AttributeError:
+			settings = portage.settings
 	if len(myslash)>2:
 		# this is illegal case.
 		mysplit=[]

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a30cc13e70baad6abf41224afadf4a91dd3eb828
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul  1 04:00:52 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul  1 04:00:52 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a30cc13e

preserve-libs: only preserve soname symlinks

This avoids calling the LinkageMapELF.isMasterLink() method, since the
only symlinks that are strictly required are the soname symlinks.

---
 pym/portage/dbapi/vartree.py |   26 +++++++++++++-------------
 1 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index b3e6f6a..5a86291 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2461,25 +2461,25 @@ class dblink(object):
 
 		preserve_paths = set()
 		for preserve_node in preserve_nodes:
-			# Make sure that at least one of the paths is not a symlink.
-			# This prevents symlinks from being erroneously preserved by
-			# themselves when the old instance installed symlinks that
-			# the new instance does not install.
-			have_lib = False
+			# Preserve the library itself, and also preserve the
+			# soname symlink which is the only symlink that is
+			# strictly required.
+			hardlinks = set()
+			soname_symlinks = set()
+			soname = linkmap.getSoname(next(iter(preserve_node.alt_paths)))
 			for f in preserve_node.alt_paths:
 				f_abs = os.path.join(root, f.lstrip(os.sep))
 				try:
 					if stat.S_ISREG(os.lstat(f_abs).st_mode):
-						have_lib = True
-						break
+						hardlinks.add(f)
+					elif os.path.basename(f) == soname:
+						soname_symlinks.add(f)
 				except OSError:
-					continue
+					pass
 
-			if have_lib:
-				# There's no point in preserving the "master" symlink, since
-				# the soname symlink is all that's strictly required.
-				preserve_paths.update(f for f in preserve_node.alt_paths
-					if not linkmap.isMasterLink(f))
+			if hardlinks:
+				preserve_paths.update(hardlinks)
+				preserve_paths.update(soname_symlinks)
 
 		return preserve_paths
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     df4641613d30945c5cb6b1287946dcdb7acf151e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jun 29 11:37:08 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jun 29 11:37:08 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=df464161

preserve-libs: don't preserve "master" symlink

There's no point in preserving the "master" symlink, since the soname
symlink is all that's strictly required.

---
 pym/portage/dbapi/vartree.py |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index eacada6..b3e6f6a 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2476,7 +2476,10 @@ class dblink(object):
 					continue
 
 			if have_lib:
-				preserve_paths.update(preserve_node.alt_paths)
+				# There's no point in preserving the "master" symlink, since
+				# the soname symlink is all that's strictly required.
+				preserve_paths.update(f for f in preserve_node.alt_paths
+					if not linkmap.isMasterLink(f))
 
 		return preserve_paths
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     2fde2c69df13932c4ff76f2cd439e4f3de466423
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 17 20:57:02 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun 17 20:57:02 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2fde2c69

treewalk: set REPLACING_VERSIONS later

Settings like this are better done after doebuild_environment, in case
doebuild_environment calls config.setcpv/reset. This avoids the need to
call backup_changes which is not really intended for per-package
settings since it tends to pollute the config instance with settings
that survive config.reset() calls.

---
 pym/portage/dbapi/vartree.py |    7 +++----
 1 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 5c6166c..eacada6 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3090,10 +3090,6 @@ class dblink(object):
 		if retval:
 			return retval
 
-		self.settings["REPLACING_VERSIONS"] = " ".join( 
-			[portage.versions.cpv_getversion(other.mycpv) for other in others_in_slot] )
-		self.settings.backup_changes("REPLACING_VERSIONS")
-
 		if slot_matches:
 			# Used by self.isprotected().
 			max_dblnk = None
@@ -3243,6 +3239,9 @@ class dblink(object):
 			myebuild = os.path.join(inforoot, self.pkg + ".ebuild")
 		doebuild_environment(myebuild, "preinst",
 			settings=self.settings, db=mydbapi)
+		self.settings["REPLACING_VERSIONS"] = " ".join(
+			[portage.versions.cpv_getversion(other.mycpv)
+			for other in others_in_slot])
 		prepare_build_dirs(settings=self.settings, cleanup=cleanup)
 
 		if collisions:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     cbdf030c3a61eaf323faff8f25e23989efb70248
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jun 13 22:13:17 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jun 13 22:13:17 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cbdf030c

bintree: don't provide default VERSION header

We want to make sure that we know when the VERSION header is missing,
so we shouldn't provide a default value.

---
 pym/portage/dbapi/bintree.py |    1 -
 1 files changed, 0 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 36bf735..ebec11f 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -273,7 +273,6 @@ class binarytree(object):
 
 			# Populate the header with appropriate defaults.
 			self._pkgindex_default_header_data = {
-				"VERSION"      : str(self._pkgindex_version),
 				"CHOST"        : self.settings.get("CHOST", ""),
 				"repository"   : "",
 			}

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4f58705b07a948d216eba18013af483f63d34a26
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 11 03:38:02 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun 11 03:38:02 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4f58705b

bintree: verify that default REPO is legit

We need to check the RepoConfig.missing_repo_name attribute to make
sure the repo_name is really defined.

---
 pym/portage/dbapi/bintree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 7af152c..36bf735 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -284,7 +284,7 @@ class binarytree(object):
 			# pkgindex header already defines these keys, then
 			# they will appropriately override our defaults.
 			main_repo = self.settings.repositories.mainRepo()
-			if main_repo is not None:
+			if main_repo is not None and not main_repo.missing_repo_name:
 				self._pkgindex_default_header_data["repository"] = \
 					main_repo.name
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1aa05134750fc3639031c852824dac1c182a05dc
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 10 17:19:36 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun 10 17:19:36 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1aa05134

fakedbapi: add myrepo args for portdbapi compat

---
 pym/portage/dbapi/virtual.py |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py
index c16e822..ec97ffe 100644
--- a/pym/portage/dbapi/virtual.py
+++ b/pym/portage/dbapi/virtual.py
@@ -38,10 +38,10 @@ class fakedbapi(dbapi):
 		self._match_cache[origdep] = result
 		return result[:]
 
-	def cpv_exists(self, mycpv):
+	def cpv_exists(self, mycpv, myrepo=None):
 		return mycpv in self.cpvdict
 
-	def cp_list(self, mycp, use_cache=1):
+	def cp_list(self, mycp, use_cache=1, myrepo=None):
 		cachelist = self._match_cache.get(mycp)
 		# cp_list() doesn't expand old-style virtuals
 		if cachelist and cachelist[0].startswith(mycp):
@@ -102,7 +102,7 @@ class fakedbapi(dbapi):
 		if not len(self.cpdict[mycp]):
 			del self.cpdict[mycp]
 
-	def aux_get(self, mycpv, wants):
+	def aux_get(self, mycpv, wants, myrepo=None):
 		if not self.cpv_exists(mycpv):
 			raise KeyError(mycpv)
 		metadata = self.cpvdict[mycpv]

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     70bf95f5bdb143f51ad962f8c7f7c841be3dab8e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 10 12:21:06 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun 10 12:21:06 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=70bf95f5

bintree: handle pkgindex CHOST header like REPO

This allows an existing CHOST header entry to properly override our
default.

---
 pym/portage/dbapi/bintree.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 756ea67..7af152c 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -249,7 +249,7 @@ class binarytree(object):
 			self._pkgindex_header_keys = set([
 				"ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
 				"ACCEPT_PROPERTIES", "CBUILD",
-				"CHOST", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
+				"CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
 				"GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"])
 			self._pkgindex_default_pkg_data = {
 				"BUILD_TIME"         : "",
@@ -274,6 +274,7 @@ class binarytree(object):
 			# Populate the header with appropriate defaults.
 			self._pkgindex_default_header_data = {
 				"VERSION"      : str(self._pkgindex_version),
+				"CHOST"        : self.settings.get("CHOST", ""),
 				"repository"   : "",
 			}
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e445fcfd94100fcf83ad39454bfcdcb801a56721
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 10 12:04:17 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun 10 12:04:17 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e445fcfd

bintree: always populate pkgindex REPO header

The previous code would only populate it when there was no existing
index file. Now, the default will be populated even if the file exists
already. If an existing pkgindex header already defines these keys,
then they will appropriately override our defaults.

---
 pym/portage/dbapi/bintree.py |   26 ++++++++++++++++----------
 1 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index ffa296c..756ea67 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -270,9 +270,23 @@ class binarytree(object):
 				"REQUIRED_USE" : ""
 			}
 			self._pkgindex_inherited_keys = ["CHOST", "repository"]
+
+			# Populate the header with appropriate defaults.
 			self._pkgindex_default_header_data = {
-				"repository":""
+				"VERSION"      : str(self._pkgindex_version),
+				"repository"   : "",
 			}
+
+			# It is especially important to populate keys like
+			# "repository" that save space when entries can
+			# inherit them from the header. If an existing
+			# pkgindex header already defines these keys, then
+			# they will appropriately override our defaults.
+			main_repo = self.settings.repositories.mainRepo()
+			if main_repo is not None:
+				self._pkgindex_default_header_data["repository"] = \
+					main_repo.name
+
 			self._pkgindex_translated_keys = (
 				("DESCRIPTION"   ,   "DESC"),
 				("repository"    ,   "REPO"),
@@ -284,7 +298,6 @@ class binarytree(object):
 				self._pkgindex_hashes,
 				self._pkgindex_default_pkg_data,
 				self._pkgindex_inherited_keys,
-				self._pkgindex_default_header_data,
 				chain(*self._pkgindex_translated_keys)
 			))
 
@@ -1280,14 +1293,7 @@ class binarytree(object):
 				mode='r', encoding=_encodings['repo.content'],
 				errors='replace')
 		except EnvironmentError:
-			# We're creating a new file, so populate the header
-			# with appropriate defaults. This is especially
-			# important for keys like REPO that save space when
-			# entries can inherit them from the header.
-			pkgindex.header["VERSION"] = str(self._pkgindex_version)
-			main_repo = self.settings.repositories.mainRepo()
-			if main_repo is not None:
-				pkgindex.header["REPO"] = main_repo.name
+			pass
 		else:
 			try:
 				pkgindex.read(f)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     531f10764b7159dd9578f914fa7a862132c582e1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jun  6 11:51:54 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jun  6 11:51:54 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=531f1076

bintree: populate REPO header for empty pkgindex

---
 pym/portage/dbapi/bintree.py |    9 ++++++++-
 1 files changed, 8 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 33cd658..ffa296c 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1280,7 +1280,14 @@ class binarytree(object):
 				mode='r', encoding=_encodings['repo.content'],
 				errors='replace')
 		except EnvironmentError:
-			pass
+			# We're creating a new file, so populate the header
+			# with appropriate defaults. This is especially
+			# important for keys like REPO that save space when
+			# entries can inherit them from the header.
+			pkgindex.header["VERSION"] = str(self._pkgindex_version)
+			main_repo = self.settings.repositories.mainRepo()
+			if main_repo is not None:
+				pkgindex.header["REPO"] = main_repo.name
 		else:
 			try:
 				pkgindex.read(f)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a9d57bfab26b7c9cff433017a3d3579c8aa501df
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  5 09:15:54 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  5 09:15:54 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a9d57bfa

match-all-cpv-only: share match-all cache when ok

---
 pym/portage/dbapi/porttree.py |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 394a57c..33c6a3b 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -814,6 +814,10 @@ class portdbapi(dbapi):
 			# to bypass unnecessary cache access for things like IUSE
 			# and SLOT.
 			if mydep == mykey:
+				# Share cache with match-all/cp_list when the result is the
+				# same. Note that this requires that mydep.repo is None and
+				# thus mytree is also None.
+				level = "match-all"
 				myval = self.cp_list(mykey, mytree=mytree)
 			else:
 				myval = match_from_list(mydep,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     92e5a5402074d4cb20c5cc9bd323acaf7a8e211e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  5 08:59:49 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  5 08:59:49 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=92e5a540

portdbapi.xmatch: remove redundant myval check

---
 pym/portage/dbapi/porttree.py |   12 +++++-------
 1 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 4c02cbe..394a57c 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -813,13 +813,11 @@ class portdbapi(dbapi):
 			# match *all* packages, only against the cpv, in order
 			# to bypass unnecessary cache access for things like IUSE
 			# and SLOT.
-
-			if myval is None:
-				if mydep == mykey:
-					myval = self.cp_list(mykey, mytree=mytree)
-				else:
-					myval = match_from_list(mydep,
-						self.cp_list(mykey, mytree=mytree))
+			if mydep == mykey:
+				myval = self.cp_list(mykey, mytree=mytree)
+			else:
+				myval = match_from_list(mydep,
+					self.cp_list(mykey, mytree=mytree))
 
 		elif level == "list-visible":
 			#a list of all visible packages, not called directly (just by xmatch())

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     eab68e779802691733838f04f99ef71bdaa09e1f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  5 08:55:27 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  5 08:55:27 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=eab68e77

portdbapi.xmatch: always pass mytree to cp_list

---
 pym/portage/dbapi/porttree.py |   47 ++++++++++++++++++++++------------------
 1 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index f971468..4c02cbe 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -793,28 +793,29 @@ class portdbapi(dbapi):
 			except KeyError:
 				pass
 
-		if not mydep:
+		if mydep is None:
 			#this stuff only runs on first call of xmatch()
 			#create mydep, mykey from origdep
 			mydep = dep_expand(origdep, mydb=self, settings=self.settings)
 			mykey = mydep.cp
 
-		if level == "match-all-cpv-only":
+		myval = None
+		mytree = None
+		if mydep.repo is not None:
+			mytree = self.treemap.get(mydep.repo)
+			if mytree is None:
+				myval = []
+
+		if myval is not None:
+			# Unknown repo, empty result.
+			pass
+		elif level == "match-all-cpv-only":
 			# match *all* packages, only against the cpv, in order
 			# to bypass unnecessary cache access for things like IUSE
 			# and SLOT.
-			myval = None
-			mytree = None
-			if mydep.repo is not None:
-				mytree = self.treemap.get(mydep.repo)
-				if mytree is None:
-					myval = []
 
 			if myval is None:
 				if mydep == mykey:
-					# Share cache with match-all/cp_list
-					# when the result is the same.
-					level = "match-all"
 					myval = self.cp_list(mykey, mytree=mytree)
 				else:
 					myval = match_from_list(mydep,
@@ -824,14 +825,17 @@ class portdbapi(dbapi):
 			#a list of all visible packages, not called directly (just by xmatch())
 			#myval = self.visible(self.cp_list(mykey))
 
-			myval = self.gvisible(self.visible(self.cp_list(mykey)))
+			myval = self.gvisible(self.visible(
+				self.cp_list(mykey, mytree=mytree)))
 		elif level == "minimum-all":
 			# Find the minimum matching version. This is optimized to
 			# minimize the number of metadata accesses (improves performance
 			# especially in cases where metadata needs to be generated).
-			cpv_iter = iter(self.cp_list(mykey))
-			if mydep != mykey:
-				cpv_iter = self._iter_match(mydep, cpv_iter)
+			if mydep == mykey:
+				cpv_iter = iter(self.cp_list(mykey, mytree=mytree))
+			else:
+				cpv_iter = self._iter_match(mydep,
+					self.cp_list(mykey, mytree=mytree))
 			try:
 				myval = next(cpv_iter)
 			except StopIteration:
@@ -842,10 +846,10 @@ class portdbapi(dbapi):
 			# minimize the number of metadata accesses (improves performance
 			# especially in cases where metadata needs to be generated).
 			if mydep == mykey:
-				mylist = self.cp_list(mykey)
+				mylist = self.cp_list(mykey, mytree=mytree)
 			else:
-				mylist = match_from_list(mydep, self.cp_list(mykey,
-					mytree=self.repositories.get_location_for_name(mydep.repo)))
+				mylist = match_from_list(mydep,
+					self.cp_list(mykey, mytree=mytree))
 			myval = ""
 			settings = self.settings
 			local_config = settings.local_config
@@ -904,13 +908,14 @@ class portdbapi(dbapi):
 			#dep match -- find all visible matches
 			#get all visible packages, then get the matching ones
 			myval = list(self._iter_match(mydep,
-				self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey)))
+				self.xmatch("list-visible", mykey, mydep=Atom(mykey), mykey=mykey)))
 		elif level == "match-all":
 			#match *all* visible *and* masked packages
 			if mydep == mykey:
-				myval = self.cp_list(mykey)
+				myval = self.cp_list(mykey, mytree=mytree)
 			else:
-				myval = list(self._iter_match(mydep, self.cp_list(mykey)))
+				myval = list(self._iter_match(mydep,
+					self.cp_list(mykey, mytree=mytree)))
 		else:
 			raise AssertionError(
 				"Invalid level argument: '%s'" % level)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d51baaba0d5cad0fc7941bd593915b1fc7f42f94
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  5 08:14:07 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  5 08:14:07 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d51baaba

dbapi._iter_match: remove redundant myrepo args

The myrepo arguments are redundant since the Atom.repo attribute
carries the same information.

---
 pym/portage/dbapi/__init__.py |   20 ++++++++++----------
 pym/portage/dbapi/porttree.py |    4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index 40631e0..e386faa 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -123,33 +123,33 @@ class dbapi(object):
 		return list(self._iter_match(mydep,
 			self.cp_list(mydep.cp, use_cache=use_cache)))
 
-	def _iter_match(self, atom, cpv_iter, myrepo=None):
+	def _iter_match(self, atom, cpv_iter):
 		cpv_iter = iter(match_from_list(atom, cpv_iter))
 		if atom.slot:
-			cpv_iter = self._iter_match_slot(atom, cpv_iter, myrepo)
+			cpv_iter = self._iter_match_slot(atom, cpv_iter)
 		if atom.unevaluated_atom.use:
-			cpv_iter = self._iter_match_use(atom, cpv_iter, myrepo)
+			cpv_iter = self._iter_match_use(atom, cpv_iter)
 		if atom.repo:
-			cpv_iter = self._iter_match_repo(atom, cpv_iter, myrepo)
+			cpv_iter = self._iter_match_repo(atom, cpv_iter)
 		return cpv_iter
 
-	def _iter_match_repo(self, atom, cpv_iter, myrepo=None):
+	def _iter_match_repo(self, atom, cpv_iter):
 		for cpv in cpv_iter:
 			try:
-				if self.aux_get(cpv, ["repository"], myrepo=myrepo)[0] == atom.repo:
+				if self.aux_get(cpv, ["repository"], myrepo=atom.repo)[0] == atom.repo:
 					yield cpv
 			except KeyError:
 				continue
 
-	def _iter_match_slot(self, atom, cpv_iter, myrepo=None):
+	def _iter_match_slot(self, atom, cpv_iter):
 		for cpv in cpv_iter:
 			try:
-				if self.aux_get(cpv, ["SLOT"], myrepo=myrepo)[0] == atom.slot:
+				if self.aux_get(cpv, ["SLOT"], myrepo=atom.repo)[0] == atom.slot:
 					yield cpv
 			except KeyError:
 				continue
 
-	def _iter_match_use(self, atom, cpv_iter, myrepo = None):
+	def _iter_match_use(self, atom, cpv_iter):
 		"""
 		1) Check for required IUSE intersection (need implicit IUSE here).
 		2) Check enabled/disabled flag states.
@@ -158,7 +158,7 @@ class dbapi(object):
 		iuse_implicit_match = self.settings._iuse_implicit_match
 		for cpv in cpv_iter:
 			try:
-				iuse, slot, use = self.aux_get(cpv, ["IUSE", "SLOT", "USE"], myrepo=myrepo)
+				iuse, slot, use = self.aux_get(cpv, ["IUSE", "SLOT", "USE"], myrepo=atom.repo)
 			except KeyError:
 				continue
 			iuse = frozenset(x.lstrip('+-') for x in iuse.split())

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index a30c9d7..f971468 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -904,13 +904,13 @@ class portdbapi(dbapi):
 			#dep match -- find all visible matches
 			#get all visible packages, then get the matching ones
 			myval = list(self._iter_match(mydep,
-				self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey), myrepo=mydep.repo))
+				self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey)))
 		elif level == "match-all":
 			#match *all* visible *and* masked packages
 			if mydep == mykey:
 				myval = self.cp_list(mykey)
 			else:
-				myval = list(self._iter_match(mydep, self.cp_list(mykey), myrepo = mydep.repo))
+				myval = list(self._iter_match(mydep, self.cp_list(mykey)))
 		else:
 			raise AssertionError(
 				"Invalid level argument: '%s'" % level)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5bf2b111e23a60b60d307a56a833575c3d514125
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  5 07:23:33 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  5 07:23:33 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5bf2b111

dep_expand: fix orig_dep for previous commit

---
 pym/portage/dbapi/dep_expand.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/dep_expand.py b/pym/portage/dbapi/dep_expand.py
index afd8b4f..ac8ccf4 100644
--- a/pym/portage/dbapi/dep_expand.py
+++ b/pym/portage/dbapi/dep_expand.py
@@ -22,7 +22,7 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
 			return mydep
 		if mydep[0] == "*":
 			mydep = mydep[1:]
-		mydep = orig_dep
+			orig_dep = mydep
 		has_cat = '/' in orig_dep
 		if not has_cat:
 			alphanum = re.search(r'\w', orig_dep)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e390b1d43556d665ca3356317ab843bd68cbfe14
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun  5 07:10:17 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun  5 07:10:17 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e390b1d4

dep_expand: optimize away cpv_expand if possible

---
 pym/portage/dbapi/dep_expand.py |   23 ++++++++++++++++-------
 1 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/dep_expand.py b/pym/portage/dbapi/dep_expand.py
index 60e30df..afd8b4f 100644
--- a/pym/portage/dbapi/dep_expand.py
+++ b/pym/portage/dbapi/dep_expand.py
@@ -14,14 +14,14 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
 	'''
 	@rtype: Atom
 	'''
-	if not len(mydep):
-		return mydep
-	if mydep[0]=="*":
-		mydep=mydep[1:]
 	orig_dep = mydep
 	if isinstance(orig_dep, Atom):
-		mydep = orig_dep.cp
+		has_cat = True
 	else:
+		if not mydep:
+			return mydep
+		if mydep[0] == "*":
+			mydep = mydep[1:]
 		mydep = orig_dep
 		has_cat = '/' in orig_dep
 		if not has_cat:
@@ -40,8 +40,17 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
 		if not has_cat:
 			null_cat, pn = catsplit(mydep.cp)
 			mydep = pn
-		else:
-			mydep = mydep.cp
+
+	if has_cat:
+		# Optimize most common cases to avoid calling cpv_expand.
+		if not mydep.cp.startswith("virtual/"):
+			return mydep
+		if not hasattr(mydb, "cp_list") or \
+			mydb.cp_list(mydep.cp):
+			return mydep
+		# Fallback to legacy cpv_expand for old-style PROVIDE virtuals.
+		mydep = mydep.cp
+
 	expanded = cpv_expand(mydep, mydb=mydb,
 		use_cache=use_cache, settings=settings)
 	return Atom(orig_dep.replace(mydep, expanded, 1), allow_repo=True)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     b194947c9a379c8ab5a43ecd5fdef9478df0c197
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun  4 01:49:02 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun  4 01:49:02 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b194947c

Revert "dblink.merge: remove autoclean emergelog calls"

This reverts commit 352280549c9b72e7b916803dc6b03b959d0817e3.
I'm planning to fix this so the emergelog calls are only enabled when
emerge is the caller.

---
 pym/portage/dbapi/vartree.py |   15 +++++++++++++++
 1 files changed, 15 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 452a38f..16cf48b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -57,6 +57,7 @@ from portage import _unicode_encode
 from _emerge.AsynchronousLock import AsynchronousLock
 from _emerge.EbuildBuildDir import EbuildBuildDir
 from _emerge.EbuildPhase import EbuildPhase
+from _emerge.emergelog import emergelog
 from _emerge.PollScheduler import PollScheduler
 from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
 
@@ -2957,6 +2958,9 @@ class dblink(object):
 			if str_buffer:
 				os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
 
+	def _emerge_log(self, msg):
+		emergelog(False, msg)
+
 	def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
 		mydbapi=None, prev_mtimes=None, counter=None):
 		"""
@@ -3457,6 +3461,8 @@ class dblink(object):
 			match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]):
 			reinstall_self = True
 
+		emerge_log = self._emerge_log
+
 		# If we have any preserved libraries then autoclean
 		# is forced so that preserve-libs logic doesn't have
 		# to account for the additional complexity of the
@@ -3464,6 +3470,9 @@ class dblink(object):
 		autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" \
 			or preserve_paths
 
+		if autoclean:
+			emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,))
+
 		others_in_slot.append(self)  # self has just been merged
 		for dblnk in list(others_in_slot):
 			if dblnk is self:
@@ -3471,6 +3480,7 @@ class dblink(object):
 			if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self):
 				continue
 			showMessage(_(">>> Safely unmerging already-installed instance...\n"))
+			emerge_log(_(" === Unmerging... (%s)") % (dblnk.mycpv,))
 			others_in_slot.remove(dblnk) # dblnk will unmerge itself now
 			dblnk._linkmap_broken = self._linkmap_broken
 			dblnk.settings["REPLACED_BY_VERSION"] = portage.versions.cpv_getversion(self.mycpv)
@@ -3480,6 +3490,11 @@ class dblink(object):
 				preserve_paths=preserve_paths)
 			dblnk.settings.pop("REPLACED_BY_VERSION", None)
 
+			if unmerge_rval == os.EX_OK:
+				emerge_log(_(" >>> unmerge success: %s") % (dblnk.mycpv,))
+			else:
+				emerge_log(_(" !!! unmerge FAILURE: %s") % (dblnk.mycpv,))
+
 			self.lockdb()
 			try:
 				# TODO: Check status and abort if necessary.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     352280549c9b72e7b916803dc6b03b959d0817e3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun  3 11:20:27 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun  3 11:20:27 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=35228054

dblink.merge: remove autoclean emergelog calls

This logging is non-essential, and it's somewhat out of place to call
emergelog from dblink.merge since emerge is not the only caller of
this code.

---
 pym/portage/dbapi/vartree.py |   15 ---------------
 1 files changed, 0 insertions(+), 15 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 16cf48b..452a38f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -57,7 +57,6 @@ from portage import _unicode_encode
 from _emerge.AsynchronousLock import AsynchronousLock
 from _emerge.EbuildBuildDir import EbuildBuildDir
 from _emerge.EbuildPhase import EbuildPhase
-from _emerge.emergelog import emergelog
 from _emerge.PollScheduler import PollScheduler
 from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
 
@@ -2958,9 +2957,6 @@ class dblink(object):
 			if str_buffer:
 				os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
 
-	def _emerge_log(self, msg):
-		emergelog(False, msg)
-
 	def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
 		mydbapi=None, prev_mtimes=None, counter=None):
 		"""
@@ -3461,8 +3457,6 @@ class dblink(object):
 			match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]):
 			reinstall_self = True
 
-		emerge_log = self._emerge_log
-
 		# If we have any preserved libraries then autoclean
 		# is forced so that preserve-libs logic doesn't have
 		# to account for the additional complexity of the
@@ -3470,9 +3464,6 @@ class dblink(object):
 		autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" \
 			or preserve_paths
 
-		if autoclean:
-			emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,))
-
 		others_in_slot.append(self)  # self has just been merged
 		for dblnk in list(others_in_slot):
 			if dblnk is self:
@@ -3480,7 +3471,6 @@ class dblink(object):
 			if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self):
 				continue
 			showMessage(_(">>> Safely unmerging already-installed instance...\n"))
-			emerge_log(_(" === Unmerging... (%s)") % (dblnk.mycpv,))
 			others_in_slot.remove(dblnk) # dblnk will unmerge itself now
 			dblnk._linkmap_broken = self._linkmap_broken
 			dblnk.settings["REPLACED_BY_VERSION"] = portage.versions.cpv_getversion(self.mycpv)
@@ -3490,11 +3480,6 @@ class dblink(object):
 				preserve_paths=preserve_paths)
 			dblnk.settings.pop("REPLACED_BY_VERSION", None)
 
-			if unmerge_rval == os.EX_OK:
-				emerge_log(_(" >>> unmerge success: %s") % (dblnk.mycpv,))
-			else:
-				emerge_log(_(" !!! unmerge FAILURE: %s") % (dblnk.mycpv,))
-
 			self.lockdb()
 			try:
 				# TODO: Check status and abort if necessary.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     efc19df1a18d39257fb76e7259798e041d7b8619
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun  3 10:13:23 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun  3 10:13:23 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=efc19df1

dblink.unmerge: fix some background logic cases

---
 pym/portage/dbapi/_MergeProcess.py |    7 +++++--
 pym/portage/dbapi/vartree.py       |    3 +++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index b52c158..78e1ecd 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -217,9 +217,12 @@ class MergeProcess(SpawnProcess):
 		# already be opened by the parent process, so we set the
 		# "subprocess" value for use in conditional logging code
 		# involving PORTAGE_LOG_FILE.
-		if not self.unmerge and self.settings.get("PORTAGE_BACKGROUND") == "1":
+		if not self.unmerge:
 			# unmerge phases have separate logs
-			self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
+			if self.settings.get("PORTAGE_BACKGROUND") == "1":
+				self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
+			else:
+				self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
 			self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
 		self.settings["PORTAGE_BACKGROUND"] = "subprocess"
 		self.settings.backup_changes("PORTAGE_BACKGROUND")

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 11eaca3..889cb58 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1629,6 +1629,9 @@ class dblink(object):
 				self.settings["PORTAGE_BACKGROUND"] = "1"
 				self.settings.backup_changes("PORTAGE_BACKGROUND")
 				background = True
+			elif self.settings.get("PORTAGE_BACKGROUND_UNMERGE") == "0":
+				self.settings["PORTAGE_BACKGROUND"] = "0"
+				self.settings.backup_changes("PORTAGE_BACKGROUND")
 		elif self.settings.get("PORTAGE_BACKGROUND") == "1":
 			background = True
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     63d0867faac7248fe499ba76b4460e1c451683e1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun  3 02:35:05 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun  3 02:35:05 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=63d0867f

dblink.unmerge: init log_path earlier, don't pop

There's no need to pop PORTAGE_LOG_FILE here, since
PORTAGE_BACKGROUND=subprocess disables logging when necessary.

---
 pym/portage/dbapi/vartree.py |    5 +----
 1 files changed, 1 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index e742358..3c7e934 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1619,7 +1619,7 @@ class dblink(object):
 				DeprecationWarning, stacklevel=2)
 
 		background = False
-		log_path = None
+		log_path = self.settings.get("PORTAGE_LOG_FILE")
 		if self._scheduler is None:
 			# We create a scheduler instance and use it to
 			# log unmerge output separately from merge output.
@@ -1629,9 +1629,6 @@ class dblink(object):
 				self.settings["PORTAGE_BACKGROUND"] = "1"
 				self.settings.backup_changes("PORTAGE_BACKGROUND")
 				background = True
-			else:
-				# Our output is redirected and logged by the parent process.
-				log_path = self.settings.pop("PORTAGE_LOG_FILE", None)
 		elif self.settings.get("PORTAGE_BACKGROUND") == "1":
 			background = True
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4b5ab00fd0fd08af8c9075acdbb3921cf39d41a3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri May 27 02:38:00 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri May 27 02:38:00 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4b5ab00f

MergeProcess: populate vardbapi cache for new pkg

---
 pym/portage/dbapi/_MergeProcess.py |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 43bec72..b52c158 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -254,6 +254,15 @@ class MergeProcess(SpawnProcess):
 		"""
 		Unregister from the scheduler and close open files.
 		"""
+
+		if not self.unmerge:
+			# Populate the vardbapi cache for the new package
+			# while its inodes are still hot.
+			try:
+				self.vartree.dbapi.aux_get(self.settings.mycpv, ["EAPI"])
+			except KeyError:
+				pass
+
 		self._unlock_vdb()
 		if self._elog_reg_id is not None:
 			self.scheduler.unregister(self._elog_reg_id)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     db603c15e4e820b309d4cf5b99186e5a02a89318
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri May 27 02:14:20 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri May 27 02:14:20 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=db603c15

counter_tick_core: flush the cache (periodically)

Since we hold a lock, this is a good opportunity to flush the cache.
Note that this will only flush the cache periodically in the main
process when _aux_cache_threshold is exceeded.

---
 pym/portage/dbapi/vartree.py |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 982324c..e742358 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -837,6 +837,12 @@ class vardbapi(dbapi):
 					self.settings._init_dirs()
 					write_atomic(self._counter_path, str(counter))
 			self._cached_counter = counter
+
+			# Since we hold a lock, this is a good opportunity
+			# to flush the cache. Note that this will only
+			# flush the cache periodically in the main process
+			# when _aux_cache_threshold is exceeded.
+			self.flush_cache()
 		finally:
 			self.unlock()
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0431bac723806b2c5539f9d2b01a017f077d190a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri May 27 00:00:22 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri May 27 00:00:22 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0431bac7

unmerge: remove redundant builddir_locked var

---
 pym/portage/dbapi/vartree.py |   11 ++++-------
 1 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 976d8e3..982324c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1684,19 +1684,17 @@ class dblink(object):
 		self._prune_plib_registry(unmerge=True, needed=needed,
 			preserve_paths=preserve_paths)
 
-		builddir_locked = "PORTAGE_BUILDIR_LOCKED" in self.settings
 		builddir_lock = None
 		scheduler = self._scheduler
 		retval = os.EX_OK
 		try:
 			# Only create builddir_lock if the caller
 			# has not already acquired the lock.
-			if not builddir_locked:
+			if "PORTAGE_BUILDIR_LOCKED" not in self.settings:
 				builddir_lock = EbuildBuildDir(
 					scheduler=scheduler,
 					settings=self.settings)
 				builddir_lock.lock()
-				builddir_locked = True
 				prepare_build_dirs(settings=self.settings, cleanup=True)
 				log_path = self.settings.get("PORTAGE_LOG_FILE")
 
@@ -1746,8 +1744,7 @@ class dblink(object):
 
 		finally:
 			self.vartree.dbapi._bump_mtime(self.mycpv)
-			if builddir_locked:
-				try:
+			try:
 					if not eapi_unsupported and os.path.isfile(myebuildpath):
 						if retval != os.EX_OK:
 							msg_lines = []
@@ -1789,7 +1786,7 @@ class dblink(object):
 
 					self._elog_process(phasefilter=("prerm", "postrm"))
 
-					if retval == os.EX_OK and builddir_locked:
+					if retval == os.EX_OK:
 						try:
 							doebuild_environment(myebuildpath, "cleanrm",
 								settings=self.settings, db=self.vartree.dbapi)
@@ -1800,7 +1797,7 @@ class dblink(object):
 							settings=self.settings)
 						phase.start()
 						retval = phase.wait()
-				finally:
+			finally:
 					if builddir_lock is not None:
 						builddir_lock.unlock()
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     cc7f820749411a2d8f2e3efff945837231bfd0eb
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 26 02:32:42 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 26 02:32:42 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cc7f8207

dblink.unmerge: always initialize myebuildpath

This is required for the doebuild_environment() call. Also, set
builddir_locked = True when appropriate.

---
 pym/portage/dbapi/vartree.py |   15 +++++----------
 1 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 47952e7..976d8e3 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1655,13 +1655,12 @@ class dblink(object):
 		contents = self.getcontents()
 		# Now, don't assume that the name of the ebuild is the same as the
 		# name of the dir; the package may have been moved.
-		myebuildpath = None
+		myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild")
 		failures = 0
 		ebuild_phase = "prerm"
 		mystuff = os.listdir(self.dbdir)
 		for x in mystuff:
 			if x.endswith(".ebuild"):
-				myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild")
 				if x[:-7] != self.pkg:
 					# Clean up after vardbapi.move_ent() breakage in
 					# portage versions before 2.1.2
@@ -1697,6 +1696,7 @@ class dblink(object):
 					scheduler=scheduler,
 					settings=self.settings)
 				builddir_lock.lock()
+				builddir_locked = True
 				prepare_build_dirs(settings=self.settings, cleanup=True)
 				log_path = self.settings.get("PORTAGE_LOG_FILE")
 
@@ -1710,7 +1710,7 @@ class dblink(object):
 					level=logging.ERROR, noiselevel=-1)
 				showMessage(_unicode_decode("%s\n") % (eapi_unsupported,),
 					level=logging.ERROR, noiselevel=-1)
-			elif myebuildpath:
+			elif os.path.isfile(myebuildpath):
 				phase = EbuildPhase(background=background,
 					phase=ebuild_phase, scheduler=scheduler,
 					settings=self.settings)
@@ -1730,7 +1730,7 @@ class dblink(object):
 				self.vartree.dbapi._fs_unlock()
 			self._clear_contents_cache()
 
-			if myebuildpath and not eapi_unsupported:
+			if not eapi_unsupported and os.path.isfile(myebuildpath):
 				ebuild_phase = "postrm"
 				phase = EbuildPhase(background=background,
 					phase=ebuild_phase, scheduler=scheduler,
@@ -1748,7 +1748,7 @@ class dblink(object):
 			self.vartree.dbapi._bump_mtime(self.mycpv)
 			if builddir_locked:
 				try:
-					if myebuildpath and not eapi_unsupported:
+					if not eapi_unsupported and os.path.isfile(myebuildpath):
 						if retval != os.EX_OK:
 							msg_lines = []
 							msg = _("The '%(ebuild_phase)s' "
@@ -1790,11 +1790,6 @@ class dblink(object):
 					self._elog_process(phasefilter=("prerm", "postrm"))
 
 					if retval == os.EX_OK and builddir_locked:
-						# myebuildpath might be None, so ensure
-						# it has a sane value for the clean phase,
-						# even though it won't really be sourced.
-						myebuildpath = os.path.join(self.dbdir,
-							self.pkg + ".ebuild")
 						try:
 							doebuild_environment(myebuildpath, "cleanrm",
 								settings=self.settings, db=self.vartree.dbapi)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     87cf046b84f153a9e35fb3d248c50479312076a5
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 26 01:37:55 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 26 01:37:55 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=87cf046b

counter_tick: respect incrementing param always

Every package install must have a unique counter, since a slotmove
update can move two packages into the same SLOT and in that case it's
important that both packages have different COUNTER metadata.

---
 pym/portage/dbapi/vartree.py |   28 ++++++++++++++++------------
 1 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 62a2332..47952e7 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -812,27 +812,31 @@ class vardbapi(dbapi):
 	def counter_tick_core(self, myroot=None, incrementing=1, mycpv=None):
 		"""
 		This method will grab the next COUNTER value and record it back
-		to the global file.  Returns new counter value.
+		to the global file. Note that every package install must have
+		a unique counter, since a slotmove update can move two packages
+		into the same SLOT and in that case it's important that both
+		packages have different COUNTER metadata.
 
 		@param myroot: ignored, self._eroot is used instead
 		@param mycpv: ignored
+		@rtype: int
+		@returns: new counter value
 		"""
 		myroot = None
 		mycpv = None
 		self.lock()
 		try:
 			counter = self.get_counter_tick_core() - 1
-			if self._cached_counter != counter:
-				if incrementing:
-					#increment counter
-					counter += 1
-					# update new global counter file
-					try:
-						write_atomic(self._counter_path, str(counter))
-					except InvalidLocation:
-						self.settings._init_dirs()
-						write_atomic(self._counter_path, str(counter))
-				self._cached_counter = counter
+			if incrementing:
+				#increment counter
+				counter += 1
+				# update new global counter file
+				try:
+					write_atomic(self._counter_path, str(counter))
+				except InvalidLocation:
+					self.settings._init_dirs()
+					write_atomic(self._counter_path, str(counter))
+			self._cached_counter = counter
 		finally:
 			self.unlock()
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d9468f78c67e35f817389d9a36a570506720c343
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 26 00:57:05 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 26 00:57:05 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d9468f78

unmerge: fix logging for unsupported EAPI

---
 pym/portage/dbapi/vartree.py |   55 ++++++++++++++++++++++-------------------
 1 files changed, 29 insertions(+), 26 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 021191f..62a2332 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1670,19 +1670,13 @@ class dblink(object):
 			# We avoid a redundant setcpv call here when
 			# the caller has already taken care of it.
 			self.settings.setcpv(self.mycpv, mydb=self.vartree.dbapi)
-		if myebuildpath:
-			try:
-				doebuild_environment(myebuildpath, "prerm",
-					settings=self.settings, db=self.vartree.dbapi)
-			except UnsupportedAPIException as e:
-				failures += 1
-				# Sometimes this happens due to corruption of the EAPI file.
-				showMessage(_("!!! FAILED prerm: %s\n") % \
-					os.path.join(self.dbdir, "EAPI"),
-					level=logging.ERROR, noiselevel=-1)
-				showMessage(_unicode_decode("%s\n") % (e,),
-					level=logging.ERROR, noiselevel=-1)
-				myebuildpath = None
+
+		eapi_unsupported = False
+		try:
+			doebuild_environment(myebuildpath, "prerm",
+				settings=self.settings, db=self.vartree.dbapi)
+		except UnsupportedAPIException as e:
+			eapi_unsupported = e
 
 		self._prune_plib_registry(unmerge=True, needed=needed,
 			preserve_paths=preserve_paths)
@@ -1692,18 +1686,27 @@ class dblink(object):
 		scheduler = self._scheduler
 		retval = os.EX_OK
 		try:
-			if myebuildpath:
-				# Only create builddir_lock if the caller
-				# has not already acquired the lock.
-				if not builddir_locked:
-					builddir_lock = EbuildBuildDir(
-						scheduler=scheduler,
-						settings=self.settings)
-					builddir_lock.lock()
-					builddir_locked = True
-					prepare_build_dirs(settings=self.settings, cleanup=True)
-					log_path = self.settings.get("PORTAGE_LOG_FILE")
+			# Only create builddir_lock if the caller
+			# has not already acquired the lock.
+			if not builddir_locked:
+				builddir_lock = EbuildBuildDir(
+					scheduler=scheduler,
+					settings=self.settings)
+				builddir_lock.lock()
+				prepare_build_dirs(settings=self.settings, cleanup=True)
+				log_path = self.settings.get("PORTAGE_LOG_FILE")
 
+			# Log the error after PORTAGE_LOG_FILE is initialized
+			# by prepare_build_dirs above.
+			if eapi_unsupported:
+				# Sometimes this happens due to corruption of the EAPI file.
+				failures += 1
+				showMessage(_("!!! FAILED prerm: %s\n") % \
+					os.path.join(self.dbdir, "EAPI"),
+					level=logging.ERROR, noiselevel=-1)
+				showMessage(_unicode_decode("%s\n") % (eapi_unsupported,),
+					level=logging.ERROR, noiselevel=-1)
+			elif myebuildpath:
 				phase = EbuildPhase(background=background,
 					phase=ebuild_phase, scheduler=scheduler,
 					settings=self.settings)
@@ -1723,7 +1726,7 @@ class dblink(object):
 				self.vartree.dbapi._fs_unlock()
 			self._clear_contents_cache()
 
-			if myebuildpath:
+			if myebuildpath and not eapi_unsupported:
 				ebuild_phase = "postrm"
 				phase = EbuildPhase(background=background,
 					phase=ebuild_phase, scheduler=scheduler,
@@ -1741,7 +1744,7 @@ class dblink(object):
 			self.vartree.dbapi._bump_mtime(self.mycpv)
 			if builddir_locked:
 				try:
-					if myebuildpath:
+					if myebuildpath and not eapi_unsupported:
 						if retval != os.EX_OK:
 							msg_lines = []
 							msg = _("The '%(ebuild_phase)s' "

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5045d311b528f7e6f02223f6dc4330a6fbd1e9e9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May 25 05:09:43 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May 25 05:09:43 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5045d311

treewalk: reset cloned config instances

---
 pym/portage/dbapi/vartree.py |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 159871b..4173283 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3073,6 +3073,7 @@ class dblink(object):
 			# we need it to have private ${T} etc... for things like elog.
 			settings_clone = config(clone=self.settings)
 			settings_clone.pop("PORTAGE_BUILDIR_LOCKED", None)
+			settings_clone.reset()
 			others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
 				settings=settings_clone,
 				vartree=self.vartree, treetype="vartree",

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8f93d096a3838caf64f955bba6962b6ad0ff27fe
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May 25 05:01:35 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May 25 05:01:35 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8f93d096

dblink: use config._init_dirs() when necessary

---
 pym/portage/dbapi/vartree.py |   10 +++++++---
 1 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 0079e4f..159871b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3723,9 +3723,13 @@ class dblink(object):
 		# write out our collection of md5sums
 		if cfgfiledict != cfgfiledict_orig:
 			cfgfiledict.pop("IGNORE", None)
-			ensure_dirs(os.path.dirname(self.vartree.dbapi._conf_mem_file),
-				gid=portage_gid, mode=0o2750, mask=0o2)
-			writedict(cfgfiledict, self.vartree.dbapi._conf_mem_file)
+			try:
+				writedict(cfgfiledict, self.vartree.dbapi._conf_mem_file)
+			except IOError as e:
+				if e.errno != errno.ENOENT:
+					raise
+				self.settings._init_dirs()
+				writedict(cfgfiledict, self.vartree.dbapi._conf_mem_file)
 
 		return os.EX_OK
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     28fb9337fabb90810585c87b35f0b3a73ed29821
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May 25 04:36:36 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May 25 04:36:36 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=28fb9337

vardbapi: use config._init_dirs() when necessary

---
 pym/portage/dbapi/vartree.py |   17 +++++++++++------
 1 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index e29d70e..0079e4f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -39,7 +39,7 @@ from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_PRESERVE_LIBS
 from portage.dbapi import dbapi
 from portage.dep import _slot_separator
 from portage.exception import CommandNotFound, \
-	InvalidData, InvalidPackageName, \
+	InvalidData, InvalidLocation, InvalidPackageName, \
 	FileNotFound, PermissionDenied, UnsupportedAPIException
 from portage.localization import _
 from portage.util.movefile import movefile
@@ -222,7 +222,11 @@ class vardbapi(dbapi):
 		if self._fs_lock_count < 1:
 			if self._fs_lock_obj is not None:
 				raise AssertionError("already locked")
-			self._fs_lock_obj = lockfile(self._conf_mem_file)
+			try:
+				self._fs_lock_obj = lockfile(self._conf_mem_file)
+			except InvalidLocation:
+				self.settings._init_dirs()
+				self._fs_lock_obj = lockfile(self._conf_mem_file)
 		self._fs_lock_count += 1
 
 	def _fs_unlock(self):
@@ -822,11 +826,12 @@ class vardbapi(dbapi):
 				if incrementing:
 					#increment counter
 					counter += 1
-					# use same permissions as config._init_dirs()
-					ensure_dirs(os.path.dirname(self._counter_path),
-						gid=portage_gid, mode=0o2750, mask=0o2)
 					# update new global counter file
-					write_atomic(self._counter_path, str(counter))
+					try:
+						write_atomic(self._counter_path, str(counter))
+					except InvalidLocation:
+						self.settings._init_dirs()
+						write_atomic(self._counter_path, str(counter))
 				self._cached_counter = counter
 		finally:
 			self.unlock()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d755309c28a03302e4906a3375187d3f23154d43
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May 25 04:07:31 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May 25 04:07:31 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d755309c

cleanrm: handle UnsupportedAPIException

---
 pym/portage/dbapi/vartree.py |    7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 5aec973..e29d70e 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1783,8 +1783,11 @@ class dblink(object):
 						# even though it won't really be sourced.
 						myebuildpath = os.path.join(self.dbdir,
 							self.pkg + ".ebuild")
-						doebuild_environment(myebuildpath, "cleanrm",
-							settings=self.settings, db=self.vartree.dbapi)
+						try:
+							doebuild_environment(myebuildpath, "cleanrm",
+								settings=self.settings, db=self.vartree.dbapi)
+						except UnsupportedAPIException:
+							pass
 						phase = EbuildPhase(background=background,
 							phase="cleanrm", scheduler=scheduler,
 							settings=self.settings)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     74b175712db188b4642c159a4ac1d6d84faba354
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue May 24 10:40:23 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May 24 10:40:23 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=74b17571

treewalk: fix PORTAGE_BUILDIR_LOCKED state

This fixes breakage from commit
f0f1bbe8fa9d3f698cbe529d2a11eec1ce437119.

---
 pym/portage/dbapi/vartree.py |    4 +++-
 1 files changed, 3 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a9d2d14..5aec973 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3063,8 +3063,10 @@ class dblink(object):
 		for cur_cpv in slot_matches:
 			# Clone the config in case one of these has to be unmerged since
 			# we need it to have private ${T} etc... for things like elog.
+			settings_clone = config(clone=self.settings)
+			settings_clone.pop("PORTAGE_BUILDIR_LOCKED", None)
 			others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
-				settings=config(clone=self.settings),
+				settings=settings_clone,
 				vartree=self.vartree, treetype="vartree",
 				scheduler=self._scheduler, pipe=self._pipe))
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     fcd742f219611c4f80bf4f24c198890f8a44f85f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 21 14:22:00 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 21 14:22:00 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fcd742f2

treewalk: force AUTOCLEAN=yes if libs preserved

---
 pym/portage/dbapi/vartree.py |    7 ++++++-
 1 files changed, 6 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 60d52a8..f53c209 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3443,7 +3443,12 @@ class dblink(object):
 		else:
 			emerge_log = scheduler.dblinkEmergeLog
 
-		autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes"
+		# If we have any preserved libraries then autoclean
+		# is forced so that preserve-libs logic doesn't have
+		# to account for the additional complexity of the
+		# AUTOCLEAN=no mode.
+		autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" \
+			or preserve_paths
 
 		if autoclean:
 			emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,))

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     bc372d37ec55bf3ff0b38942f9c6080f3ca61a75
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 21 13:09:46 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 21 13:09:46 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=bc372d37

_prune_plib_registry: correct _match_contents use

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index cd93a12..60d52a8 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1532,7 +1532,7 @@ class dblink(object):
 					if unmerge_preserve:
 						for path in sorted(unmerge_preserve):
 							contents_key = self._match_contents(path)
-							if contents_key is None:
+							if not contents_key:
 								continue
 							obj_type = self.getcontents()[contents_key][0]
 							self._display_merge(_(">>> needed   %s %s\n") % \

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     2dec6268b2114a147e10503d6ddc4713443a2f99
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May 18 05:38:07 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May 18 05:38:07 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2dec6268

_find_libs_to_preserve: always return set

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index fd5c774..cd93a12 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2350,7 +2350,7 @@ class dblink(object):
 			self.vartree.dbapi._plib_registry is None or \
 			(not unmerge and self._installed_instance is None) or \
 			"preserve-libs" not in self.settings.features:
-			return None
+			return set()
 
 		os = _os_merge
 		linkmap = self.vartree.dbapi._linkmap

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4bc4e589bd925d0bad8c6e5b8e30e3c37a371c01
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May 18 05:27:07 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May 18 05:27:07 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4bc4e589

preserve-libs: check for broken linkmap more

---
 pym/portage/dbapi/vartree.py |    9 ++++++---
 1 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index cae8cd0..fd5c774 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1498,9 +1498,11 @@ class dblink(object):
 	def _prune_plib_registry(self, unmerge=False,
 		needed=None, preserve_paths=None):
 		# remove preserved libraries that don't have any consumers left
-		plib_registry = self.vartree.dbapi._plib_registry
-		if plib_registry:
+		if not (self._linkmap_broken or
+			self.vartree.dbapi._linkmap is None or
+			self.vartree.dbapi._plib_registry is None):
 			self.vartree.dbapi._fs_lock()
+			plib_registry = self.vartree.dbapi._plib_registry
 			plib_registry.lock()
 			try:
 				plib_registry.load()
@@ -3405,7 +3407,8 @@ class dblink(object):
 
 		preserve_paths = set()
 		needed = None
-		if not (linkmap is None or plib_registry is None):
+		if not (self._linkmap_broken or linkmap is None or
+			plib_registry is None):
 			self.vartree.dbapi._fs_lock()
 			plib_registry.lock()
 			try:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     b2e26ba79e1ef41594d1ec1bb032acf7fb6a1403
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 16 01:25:59 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 16 01:25:59 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b2e26ba7

preserve-libs: lock vardb for removeFromContents

---
 pym/portage/dbapi/vartree.py |   31 ++++++++++++++++++++-----------
 1 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 4d7e6b8..cae8cd0 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1548,10 +1548,14 @@ class dblink(object):
 					unmerge_no_replacement)
 				if cpv_lib_map:
 					self._remove_preserved_libs(cpv_lib_map)
-					for cpv, removed in cpv_lib_map.items():
-						if not self.vartree.dbapi.cpv_exists(cpv):
-							continue
-						self.vartree.dbapi.removeFromContents(cpv, removed)
+					self.vartree.dbapi.lock()
+					try:
+						for cpv, removed in cpv_lib_map.items():
+							if not self.vartree.dbapi.cpv_exists(cpv):
+								continue
+							self.vartree.dbapi.removeFromContents(cpv, removed)
+					finally:
+						self.vartree.dbapi.unlock()
 
 				plib_registry.store()
 			finally:
@@ -3524,12 +3528,19 @@ class dblink(object):
 						# same cpv then the vdb entry no longer belongs
 						# to it, so we'll have to get the slot and counter
 						# from plib_registry._data instead.
+						self.vartree.dbapi.lock()
 						try:
-							slot, counter = self.vartree.dbapi.aux_get(
-								cpv, ["SLOT", "COUNTER"])
-							has_vdb_entry = True
-						except KeyError:
-							pass
+							try:
+								slot, counter = self.vartree.dbapi.aux_get(
+									cpv, ["SLOT", "COUNTER"])
+							except KeyError:
+								pass
+							else:
+								has_vdb_entry = True
+								self.vartree.dbapi.removeFromContents(
+									cpv, paths)
+						finally:
+							self.vartree.dbapi.unlock()
 
 					if not has_vdb_entry:
 						# It's possible for previously unmerged packages
@@ -3553,8 +3564,6 @@ class dblink(object):
 
 					remaining = [f for f in plib_dict[cpv] if f not in paths]
 					plib_registry.register(cpv, slot, counter, remaining)
-					if has_vdb_entry:
-						self.vartree.dbapi.removeFromContents(cpv, paths)
 
 				plib_registry.store()
 			finally:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     2160fefc6b7e670906e5e76188daccf0e6c9d659
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 15 23:28:32 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 15 23:28:32 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2160fefc

Revert "counter_tick_core: don't lock if parallel-install"

This reverts commit 405ad9eed65393205ec28af8772f7ea45ce0371e.
The root problem is fixed by commit
8380ec9d7590a47b912e0d1e1db2ebb6049b903b.

---
 pym/portage/dbapi/vartree.py |   16 ++--------------
 1 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 0f8eede..4d7e6b8 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -815,18 +815,7 @@ class vardbapi(dbapi):
 		"""
 		myroot = None
 		mycpv = None
-		locked_vdb = False
-		if "parallel-install" not in self.settings.features:
-			# If parallel-install is enabled, it's unsafe to
-			# lock the vdb here because it can deadlock the
-			# Scheduler by preventing it from servicing its
-			# poll loop which is essential for at least a
-			# couple of reasons:
-			# 1) releasing locks held by the scheduler
-			# 2) handling output of subprocesses so that they
-			#    don't deadlock due to blocking on stdout
-			self.lock()
-			locked_vdb = True
+		self.lock()
 		try:
 			counter = self.get_counter_tick_core() - 1
 			if self._cached_counter != counter:
@@ -840,8 +829,7 @@ class vardbapi(dbapi):
 					write_atomic(self._counter_path, str(counter))
 				self._cached_counter = counter
 		finally:
-			if locked_vdb:
-				self.unlock()
+			self.unlock()
 
 		return counter
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     8380ec9d7590a47b912e0d1e1db2ebb6049b903b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 15 23:17:36 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 15 23:17:36 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8380ec9d

treewalk: unlockdb/lockdb typo triggers EDEADLK

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index d602a37..0f8eede 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3510,7 +3510,7 @@ class dblink(object):
 				self.vartree.dbapi.removeFromContents(blocker, iter(contents),
 					relative_paths=False)
 		finally:
-			self.lockdb()
+			self.unlockdb()
 
 		plib_registry = self.vartree.dbapi._plib_registry
 		if plib_registry:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     405ad9eed65393205ec28af8772f7ea45ce0371e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 15 09:30:15 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 15 09:30:15 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=405ad9ee

counter_tick_core: don't lock if parallel-install

This is the same as commit 461564ae94ff936918eeaa18493bc1da3846796f
but this time with comments that make sense.

---
 pym/portage/dbapi/vartree.py |   16 ++++++++++++++--
 1 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 40f0cfb..d602a37 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -815,7 +815,18 @@ class vardbapi(dbapi):
 		"""
 		myroot = None
 		mycpv = None
-		self.lock()
+		locked_vdb = False
+		if "parallel-install" not in self.settings.features:
+			# If parallel-install is enabled, it's unsafe to
+			# lock the vdb here because it can deadlock the
+			# Scheduler by preventing it from servicing its
+			# poll loop which is essential for at least a
+			# couple of reasons:
+			# 1) releasing locks held by the scheduler
+			# 2) handling output of subprocesses so that they
+			#    don't deadlock due to blocking on stdout
+			self.lock()
+			locked_vdb = True
 		try:
 			counter = self.get_counter_tick_core() - 1
 			if self._cached_counter != counter:
@@ -829,7 +840,8 @@ class vardbapi(dbapi):
 					write_atomic(self._counter_path, str(counter))
 				self._cached_counter = counter
 		finally:
-			self.unlock()
+			if locked_vdb:
+				self.unlock()
 
 		return counter
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     521f0739caae714edaebede89b7143c26330804b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 15 04:29:48 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 15 04:29:48 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=521f0739

Revert "counter_tick_core: don't lock if parallel-install"

This reverts commit 461564ae94ff936918eeaa18493bc1da3846796f.
Disabling the lock here doesn't really make sense, and I'm still
triggering EDEADLK is some other places.

---
 pym/portage/dbapi/vartree.py |   13 ++-----------
 1 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 5db267d..40f0cfb 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -815,15 +815,7 @@ class vardbapi(dbapi):
 		"""
 		myroot = None
 		mycpv = None
-		locked_vdb = False
-		if "parallel-install" not in self.settings.features:
-			# If parallel-install is enabled, it's unsafe to
-			# lock the vdb here since the portage.locks module
-			# does not behave as desired if we try to lock the
-			# same file multiple times concurrently from the
-			# same process.
-			self.lock()
-			locked_vdb = True
+		self.lock()
 		try:
 			counter = self.get_counter_tick_core() - 1
 			if self._cached_counter != counter:
@@ -837,8 +829,7 @@ class vardbapi(dbapi):
 					write_atomic(self._counter_path, str(counter))
 				self._cached_counter = counter
 		finally:
-			if locked_vdb:
-				self.unlock()
+			self.unlock()
 
 		return counter
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     03d2647d5c8d74088b29283598f8c4a0fef5db96
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 15 02:55:24 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 15 02:55:24 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=03d2647d

preserve-libs: use vardbapi _fs_lock/unlock

The preserve-libs code is too dependent on the filesystem state to
allow merging/unmerge/env_update to execute concurrently.

---
 pym/portage/dbapi/vartree.py |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a2e38f6..5db267d 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1509,6 +1509,7 @@ class dblink(object):
 		# remove preserved libraries that don't have any consumers left
 		plib_registry = self.vartree.dbapi._plib_registry
 		if plib_registry:
+			self.vartree.dbapi._fs_lock()
 			plib_registry.lock()
 			try:
 				plib_registry.load()
@@ -1564,6 +1565,7 @@ class dblink(object):
 				plib_registry.store()
 			finally:
 				plib_registry.unlock()
+				self.vartree.dbapi._fs_unlock()
 
 	def unmerge(self, pkgfiles=None, trimworld=None, cleanup=True,
 		ldpath_mtimes=None, others_in_slot=None, needed=None,
@@ -3409,6 +3411,7 @@ class dblink(object):
 		preserve_paths = set()
 		needed = None
 		if not (linkmap is None or plib_registry is None):
+			self.vartree.dbapi._fs_lock()
 			plib_registry.lock()
 			try:
 				plib_registry.load()
@@ -3423,6 +3426,7 @@ class dblink(object):
 				preserve_paths = self._find_libs_to_preserve()
 			finally:
 				plib_registry.unlock()
+				self.vartree.dbapi._fs_unlock()
 
 			if preserve_paths:
 				self._add_preserve_libs_to_contents(preserve_paths)
@@ -3507,6 +3511,7 @@ class dblink(object):
 
 		plib_registry = self.vartree.dbapi._plib_registry
 		if plib_registry:
+			self.vartree.dbapi._fs_lock()
 			plib_registry.lock()
 			try:
 				plib_registry.load()
@@ -3563,6 +3568,7 @@ class dblink(object):
 				plib_registry.store()
 			finally:
 				plib_registry.unlock()
+				self.vartree.dbapi._fs_unlock()
 
 		self.vartree.dbapi._add(self)
 		contents = self.getcontents()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     39b034d11b8a3118e8c1cfc6f6d06df43e5efa35
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 15 02:43:12 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 15 02:43:12 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=39b034d1

vardbapi: add reentrant _fs_lock/unlock methods

---
 pym/portage/dbapi/vartree.py |   65 +++++++++++++++++++++++++++--------------
 1 files changed, 43 insertions(+), 22 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index d190566..a2e38f6 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -140,6 +140,10 @@ class vardbapi(dbapi):
 		self._lock = None
 		self._lock_count = 0
 
+		self._conf_mem_file = self._eroot + CONFIG_MEMORY_FILE
+		self._fs_lock_obj = None
+		self._fs_lock_count = 0
+
 		if vartree is None:
 			vartree = portage.db[self.root]["vartree"]
 		self.vartree = vartree
@@ -210,6 +214,28 @@ class vardbapi(dbapi):
 			unlockdir(self._lock)
 			self._lock = None
 
+	def _fs_lock(self):
+		"""
+		Acquire a reentrant lock, blocking, for cooperation with concurrent
+		processes.
+		"""
+		if self._fs_lock_count < 1:
+			if self._fs_lock_obj is not None:
+				raise AssertionError("already locked")
+			self._fs_lock_obj = lockfile(self._conf_mem_file)
+		self._fs_lock_count += 1
+
+	def _fs_unlock(self):
+		"""
+		Release a lock, decrementing the recursion level.
+		"""
+		if self._fs_lock_count <= 1:
+			if self._fs_lock_obj is None:
+				raise AssertionError("not locked")
+			unlockfile(self._fs_lock_obj)
+			self._fs_lock_obj = None
+		self._fs_lock_count -= 1
+
 	def _bump_mtime(self, cpv):
 		"""
 		This is called before an after any modifications, so that consumers
@@ -1680,12 +1706,11 @@ class dblink(object):
 					showMessage(_("!!! FAILED prerm: %s\n") % retval,
 						level=logging.ERROR, noiselevel=-1)
 
-			conf_mem_file = os.path.join(self._eroot, CONFIG_MEMORY_FILE)
-			conf_mem_lock = lockfile(conf_mem_file)
+			self.vartree.dbapi._fs_lock()
 			try:
-				self._unmerge_pkgfiles(pkgfiles, others_in_slot, conf_mem_file)
+				self._unmerge_pkgfiles(pkgfiles, others_in_slot)
 			finally:
-				unlockfile(conf_mem_lock)
+				self.vartree.dbapi._fs_unlock()
 			self._clear_contents_cache()
 
 			if myebuildpath:
@@ -1795,15 +1820,14 @@ class dblink(object):
 
 		# Lock the config memory file to prevent symlink creation
 		# in merge_contents from overlapping with env-update.
-		conf_mem_file = os.path.join(self._eroot, CONFIG_MEMORY_FILE)
-		conf_mem_lock = lockfile(conf_mem_file)
+		self.vartree.dbapi._fs_lock()
 		try:
 			env_update(target_root=self.settings['ROOT'],
 				prev_mtimes=ldpath_mtimes,
 				contents=contents, env=self.settings.environ(),
 				writemsg_level=self._display_merge)
 		finally:
-			unlockfile(conf_mem_lock)
+			self.vartree.dbapi._fs_unlock()
 
 		return os.EX_OK
 
@@ -1826,7 +1850,7 @@ class dblink(object):
 					log_path=log_path, background=background,
 					level=level, noiselevel=noiselevel)
 
-	def _unmerge_pkgfiles(self, pkgfiles, others_in_slot, conf_mem_file):
+	def _unmerge_pkgfiles(self, pkgfiles, others_in_slot):
 		"""
 		
 		Unmerges the contents of a package from the liveFS
@@ -1862,7 +1886,7 @@ class dblink(object):
 		dest_root = self._eroot
 		dest_root_len = len(dest_root) - 1
 
-		cfgfiledict = grabdict(conf_mem_file)
+		cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file)
 		stale_confmem = []
 
 		unmerge_orphans = "unmerge-orphans" in self.settings.features
@@ -2112,7 +2136,7 @@ class dblink(object):
 		if stale_confmem:
 			for filename in stale_confmem:
 				del cfgfiledict[filename]
-			writedict(cfgfiledict, conf_mem_file)
+			writedict(cfgfiledict, self.vartree.dbapi._conf_mem_file)
 
 		#remove self from vartree database so that our own virtual gets zapped if we're the last node
 		self.vartree.zap(self.mycpv)
@@ -3342,10 +3366,9 @@ class dblink(object):
 		self.updateprotect()
 
 		#if we have a file containing previously-merged config file md5sums, grab it.
-		conf_mem_file = os.path.join(self._eroot, CONFIG_MEMORY_FILE)
-		conf_mem_lock = lockfile(conf_mem_file)
+		self.vartree.dbapi._fs_lock()
 		try:
-			cfgfiledict = grabdict(conf_mem_file)
+			cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file)
 			if "NOCONFMEM" in self.settings:
 				cfgfiledict["IGNORE"]=1
 			else:
@@ -3360,12 +3383,11 @@ class dblink(object):
 					cfgfiledict["IGNORE"] = 1
 					break
 
-			rval = self._merge_contents(srcroot, destroot, cfgfiledict,
-				conf_mem_file)
+			rval = self._merge_contents(srcroot, destroot, cfgfiledict)
 			if rval != os.EX_OK:
 				return rval
 		finally:
-			unlockfile(conf_mem_lock)
+			self.vartree.dbapi._fs_unlock()
 
 		# These caches are populated during collision-protect and the data
 		# they contain is now invalid. It's very important to invalidate
@@ -3573,8 +3595,7 @@ class dblink(object):
 
 		# Lock the config memory file to prevent symlink creation
 		# in merge_contents from overlapping with env-update.
-		conf_mem_file = os.path.join(self._eroot, CONFIG_MEMORY_FILE)
-		conf_mem_lock = lockfile(conf_mem_file)
+		self.vartree.dbapi._fs_lock()
 		try:
 			#update environment settings, library paths. DO NOT change symlinks.
 			env_update(makelinks=(not downgrade),
@@ -3582,7 +3603,7 @@ class dblink(object):
 				contents=contents, env=self.settings.environ(),
 				writemsg_level=self._display_merge)
 		finally:
-			unlockfile(conf_mem_lock)
+			self.vartree.dbapi._fs_unlock()
 
 		# For gcc upgrades, preserved libs have to be removed after the
 		# the library path has been updated.
@@ -3610,7 +3631,7 @@ class dblink(object):
 
 		return backup_p
 
-	def _merge_contents(self, srcroot, destroot, cfgfiledict, conf_mem_file):
+	def _merge_contents(self, srcroot, destroot, cfgfiledict):
 
 		cfgfiledict_orig = cfgfiledict.copy()
 
@@ -3672,9 +3693,9 @@ class dblink(object):
 		# write out our collection of md5sums
 		if cfgfiledict != cfgfiledict_orig:
 			cfgfiledict.pop("IGNORE", None)
-			ensure_dirs(os.path.dirname(conf_mem_file),
+			ensure_dirs(os.path.dirname(self.vartree.dbapi._conf_mem_file),
 				gid=portage_gid, mode=0o2750, mask=0o2)
-			writedict(cfgfiledict, conf_mem_file)
+			writedict(cfgfiledict, self.vartree.dbapi._conf_mem_file)
 
 		return os.EX_OK
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     461564ae94ff936918eeaa18493bc1da3846796f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 14 23:53:35 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 14 23:53:35 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=461564ae

counter_tick_core: don't lock if parallel-install

Hopefully this avoids the following exception:

  File "/usr/lib/portage/pym/portage/locks.py", line 138, in lockfile
    fcntl.lockf(myfd, fcntl.LOCK_EX)
IOError: [Errno 35] Resource deadlock avoided

---
 pym/portage/dbapi/vartree.py |   13 +++++++++++--
 1 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index fca590a..802253a 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -789,7 +789,15 @@ class vardbapi(dbapi):
 		"""
 		myroot = None
 		mycpv = None
-		self.lock()
+		locked_vdb = False
+		if "parallel-install" not in self.settings.features:
+			# If parallel-install is enabled, it's unsafe to
+			# lock the vdb here since the portage.locks module
+			# does not behave as desired if we try to lock the
+			# same file multiple times concurrently from the
+			# same process.
+			self.lock()
+			locked_vdb = True
 		try:
 			counter = self.get_counter_tick_core() - 1
 			if self._cached_counter != counter:
@@ -803,7 +811,8 @@ class vardbapi(dbapi):
 					write_atomic(self._counter_path, str(counter))
 				self._cached_counter = counter
 		finally:
-			self.unlock()
+			if locked_vdb:
+				self.unlock()
 
 		return counter
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Arfrever Frehtes Taifersar Arahesis]

commit:     2610d89a6bc1f7df847878bbd1ebbfbe747dacc2
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Gentoo <DOT> Org>
AuthorDate: Sat May 14 22:56:35 2011 +0000
Commit:     Arfrever Frehtes Taifersar Arahesis <arfrever <AT> gentoo <DOT> org>
CommitDate: Sat May 14 22:56:35 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2610d89a

Fix a typo in a comment.

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 7d1e92b..fca590a 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3495,7 +3495,7 @@ class dblink(object):
 					if cpv != self.mycpv:
 						# If we've replaced another instance with the
 						# same cpv then the vdb entry no longer belongs
-						# to it, so we'll have to get the slot and couter
+						# to it, so we'll have to get the slot and counter
 						# from plib_registry._data instead.
 						try:
 							slot, counter = self.vartree.dbapi.aux_get(

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     828768a94b8aec78f2b044becf3c32a20fe05b4e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 14 22:14:45 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 14 22:14:45 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=828768a9

find_unused_preserved_libs: fix unmerge handling

Since commit c64d4abee145d083c70273be8fd23bd56dffe7ec (bug #286714),
preserved libs have not been properly garbage collected when packages
are unmerged without replacement, due to a difference in LinkageMap
state. It should behave correcly now.

---
 pym/portage/dbapi/vartree.py |   46 ++++++++++++++++++++++++++++-------------
 1 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a7e285c..7d1e92b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1492,20 +1492,11 @@ class dblink(object):
 				self._linkmap_rebuild(exclude_pkgs=exclude_pkgs,
 					include_file=needed, preserve_paths=preserve_paths)
 
-				unmerge_preserve = None
-				if unmerge and not unmerge_with_replacement:
-					unmerge_preserve = \
-						self._find_libs_to_preserve(unmerge=True)
-
-				cpv_lib_map = self._find_unused_preserved_libs()
-				if cpv_lib_map:
-					self._remove_preserved_libs(cpv_lib_map)
-					for cpv, removed in cpv_lib_map.items():
-						if not self.vartree.dbapi.cpv_exists(cpv):
-							continue
-						self.vartree.dbapi.removeFromContents(cpv, removed)
-
 				if unmerge:
+					unmerge_preserve = None
+					if not unmerge_with_replacement:
+						unmerge_preserve = \
+							self._find_libs_to_preserve(unmerge=True)
 					counter = self.vartree.dbapi.cpv_counter(self.mycpv)
 					plib_registry.unregister(self.mycpv,
 						self.settings["SLOT"], counter)
@@ -1524,6 +1515,17 @@ class dblink(object):
 						self.vartree.dbapi.removeFromContents(self,
 							unmerge_preserve)
 
+				unmerge_no_replacement = \
+					unmerge and not unmerge_with_replacement
+				cpv_lib_map = self._find_unused_preserved_libs(
+					unmerge_no_replacement)
+				if cpv_lib_map:
+					self._remove_preserved_libs(cpv_lib_map)
+					for cpv, removed in cpv_lib_map.items():
+						if not self.vartree.dbapi.cpv_exists(cpv):
+							continue
+						self.vartree.dbapi.removeFromContents(cpv, removed)
+
 				plib_registry.store()
 			finally:
 				plib_registry.unlock()
@@ -2307,7 +2309,7 @@ class dblink(object):
 	def _find_libs_to_preserve(self, unmerge=False):
 		"""
 		Get set of relative paths for libraries to be preserved. When
-		unmerge is False, file paths to preserver are selected from
+		unmerge is False, file paths to preserve are selected from
 		self._installed_instance. Otherwise, paths are selected from
 		self.
 		"""
@@ -2474,7 +2476,7 @@ class dblink(object):
 		outfile.close()
 		self._clear_contents_cache()
 
-	def _find_unused_preserved_libs(self):
+	def _find_unused_preserved_libs(self, unmerge_no_replacement):
 		"""
 		Find preserved libraries that don't have any consumers left.
 		"""
@@ -2527,12 +2529,26 @@ class dblink(object):
 		# installed library that is not preserved. This eliminates
 		# libraries that are erroneously preserved due to a move from one
 		# directory to another.
+		# Also eliminate consumers that are going to be unmerged if
+		# unmerge_no_replacement is True.
 		provider_cache = {}
 		for preserved_node in preserved_nodes:
 			soname = linkmap.getSoname(preserved_node)
 			for consumer_node in lib_graph.parent_nodes(preserved_node):
 				if consumer_node in preserved_nodes:
 					continue
+				if unmerge_no_replacement:
+					will_be_unmerged = True
+					for path in consumer_node.alt_paths:
+						if not self.isowner(path):
+							will_be_unmerged = False
+							break
+					if will_be_unmerged:
+						# This consumer is not preserved and it is
+						# being unmerged, so drop this edge.
+						lib_graph.remove_edge(preserved_node, consumer_node)
+						continue
+
 				providers = provider_cache.get(consumer_node)
 				if providers is None:
 					providers = linkmap.findProviders(consumer_node)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1a6f909f838f50f5abadf929d6ff7acf30705fbe
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 14 21:41:22 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 14 21:41:22 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1a6f909f

prune_plib_registry: display preserved libs

Since commit c64d4abee145d083c70273be8fd23bd56dffe7ec, sometimes we
preserve libs for unmerged packages here (bug #286714), so we should
display them like we do in _add_preserve_libs_to_contents().

---
 pym/portage/dbapi/vartree.py |    7 +++++++
 1 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 13d9ab2..a7e285c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1510,6 +1510,13 @@ class dblink(object):
 					plib_registry.unregister(self.mycpv,
 						self.settings["SLOT"], counter)
 					if unmerge_preserve:
+						for path in sorted(unmerge_preserve):
+							contents_key = self._match_contents(path)
+							if contents_key is None:
+								continue
+							obj_type = self.getcontents()[contents_key][0]
+							self._display_merge(_(">>> needed   %s %s\n") % \
+								(obj_type, contents_key), noiselevel=-1)
 						plib_registry.register(self.mycpv,
 							self.settings["SLOT"], counter, unmerge_preserve)
 						# Remove the preserved files from our contents

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     c331b8725f4ef53d7f65b624d085cd6dad5f29e9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 14 21:25:16 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 14 21:25:16 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c331b872

treewalk: fix plib_collisions with same cpv

---
 pym/portage/dbapi/vartree.py |   22 ++++++++++++++--------
 1 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index cdae340..13d9ab2 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3468,14 +3468,20 @@ class dblink(object):
 				for cpv, paths in plib_collisions.items():
 					if cpv not in plib_dict:
 						continue
-					if cpv == self.mycpv:
-						continue
-					has_vdb_entry = True
-					try:
-						slot, counter = self.vartree.dbapi.aux_get(
-							cpv, ["SLOT", "COUNTER"])
-					except KeyError:
-						has_vdb_entry = False
+					has_vdb_entry = False
+					if cpv != self.mycpv:
+						# If we've replaced another instance with the
+						# same cpv then the vdb entry no longer belongs
+						# to it, so we'll have to get the slot and couter
+						# from plib_registry._data instead.
+						try:
+							slot, counter = self.vartree.dbapi.aux_get(
+								cpv, ["SLOT", "COUNTER"])
+							has_vdb_entry = True
+						except KeyError:
+							pass
+
+					if not has_vdb_entry:
 						# It's possible for previously unmerged packages
 						# to have preserved libs in the registry, so try
 						# to retrieve the slot and counter from there.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     5d76c94e173df56aafcc1a5a740c01ffc51f2f0b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 12 19:02:27 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 12 19:02:27 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5d76c94e

porttree: use pure volatile cache if necessary

---
 pym/portage/dbapi/porttree.py |   18 ++++++++++++------
 1 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 379cc68..a30c9d7 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -170,12 +170,18 @@ class portdbapi(dbapi):
 		# to the cache entries/directories.
 		if secpass < 1 or not depcachedir_w_ok:
 			for x in self.porttrees:
-				db_ro = self.auxdbmodule(self.depcachedir, x,
-					filtered_auxdbkeys, gid=portage_gid, readonly=True)
-				self.auxdb[x] = metadata_overlay.database(
-					self.depcachedir, x, filtered_auxdbkeys,
-					gid=portage_gid, db_rw=volatile.database,
-					db_ro=db_ro)
+				try:
+					db_ro = self.auxdbmodule(self.depcachedir, x,
+						filtered_auxdbkeys, readonly=True, **cache_kwargs)
+				except CacheError:
+					self.auxdb[x] = volatile.database(
+						self.depcachedir, x, filtered_auxdbkeys,
+						**cache_kwargs)
+				else:
+					self.auxdb[x] = metadata_overlay.database(
+						self.depcachedir, x, filtered_auxdbkeys,
+						db_rw=volatile.database, db_ro=db_ro,
+						**cache_kwargs)
 		else:
 			for x in self.porttrees:
 				if x in self.auxdb:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     366d75c5ed399b3ad6709b103c32125da70ccdd8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 12 17:51:08 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 12 17:51:08 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=366d75c5

expand_new_virt: return early for non-virtual cat

---
 pym/portage/dbapi/_expand_new_virt.py |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/_expand_new_virt.py b/pym/portage/dbapi/_expand_new_virt.py
index 6fccd16..6d6a27d 100644
--- a/pym/portage/dbapi/_expand_new_virt.py
+++ b/pym/portage/dbapi/_expand_new_virt.py
@@ -13,6 +13,11 @@ def expand_new_virt(vardb, atom):
 	"""
 	if not isinstance(atom, Atom):
 		atom = Atom(atom)
+
+	if not atom.cp.startswith("virtual/"):
+		yield atom
+		return
+
 	traversed = set()
 	stack = [atom]
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     72e30f993ac7b964d445f5c922139574e2c9893d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 12 07:09:19 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 12 07:09:19 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=72e30f99

expand_new_virt: skip match for non-virtual cat

---
 pym/portage/dbapi/_expand_new_virt.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/_expand_new_virt.py b/pym/portage/dbapi/_expand_new_virt.py
index 7a233f1..6fccd16 100644
--- a/pym/portage/dbapi/_expand_new_virt.py
+++ b/pym/portage/dbapi/_expand_new_virt.py
@@ -18,7 +18,8 @@ def expand_new_virt(vardb, atom):
 
 	while stack:
 		atom = stack.pop()
-		if atom.blocker:
+		if atom.blocker or \
+			not atom.cp.startswith("virtual/"):
 			yield atom
 			continue
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f8008b86ae0f6b324c74cf8cbc999f2bca881e92
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 12 05:35:40 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 12 05:35:40 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f8008b86

treewalk: fix inverted counter init logic

---
 pym/portage/dbapi/vartree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 41a9329..989e969 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3277,7 +3277,7 @@ class dblink(object):
 			self.copyfile(inforoot+"/"+x)
 
 		# write local package counter for recording
-		if counter is not None:
+		if counter is None:
 			counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
 		codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
 			encoding=_encodings['fs'], errors='strict'),

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     15ec54ba9615feadbc2b6bb1d32df16a826c6159
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 12 01:14:54 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 12 01:14:54 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=15ec54ba

portdbapi: tweak cache permission handling

This is a reponse to the following issue:

  http://code.google.com/p/chromium-os/issues/detail?id=15234

---
 pym/portage/dbapi/porttree.py |   20 ++++++++++++--------
 1 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index cd74d21..379cc68 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -17,6 +17,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp',
 )
 
+from portage.cache import metadata_overlay, volatile
 from portage.cache.cache_errors import CacheError
 from portage.cache.mappings import Mapping
 from portage.dbapi import dbapi
@@ -40,6 +41,7 @@ import codecs
 import logging
 import stat
 import sys
+import traceback
 import warnings
 
 if sys.hexversion >= 0x3000000:
@@ -159,17 +161,14 @@ class portdbapi(dbapi):
 			'perms'   : 0o664
 		}
 
-		if secpass < 1:
-			# portage_gid is irrelevant, so just obey umask
-			cache_kwargs['gid']   = -1
-			cache_kwargs['perms'] = -1
-
 		# XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
 		# ~harring
 		filtered_auxdbkeys = [x for x in auxdbkeys if not x.startswith("UNUSED_0")]
 		filtered_auxdbkeys.sort()
-		from portage.cache import metadata_overlay, volatile
-		if not depcachedir_w_ok:
+		# If secpass < 1, we don't want to write to the cache
+		# since then we won't be able to apply group permissions
+		# to the cache entries/directories.
+		if secpass < 1 or not depcachedir_w_ok:
 			for x in self.porttrees:
 				db_ro = self.auxdbmodule(self.depcachedir, x,
 					filtered_auxdbkeys, gid=portage_gid, readonly=True)
@@ -370,7 +369,12 @@ class portdbapi(dbapi):
 				metadata[k] = ""
 			metadata["EAPI"] = "-" + eapi.lstrip("-")
 
-		self.auxdb[repo_path][cpv] = metadata
+		try:
+			self.auxdb[repo_path][cpv] = metadata
+		except CacheError:
+			# Normally this shouldn't happen, so we'll show
+			# a traceback for debugging purposes.
+			traceback.print_exc()
 		return metadata
 
 	def _pull_valid_cache(self, cpv, ebuild_path, repo_path):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0ed15bb56ce1f5d33a0974f197d99aa234c277af
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue May 10 19:52:36 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May 10 19:52:36 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0ed15bb5

MergeProcess: lock vdb earlier when appropriate

---
 pym/portage/dbapi/_MergeProcess.py |   30 +++++++++++++++++++++++++++++-
 1 files changed, 29 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 05f45d5..62f5bec 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -27,7 +27,7 @@ class MergeProcess(SpawnProcess):
 	__slots__ = ('dblink', 'mycat', 'mypkg', 'settings', 'treetype',
 		'vartree', 'scheduler', 'blockers', 'pkgloc', 'infloc', 'myebuild',
 		'mydbapi', 'prev_mtimes', '_elog_reader_fd', '_elog_reg_id',
-		'_buf', '_elog_keys')
+		'_buf', '_elog_keys', '_locked_vdb')
 
 	def _start(self):
 		# Portage should always call setcpv prior to this
@@ -48,6 +48,25 @@ class MergeProcess(SpawnProcess):
 		self._handle_self_reinstall()
 		super(MergeProcess, self)._start()
 
+	def _lock_vdb(self):
+		"""
+		Lock the vdb if FEATURES=parallel-install is NOT enabled,
+		otherwise do nothing. This is implemented with
+		vardbapi.lock(), which supports reentrance by the
+		subprocess that we spawn.
+		"""
+		if "parallel-install" not in self.settings.features:
+			self.vartree.dbapi.lock()
+			self._locked_vdb = True
+
+	def _unlock_vdb(self):
+		"""
+		Unlock the vdb if we hold a lock, otherwise do nothing.
+		"""
+		if self._locked_vdb:
+			self.vartree.dbapi.unlock()
+			self._locked_vdb = False
+
 	def _handle_self_reinstall(self):
 		"""
 		If portage is reinstalling itself, create temporary
@@ -143,6 +162,14 @@ class MergeProcess(SpawnProcess):
 		fd_pipes[elog_writer_fd] = elog_writer_fd
 		self._elog_reg_id = self.scheduler.register(elog_reader_fd,
 			self._registered_events, self._elog_output_handler)
+
+		# If a concurrent emerge process tries to install a package
+		# in the same SLOT as this one at the same time, there is an
+		# extremely unlikely chance that the COUNTER values will not be
+		# ordered correctly unless we lock the vdb here.
+		# FEATURES=parallel-install skips this lock in order to
+		# improve performance, and the risk is practically negligible.
+		self._lock_vdb()
 		counter = self.vartree.dbapi.counter_tick()
 
 		pid = os.fork()
@@ -211,6 +238,7 @@ class MergeProcess(SpawnProcess):
 		"""
 		Unregister from the scheduler and close open files.
 		"""
+		self._unlock_vdb()
 		if self._elog_reg_id is not None:
 			self.scheduler.unregister(self._elog_reg_id)
 			self._elog_reg_id = None

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e5a51b2b7952eabc56bed8e2e04b2596e7495e16
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue May 10 05:05:40 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May 10 05:05:40 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e5a51b2b

get_counter_tick_core: use cpv_all

It's more efficient to use cpv_all since cp_all calls cpv_all anyway,
and calls to cp_list induce additional stat calls.

---
 pym/portage/dbapi/vartree.py |   16 +++++++---------
 1 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 7c66027..41a9329 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -764,16 +764,14 @@ class vardbapi(dbapi):
 			# to newly installed packages having lower
 			# COUNTERs than the previous version in the
 			# same slot.
-			cp_list = self.cp_list
 			max_counter = counter
-			for cp in self.cp_all():
-				for cpv in cp_list(cp):
-					try:
-						pkg_counter = int(self.aux_get(cpv, ["COUNTER"])[0])
-					except (KeyError, OverflowError, ValueError):
-						continue
-					if pkg_counter > max_counter:
-						max_counter = pkg_counter
+			for cpv in self.cpv_all():
+				try:
+					pkg_counter = int(self.aux_get(cpv, ["COUNTER"])[0])
+				except (KeyError, OverflowError, ValueError):
+					continue
+				if pkg_counter > max_counter:
+					max_counter = pkg_counter
 
 		if counter < 0 and not new_vdb:
 			writemsg(_("!!! Initializing COUNTER to " \

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     928bc527a8c3eab3784df54226df2f2d83c6c85c
Author:     David James <davidjames <AT> chromium <DOT> org>
AuthorDate: Tue May 10 04:11:47 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May 10 04:47:42 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=928bc527

Cache counter values, avoiding I/O when it doesn't change.

This improves merge times by up to 25%, since looping over the vardb for
each package install is slow.

TEST=Emerge a bunch of packages, notice 25% speed improvement.
BUG=chromium-os:15112

Change-Id: I51dd617219cd1820ceeb702291bd790990995be4

---
 pym/portage/dbapi/_MergeProcess.py |    3 +-
 pym/portage/dbapi/vartree.py       |   71 +++++++++++++++++++-----------------
 2 files changed, 40 insertions(+), 34 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 12a0baf..05f45d5 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -143,6 +143,7 @@ class MergeProcess(SpawnProcess):
 		fd_pipes[elog_writer_fd] = elog_writer_fd
 		self._elog_reg_id = self.scheduler.register(elog_reader_fd,
 			self._registered_events, self._elog_output_handler)
+		counter = self.vartree.dbapi.counter_tick()
 
 		pid = os.fork()
 		if pid != 0:
@@ -196,7 +197,7 @@ class MergeProcess(SpawnProcess):
 		try:
 			rval = mylink.merge(self.pkgloc, self.infloc,
 				myebuild=self.myebuild, mydbapi=self.mydbapi,
-				prev_mtimes=self.prev_mtimes)
+				prev_mtimes=self.prev_mtimes, counter=counter)
 		except SystemExit:
 			raise
 		except:

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c81e99b..7c66027 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -166,6 +166,8 @@ class vardbapi(dbapi):
 			self._linkmap = LinkageMap(self)
 		self._owners = self._owners_db(self)
 
+		self._cached_counter = None
+
 	def getpath(self, mykey, filename=None):
 		# This is an optimized hotspot, so don't use unicode-wrapped
 		# os module and don't use os.path.join().
@@ -724,17 +726,6 @@ class vardbapi(dbapi):
 		@param myroot: ignored, self._eroot is used instead
 		"""
 		myroot = None
-		cp_list = self.cp_list
-		max_counter = 0
-		for cp in self.cp_all():
-			for cpv in cp_list(cp):
-				try:
-					counter = int(self.aux_get(cpv, ["COUNTER"])[0])
-				except (KeyError, OverflowError, ValueError):
-					continue
-				if counter > max_counter:
-					max_counter = counter
-
 		new_vdb = False
 		counter = -1
 		try:
@@ -762,16 +753,27 @@ class vardbapi(dbapi):
 				writemsg("!!! %s\n" % str(e), noiselevel=-1)
 				del e
 
-		# We must ensure that we return a counter
-		# value that is at least as large as the
-		# highest one from the installed packages,
-		# since having a corrupt value that is too low
-		# can trigger incorrect AUTOCLEAN behavior due
-		# to newly installed packages having lower
-		# COUNTERs than the previous version in the
-		# same slot.
-		if counter > max_counter:
+		if self._cached_counter == counter:
 			max_counter = counter
+		else:
+			# We must ensure that we return a counter
+			# value that is at least as large as the
+			# highest one from the installed packages,
+			# since having a corrupt value that is too low
+			# can trigger incorrect AUTOCLEAN behavior due
+			# to newly installed packages having lower
+			# COUNTERs than the previous version in the
+			# same slot.
+			cp_list = self.cp_list
+			max_counter = counter
+			for cp in self.cp_all():
+				for cpv in cp_list(cp):
+					try:
+						pkg_counter = int(self.aux_get(cpv, ["COUNTER"])[0])
+					except (KeyError, OverflowError, ValueError):
+						continue
+					if pkg_counter > max_counter:
+						max_counter = pkg_counter
 
 		if counter < 0 and not new_vdb:
 			writemsg(_("!!! Initializing COUNTER to " \
@@ -789,18 +791,19 @@ class vardbapi(dbapi):
 		"""
 		myroot = None
 		mycpv = None
-
 		self.lock()
 		try:
 			counter = self.get_counter_tick_core() - 1
-			if incrementing:
-				#increment counter
-				counter += 1
-				# use same permissions as config._init_dirs()
-				ensure_dirs(os.path.dirname(self._counter_path),
-					gid=portage_gid, mode=0o2750, mask=0o2)
-				# update new global counter file
-				write_atomic(self._counter_path, str(counter))
+			if self._cached_counter != counter:
+				if incrementing:
+					#increment counter
+					counter += 1
+					# use same permissions as config._init_dirs()
+					ensure_dirs(os.path.dirname(self._counter_path),
+						gid=portage_gid, mode=0o2750, mask=0o2)
+					# update new global counter file
+					write_atomic(self._counter_path, str(counter))
+				self._cached_counter = counter
 		finally:
 			self.unlock()
 
@@ -2860,7 +2863,7 @@ class dblink(object):
 				os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
 
 	def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
-		mydbapi=None, prev_mtimes=None):
+		mydbapi=None, prev_mtimes=None, counter=None):
 		"""
 		
 		This function does the following:
@@ -3276,7 +3279,8 @@ class dblink(object):
 			self.copyfile(inforoot+"/"+x)
 
 		# write local package counter for recording
-		counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
+		if counter is not None:
+			counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
 		codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
 			encoding=_encodings['fs'], errors='strict'),
 			'w', encoding=_encodings['repo.content'], errors='backslashreplace'
@@ -3906,7 +3910,7 @@ class dblink(object):
 				showMessage(zing + " " + mydest + "\n")
 
 	def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
-		mydbapi=None, prev_mtimes=None):
+		mydbapi=None, prev_mtimes=None, counter=None):
 		"""
 		@param myroot: ignored, self._eroot is used instead
 		"""
@@ -3918,7 +3922,8 @@ class dblink(object):
 		self.vartree.dbapi._bump_mtime(self.mycpv)
 		try:
 			retval = self.treewalk(mergeroot, myroot, inforoot, myebuild,
-				cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes)
+				cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes,
+				counter=counter)
 
 			# If PORTAGE_BUILDDIR doesn't exist, then it probably means
 			# fail-clean is enabled, and the success/die hooks have

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     ff65607c5dbd5f320dfbc4e77e97d7bfd69ab467
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May  9 20:36:22 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May  9 20:36:22 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ff65607c

cpv_expand: avoid old-style virtuals more

---
 pym/portage/dbapi/cpv_expand.py |    9 ++++++++-
 1 files changed, 8 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/cpv_expand.py b/pym/portage/dbapi/cpv_expand.py
index b77c283..7077b1f 100644
--- a/pym/portage/dbapi/cpv_expand.py
+++ b/pym/portage/dbapi/cpv_expand.py
@@ -26,7 +26,14 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
 			mykey=myslash[0]+"/"+mysplit[0]
 		else:
 			mykey=mycpv
-		if hasattr(mydb, "cp_list") and \
+
+		# Since Gentoo stopped using old-style virtuals in
+		# 2011, typically it's possible to avoid getvirtuals()
+		# calls entirely. Therefore, only call getvirtuals()
+		# if the atom category is "virtual" and cp_list()
+		# returns nothing.
+		if mykey.startswith("virtual/") and \
+			hasattr(mydb, "cp_list") and \
 			not mydb.cp_list(mykey, use_cache=use_cache):
 				if hasattr(mydb, "vartree"):
 					settings._populate_treeVirtuals_if_needed(mydb.vartree)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     16f3856ff76aca1ee668a464aefa4102b3cccb22
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May  9 06:23:07 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May  9 06:23:07 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=16f3856f

cpv_expand: populate old-style virts on demand

If old-style virtuals code is triggered here and we are given a
vartree, use it to avoid creating a temporary instance.

---
 pym/portage/dbapi/cpv_expand.py |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/cpv_expand.py b/pym/portage/dbapi/cpv_expand.py
index 26f9948..b77c283 100644
--- a/pym/portage/dbapi/cpv_expand.py
+++ b/pym/portage/dbapi/cpv_expand.py
@@ -28,6 +28,8 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
 			mykey=mycpv
 		if hasattr(mydb, "cp_list") and \
 			not mydb.cp_list(mykey, use_cache=use_cache):
+				if hasattr(mydb, "vartree"):
+					settings._populate_treeVirtuals_if_needed(mydb.vartree)
 				virts = settings.getvirtuals().get(mykey)
 				if virts:
 					mykey_orig = mykey
@@ -76,6 +78,8 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
 			mykey=matches[0]
 
 		if not mykey and not isinstance(mydb, list):
+			if hasattr(mydb, "vartree"):
+				settings._populate_treeVirtuals_if_needed(mydb.vartree)
 			virts_p = settings.get_virts_p().get(myp)
 			if virts_p:
 				mykey = virts_p[0]

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a983fc1c83402dbfe0a423bf729c72eeb795388c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May  9 04:51:47 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May  9 04:51:47 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a983fc1c

vardbapi: disable subprocess vdb cache updates

Since commit 7535cabdf2fab76fc55df83643157613dfd66be9,
vardbapi.flush_cache() is often called within subprocesses spawned
from MergeProcess. The _aux_cache_threshold doesn't work as designed
if the cache is flushed from a subprocess like this, can lead to the
vdb cache being flushed for every single merge. This is a waste of
disk IO, so disable vdb cache updates in subprocesses.

---
 pym/portage/dbapi/_MergeProcess.py |    3 +++
 pym/portage/dbapi/vartree.py       |    8 +++++++-
 2 files changed, 10 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index afb2e08..12a0baf 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -176,6 +176,9 @@ class MergeProcess(SpawnProcess):
 		# is triggered when mylink._scheduler is None.
 		mylink._scheduler = None
 
+		# Avoid wastful updates of the vdb cache.
+		self.vartree.dbapi._flush_cache_enabled = False
+
 		# In this subprocess we don't want PORTAGE_BACKGROUND to
 		# suppress stdout/stderr output since they are pipes. We
 		# also don't want to open PORTAGE_LOG_FILE, since it will

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a34f1ec..c81e99b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -108,6 +108,11 @@ class vardbapi(dbapi):
 		# have been added or removed.
 		self._pkgs_changed = False
 
+		# The _aux_cache_threshold doesn't work as designed
+		# if the cache is flushed from a subprocess, so we
+		# use this to avoid waste vdb cache updates.
+		self._flush_cache_enabled = True
+
 		#cache for category directory mtimes
 		self.mtdircache = {}
 
@@ -467,7 +472,8 @@ class vardbapi(dbapi):
 		superuser privileges (since that's required to obtain a lock), but all
 		users have read access and benefit from faster metadata lookups (as
 		long as at least part of the cache is still valid)."""
-		if self._aux_cache is not None and \
+		if self._flush_cache_enabled and \
+			self._aux_cache is not None and \
 			len(self._aux_cache["modified"]) >= self._aux_cache_threshold and \
 			secpass >= 2:
 			self._owners.populate() # index any unindexed contents

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     3a68bf5dff34a8e5e991778c1e28a3a6dda4d141
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May  8 20:19:47 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May  8 20:19:47 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3a68bf5d

Remove obsolete others_in_slot plibs code.

This code dates back to commit 79a2ade5a6cb5a6c2d7ace838a39a634265c5522
and the preserve-libs logic was different back then. At that time, the
dblink._preserve_libs() method would earlier register preserved libs
for the new package currently being merged, while the vardb entry was
still a -MERGING- node. Current code does not register preserve_paths
until later, so this others_in_slot code is obsolete.

---
 pym/portage/dbapi/vartree.py |   14 ++------------
 1 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 28b67f0..b4e231e 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1462,7 +1462,7 @@ class dblink(object):
 		self.contentscache = pkgfiles
 		return pkgfiles
 
-	def _prune_plib_registry(self, unmerge=False, others_in_slot=[],
+	def _prune_plib_registry(self, unmerge=False,
 		needed=None, preserve_paths=None):
 		# remove preserved libraries that don't have any consumers left
 		plib_registry = self.vartree.dbapi._plib_registry
@@ -1485,15 +1485,6 @@ class dblink(object):
 				if cpv_lib_map:
 					self._remove_preserved_libs(cpv_lib_map)
 					for cpv, removed in cpv_lib_map.items():
-						if not self.vartree.dbapi.cpv_exists(cpv):
-							for dblnk in others_in_slot:
-								if dblnk.mycpv == cpv:
-									# This one just got merged so it doesn't
-									# register with cpv_exists() yet.
-									self.vartree.dbapi.removeFromContents(
-										dblnk, removed)
-									break
-							continue
 						self.vartree.dbapi.removeFromContents(cpv, removed)
 
 				if unmerge:
@@ -1664,8 +1655,7 @@ class dblink(object):
 					showMessage(_("!!! FAILED postrm: %s\n") % retval,
 						level=logging.ERROR, noiselevel=-1)
 
-			self._prune_plib_registry(unmerge=True,
-				others_in_slot=others_in_slot, needed=needed,
+			self._prune_plib_registry(unmerge=True, needed=needed,
 				preserve_paths=preserve_paths)
 		finally:
 			self.vartree.dbapi._bump_mtime(self.mycpv)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f2091f05a0a5bf495098fcb58ea6912732c1c5a6
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May  8 19:31:07 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May  8 19:31:07 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f2091f05

treewalk: always initialize "needed"

---
 pym/portage/dbapi/vartree.py |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a4a4203..28b67f0 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3324,9 +3324,9 @@ class dblink(object):
 		linkmap = self.vartree.dbapi._linkmap
 		plib_registry = self.vartree.dbapi._plib_registry
 		include_file = None
-		if linkmap is None or plib_registry is None:
-			preserve_paths = None
-		else:
+		preserve_paths = None
+		needed = None
+		if not (linkmap is None or plib_registry is None):
 			plib_registry.lock()
 			try:
 				plib_registry.load()

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     05b16b9d1c4eacc3442b78152f46f5d07943de97
Author:     David James <davidjames <AT> chromium <DOT> org>
AuthorDate: Sun May  8 07:26:42 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May  8 07:30:13 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=05b16b9d

Fix typos in unmerge function.

BUG=none
TEST=Try unmerging some packages.

Change-Id: Ib643ec95b8da14b49a6b519e445fe57f8995c52c

---
 pym/portage/dbapi/vartree.py |   10 +++++-----
 1 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 92c467d..a4a4203 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4071,18 +4071,18 @@ def unmerge(cat, pkg, myroot=None, settings=None,
 	mylink = dblink(cat, pkg, settings=settings, treetype="vartree",
 		vartree=vartree, scheduler=scheduler)
 	vartree = mylink.vartree
-	parallel_install = "parallel-install" in self.settings.features
+	parallel_install = "parallel-install" in settings.features
 	if not parallel_install:
-		self.lockdb()
+		mylink.lockdb()
 	try:
 		if mylink.exists():
 			retval = mylink.unmerge(ldpath_mtimes=ldpath_mtimes)
 			if retval == os.EX_OK:
-				self.lockdb()
+				mylink.lockdb()
 				try:
 					mylink.delete()
 				finally:
-					self.unlockdb()
+					mylink.unlockdb()
 			return retval
 		return os.EX_OK
 	finally:
@@ -4092,7 +4092,7 @@ def unmerge(cat, pkg, myroot=None, settings=None,
 		else:
 			vartree.dbapi._linkmap._clear_cache()
 		if not parallel_install:
-			self.unlockdb()
+			mylink.unlockdb()
 
 def write_contents(contents, root, f):
 	"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     bc5d73fa3db0569c55d48c2e738f12742579536c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May  8 03:05:21 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May  8 04:35:09 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=bc5d73fa

vardbapi: acquire lock during counter_tick_core

---
 pym/portage/dbapi/vartree.py |   26 +++++++++++++++++---------
 1 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index bc9916e..2708e43 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -779,17 +779,25 @@ class vardbapi(dbapi):
 		to the global file.  Returns new counter value.
 
 		@param myroot: ignored, self._eroot is used instead
+		@param mycpv: ignored
 		"""
 		myroot = None
-		counter = self.get_counter_tick_core(mycpv=mycpv) - 1
-		if incrementing:
-			#increment counter
-			counter += 1
-			# use same permissions as config._init_dirs()
-			ensure_dirs(os.path.dirname(self._counter_path),
-				gid=portage_gid, mode=0o2750, mask=0o2)
-			# update new global counter file
-			write_atomic(self._counter_path, str(counter))
+		mycpv = None
+
+		self.lock()
+		try:
+			counter = self.get_counter_tick_core() - 1
+			if incrementing:
+				#increment counter
+				counter += 1
+				# use same permissions as config._init_dirs()
+				ensure_dirs(os.path.dirname(self._counter_path),
+					gid=portage_gid, mode=0o2750, mask=0o2)
+				# update new global counter file
+				write_atomic(self._counter_path, str(counter))
+		finally:
+			self.unlock()
+
 		return counter
 
 	def _dblink(self, cpv):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     646c21b911ca2d14fb0f8489ff22e05fe9c4199a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May  8 02:57:29 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May  8 04:34:58 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=646c21b9

Add vardbapi reentrant lock/unlock methods.

---
 pym/portage/dbapi/vartree.py |   54 +++++++++++++++++++++++++++++------------
 1 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 7f9fb99..bc9916e 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -131,6 +131,10 @@ class vardbapi(dbapi):
 				DeprecationWarning, stacklevel=2)
 
 		self._eroot = settings['EROOT']
+		self._dbroot = self._eroot + VDB_PATH
+		self._lock = None
+		self._lock_count = 0
+
 		if vartree is None:
 			vartree = portage.db[self.root]["vartree"]
 		self.vartree = vartree
@@ -167,6 +171,38 @@ class vardbapi(dbapi):
 			rValue = _os.path.join(rValue, filename)
 		return rValue
 
+	def lock(self):
+		"""
+		Acquire a reentrant lock, blocking, for cooperation with concurrent
+		processes. State is inherited by subprocesses, allowing subprocesses
+		to reenter a lock that was acquired by a parent process. However,
+		a lock can be released only by the same process that acquired it.
+		"""
+		if self._lock_count:
+			self._lock_count += 1
+		else:
+			if self._lock is not None:
+				raise AssertionError("already locked")
+			# At least the parent needs to exist for the lock file.
+			ensure_dirs(self._dbroot)
+			self._lock = lockdir(self._dbroot)
+			self._lock_count += 1
+
+	def unlock(self):
+		"""
+		Release a lock, decrementing the recursion level. Each unlock() call
+		must be matched with a prior lock() call, or else an AssertionError
+		will be raised if unlock() is called while not locked.
+		"""
+		if self._lock_count > 1:
+			self._lock_count -= 1
+		else:
+			if self._lock is None:
+				raise AssertionError("not locked")
+			self._lock_count = 0
+			unlockdir(self._lock)
+			self._lock = None
+
 	def _bump_mtime(self, cpv):
 		"""
 		This is called before an after any modifications, so that consumers
@@ -1226,10 +1262,6 @@ class dblink(object):
 		self.dbpkgdir = self.dbcatdir+"/"+pkg
 		self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg
 		self.dbdir = self.dbpkgdir
-
-		self._lock_vdb = None
-		self._lock_vdb_count = 0
-
 		self.settings = mysettings
 		self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1"
 
@@ -1269,20 +1301,10 @@ class dblink(object):
 		self._get_protect_obj().updateprotect()
 
 	def lockdb(self):
-		if self._lock_vdb_count:
-			self._lock_vdb_count += 1
-		else:
-			# At least the parent needs to exist for the lock file.
-			ensure_dirs(self.dbroot)
-			self._lock_vdb = lockdir(self.dbroot)
+		self.vartree.dbapi.lock()
 
 	def unlockdb(self):
-		if self._lock_vdb_count > 1:
-			self._lock_vdb_count -= 1
-		else:
-			self._lock_vdb_count = 0
-			unlockdir(self._lock_vdb)
-			self._lock_vdb = None
+		self.vartree.dbapi.unlock()
 
 	def getpath(self):
 		"return path to location of db information (for >>> informational display)"

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7edab967a1660094eb3f55fd13679d0939384f27
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 03:00:54 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May  3 03:01:48 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7edab967

cpv_expand: avoid old-style virtual code

This allows portageq calls to avoid loading vdb_metadata.pickle in
most cases, which greatly improves performance.

---
 pym/portage/dbapi/cpv_expand.py |   24 ++++++++++--------------
 1 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/pym/portage/dbapi/cpv_expand.py b/pym/portage/dbapi/cpv_expand.py
index c42e427..9ce0d3d 100644
--- a/pym/portage/dbapi/cpv_expand.py
+++ b/pym/portage/dbapi/cpv_expand.py
@@ -17,8 +17,6 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
 	mysplit = _pkgsplit(myslash[-1])
 	if settings is None:
 		settings = globals()["settings"]
-	virts = settings.getvirtuals()
-	virts_p = settings.get_virts_p()
 	if len(myslash)>2:
 		# this is illegal case.
 		mysplit=[]
@@ -28,24 +26,21 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
 			mykey=myslash[0]+"/"+mysplit[0]
 		else:
 			mykey=mycpv
-		if mydb and virts and mykey in virts:
-			writemsg("mydb.__class__: %s\n" % (mydb.__class__), 1)
-			if hasattr(mydb, "cp_list"):
-				if not mydb.cp_list(mykey, use_cache=use_cache):
-					writemsg("virts[%s]: %s\n" % (str(mykey),virts[mykey]), 1)
-					mykey_orig = mykey[:]
-					for vkey in virts[mykey]:
+		if hasattr(mydb, "cp_list") and \
+			not mydb.cp_list(mykey, use_cache=use_cache):
+				virts = settings.getvirtuals().get(mykey)
+				if virts:
+					mykey_orig = mykey
+					for vkey in virts:
 						# The virtuals file can contain a versioned atom, so
 						# it may be necessary to remove the operator and
 						# version from the atom before it is passed into
 						# dbapi.cp_list().
 						if mydb.cp_list(vkey.cp):
 							mykey = str(vkey)
-							writemsg(_("virts chosen: %s\n") % (mykey), 1)
 							break
 					if mykey == mykey_orig:
-						mykey = str(virts[mykey][0])
-						writemsg(_("virts defaulted: %s\n") % (mykey), 1)
+						mykey = str(virts[0])
 			#we only perform virtual expansion if we are passed a dbapi
 	else:
 		#specific cpv, no category, ie. "foo-1.0"
@@ -81,8 +76,9 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
 			mykey=matches[0]
 
 		if not mykey and not isinstance(mydb, list):
-			if myp in virts_p:
-				mykey=virts_p[myp][0]
+			virts_p = settings.get_virts_p().get(myp)
+			if virts_p:
+				mykey = virts_p[0]
 			#again, we only perform virtual expansion if we have a dbapi (not a list)
 		if not mykey:
 			mykey="null/"+myp

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1d6e6b2fe3b0107153b76dfe84ef8b2dc8240658
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 27 22:37:47 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 22:37:47 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1d6e6b2f

display_merge: tweak background/logging logic

---
 pym/portage/dbapi/vartree.py |    8 +++++---
 1 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index e9b183e..08263d6 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1731,11 +1731,13 @@ class dblink(object):
 		if self._scheduler is None:
 			writemsg_level(msg, level=level, noiselevel=noiselevel)
 		else:
-			log_path = self.settings.get("PORTAGE_LOG_FILE")
+			log_path = None
+			if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
+				log_path = self.settings.get("PORTAGE_LOG_FILE")
 			background = self.settings.get("PORTAGE_BACKGROUND") == "1"
 
-			if log_path is None:
-				if not (background and level < logging.WARN):
+			if background and log_path is None:
+				if level >= logging.WARN:
 					writemsg_level(msg, level=level, noiselevel=noiselevel)
 			else:
 				self._scheduler.output(msg,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     eee5cda2c8be4957b8e2e5fe1cd1bb42e2d4edcf
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 27 21:26:08 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 21:26:08 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=eee5cda2

dblink: rename 'buffer' var in _elog_process

Avoid name collision with built-in 'buffer' function.

---
 pym/portage/dbapi/vartree.py |   10 +++++-----
 1 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index afcbdc9..e9b183e 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2800,7 +2800,7 @@ class dblink(object):
 				"QA": "eqawarn",
 				"ERROR": "eerror"
 			}
-			buffer = []
+			str_buffer = []
 			for phase, messages in logentries.items():
 				for key, lines in messages:
 					funcname = funcnames[key]
@@ -2808,10 +2808,10 @@ class dblink(object):
 						lines = [lines]
 					for line in lines:
 						fields = (funcname, phase, cpv, line.rstrip('\n'))
-						buffer.append(' '.join(fields))
-						buffer.append('\n')
-			if buffer:
-				os.write(self._pipe, _unicode_encode(''.join(buffer)))
+						str_buffer.append(' '.join(fields))
+						str_buffer.append('\n')
+			if str_buffer:
+				os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
 
 	def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
 		mydbapi=None, prev_mtimes=None):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     a46057efe4782553e8e292759c6153b27e4b4fe9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 27 20:57:35 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 20:57:35 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a46057ef

unmerge: fix lock/UnsupportedAPIException logic

---
 pym/portage/dbapi/vartree.py |   19 ++++++++++++-------
 1 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index dd041d8..afcbdc9 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1544,16 +1544,21 @@ class dblink(object):
 				myebuildpath = None
 
 		builddir_lock = None
+		log_path = None
 		scheduler = self._scheduler
 		retval = os.EX_OK
 		try:
-			builddir_lock = EbuildBuildDir(
-				scheduler=scheduler,
-				settings=self.settings)
-			builddir_lock.lock()
-			prepare_build_dirs(settings=self.settings, cleanup=True)
-			log_path = self.settings.get("PORTAGE_LOG_FILE")
 			if myebuildpath:
+				# Only create builddir_lock if doebuild_environment
+				# succeeded, since that's needed to initialize
+				# PORTAGE_BUILDDIR.
+				builddir_lock = EbuildBuildDir(
+					scheduler=scheduler,
+					settings=self.settings)
+				builddir_lock.lock()
+				prepare_build_dirs(settings=self.settings, cleanup=True)
+				log_path = self.settings.get("PORTAGE_LOG_FILE")
+
 				phase = EbuildPhase(background=background,
 					phase=ebuild_phase, scheduler=scheduler,
 					settings=self.settings)
@@ -1668,7 +1673,7 @@ class dblink(object):
 
 					self._elog_process(phasefilter=("prerm", "postrm"))
 
-					if retval == os.EX_OK:
+					if retval == os.EX_OK and builddir_lock is not None:
 						# myebuildpath might be None, so ensure
 						# it has a sane value for the clean phase,
 						# even though it won't really be sourced.

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     cdf04882ce380579814b9cfe63ee593dbfba5ef9
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 27 06:09:27 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar 27 06:09:27 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cdf04882

MergeProcess: add fallback setcpv call

It's important that this metadata access happens in the parent process,
since closing of file descriptors in the subprocess can prevent access
to open database connections such as that used by the sqlite metadata
cache module.

---
 pym/portage/dbapi/_MergeProcess.py |   22 +++++++++++++++++++---
 1 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index ea7c3e6..afb2e08 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -30,6 +30,21 @@ class MergeProcess(SpawnProcess):
 		'_buf', '_elog_keys')
 
 	def _start(self):
+		# Portage should always call setcpv prior to this
+		# point, but here we have a fallback as a convenience
+		# for external API consumers. It's important that
+		# this metadata access happens in the parent process,
+		# since closing of file descriptors in the subprocess
+		# can prevent access to open database connections such
+		# as that used by the sqlite metadata cache module.
+		cpv = "%s/%s" % (self.mycat, self.mypkg)
+		settings = self.settings
+		if cpv != settings.mycpv or \
+			"IUSE" not in settings.configdict["pkg"]:
+			settings.reload()
+			settings.reset()
+			settings.setcpv(cpv, mydb=self.mydbapi)
+
 		self._handle_self_reinstall()
 		super(MergeProcess, self)._start()
 
@@ -116,9 +131,10 @@ class MergeProcess(SpawnProcess):
 			fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
 		blockers = None
 		if self.blockers is not None:
-			# Query blockers in the main process, since metadata cache
-			# queries may not work for some databases from within a
-			# subprocess. For example, sqlite is known to misbehave.
+			# Query blockers in the main process, since closing
+			# of file descriptors in the subprocess can prevent
+			# access to open database connections such as that
+			# used by the sqlite metadata cache module.
 			blockers = self.blockers()
 		mylink = self.dblink(self.mycat, self.mypkg, settings=self.settings,
 			treetype=self.treetype, vartree=self.vartree,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     dfc5e174bdbcb21a16c5d5bb464ea4fc43b456a5
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 26 23:25:15 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 26 23:25:15 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=dfc5e174

MergeProcess: query blockers in the main process

Metadata cache queries may not work for some databases from within a
subprocess. For example, sqlite is known to misbehave.

---
 pym/portage/dbapi/_MergeProcess.py |    8 +++++++-
 pym/portage/dbapi/vartree.py       |    4 +---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 43111cb..ea7c3e6 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -114,9 +114,15 @@ class MergeProcess(SpawnProcess):
 		elog_reader_fd, elog_writer_fd = os.pipe()
 		fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
 			fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
+		blockers = None
+		if self.blockers is not None:
+			# Query blockers in the main process, since metadata cache
+			# queries may not work for some databases from within a
+			# subprocess. For example, sqlite is known to misbehave.
+			blockers = self.blockers()
 		mylink = self.dblink(self.mycat, self.mypkg, settings=self.settings,
 			treetype=self.treetype, vartree=self.vartree,
-			blockers=self.blockers, scheduler=self.scheduler,
+			blockers=blockers, scheduler=self.scheduler,
 			pipe=elog_writer_fd)
 		fd_pipes[elog_writer_fd] = elog_writer_fd
 		self._elog_reg_id = self.scheduler.register(elog_reader_fd,

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 4be783d..45b5a69 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3076,9 +3076,7 @@ class dblink(object):
 				return 1
 
 		# check for package collisions
-		blockers = None
-		if self._blockers is not None:
-			blockers = self._blockers()
+		blockers = self._blockers
 		if blockers is None:
 			blockers = []
 		collisions, plib_collisions = \

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7ab1280c237d7c83fcc2227259f7b52a06445525
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 26 18:00:44 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 26 18:00:44 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7ab1280c

display_merge: pass through noiselevel args

---
 pym/portage/dbapi/vartree.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index f8ce331..4be783d 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1734,7 +1734,8 @@ class dblink(object):
 					writemsg_level(msg, level=level, noiselevel=noiselevel)
 			else:
 				self._scheduler.output(msg,
-					background=background, log_path=log_path)
+					log_path=log_path, background=background,
+					level=level, noiselevel=noiselevel)
 
 	def _unmerge_pkgfiles(self, pkgfiles, others_in_slot):
 		"""

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     84e8783a421238e9982e28800c7b71b4892515e1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 26 08:18:12 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 26 08:18:12 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=84e8783a

MergeProcess: relocate portage reinstall code

This code goes inside _start since it needs to execute in the parent
process.

---
 pym/portage/dbapi/_MergeProcess.py |   67 +++++++++++++++++++++++++++++++++++-
 pym/portage/dbapi/vartree.py       |   61 +--------------------------------
 2 files changed, 67 insertions(+), 61 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 10d8873..43111cb 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -1,14 +1,20 @@
 # Copyright 2010-2011 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import shutil
 import signal
+import tempfile
 import traceback
 
 import errno
 import fcntl
 import portage
 from portage import os, StringIO, _unicode_decode
+from portage.const import PORTAGE_PACKAGE_ATOM
+from portage.dep import match_from_list
 import portage.elog.messages
+from portage.elog import _preload_elog_modules
+from portage.util import ensure_dirs
 from _emerge.PollConstants import PollConstants
 from _emerge.SpawnProcess import SpawnProcess
 
@@ -23,6 +29,60 @@ class MergeProcess(SpawnProcess):
 		'mydbapi', 'prev_mtimes', '_elog_reader_fd', '_elog_reg_id',
 		'_buf', '_elog_keys')
 
+	def _start(self):
+		self._handle_self_reinstall()
+		super(MergeProcess, self)._start()
+
+	def _handle_self_reinstall(self):
+		"""
+		If portage is reinstalling itself, create temporary
+		copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH in order
+		to avoid relying on the new versions which may be
+		incompatible. Register an atexit hook to clean up the
+		temporary directories. Pre-load elog modules here since
+		we won't be able to later if they get unmerged (happens
+		when namespace changes).
+		"""
+
+		settings = self.settings
+		cpv = settings.mycpv
+		reinstall_self = False
+		if self.settings["ROOT"] == "/" and \
+			match_from_list(PORTAGE_PACKAGE_ATOM, [cpv]):
+			inherited = frozenset(self.settings.get('INHERITED', '').split())
+			if not self.vartree.dbapi.cpv_exists(cpv) or \
+				'9999' in cpv or \
+				'git' in inherited or \
+				'git-2' in inherited:
+				reinstall_self = True
+
+		if reinstall_self:
+			# Load lazily referenced portage submodules into memory,
+			# so imports won't fail during portage upgrade/downgrade.
+			_preload_elog_modules(self.settings)
+			portage.proxy.lazyimport._preload_portage_submodules()
+
+			# Make the temp directory inside $PORTAGE_TMPDIR/portage, since
+			# it's common for /tmp and /var/tmp to be mounted with the
+			# "noexec" option (see bug #346899).
+			build_prefix = os.path.join(settings["PORTAGE_TMPDIR"], "portage")
+			ensure_dirs(build_prefix)
+			base_path_tmp = tempfile.mkdtemp(
+				"", "._portage_reinstall_.", build_prefix)
+			portage.process.atexit_register(shutil.rmtree, base_path_tmp)
+			dir_perms = 0o755
+			for subdir in "bin", "pym":
+				var_name = "PORTAGE_%s_PATH" % subdir.upper()
+				var_orig = settings[var_name]
+				var_new = os.path.join(base_path_tmp, subdir)
+				settings[var_name] = var_new
+				settings.backup_changes(var_name)
+				shutil.copytree(var_orig, var_new, symlinks=True)
+				os.chmod(var_new, dir_perms)
+			portage._bin_path = settings['PORTAGE_BIN_PATH']
+			portage._pym_path = settings['PORTAGE_PYM_PATH']
+			os.chmod(base_path_tmp, dir_perms)
+
 	def _elog_output_handler(self, fd, event):
 		output = None
 		if event & PollConstants.POLLIN:
@@ -51,7 +111,6 @@ class MergeProcess(SpawnProcess):
 		dblink.merge().
 		"""
 
-		files = self._files
 		elog_reader_fd, elog_writer_fd = os.pipe()
 		fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
 			fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
@@ -69,7 +128,13 @@ class MergeProcess(SpawnProcess):
 			self._elog_reader_fd = elog_reader_fd
 			self._buf = ""
 			self._elog_keys = set()
+
+			# invalidate relevant vardbapi caches
+			if self.vartree.dbapi._categories is not None:
+				self.vartree.dbapi._categories = None
 			self.vartree.dbapi._pkgs_changed = True
+			self.vartree.dbapi._clear_pkg_cache(mylink)
+
 			portage.process.spawned_pids.append(pid)
 			return [pid]
 

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 98f578f..f8ce331 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -14,7 +14,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.dep:dep_getkey,isjustname,match_from_list,' + \
 	 	'use_reduce,_slot_re',
 	'portage.elog:collect_ebuild_messages,collect_messages,' + \
-		'elog_process,_merge_logentries,_preload_elog_modules',
+		'elog_process,_merge_logentries',
 	'portage.locks:lockdir,unlockdir',
 	'portage.output:bold,colorize',
 	'portage.package.ebuild.doebuild:doebuild_environment,' + \
@@ -3826,65 +3826,6 @@ class dblink(object):
 	def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
 		mydbapi=None, prev_mtimes=None):
 		"""
-		If portage is reinstalling itself, create temporary
-		copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH in order
-		to avoid relying on the new versions which may be
-		incompatible. Register an atexit hook to clean up the
-		temporary directories. Pre-load elog modules here since
-		we won't be able to later if they get unmerged (happens
-		when namespace changes).
-
-		@param myroot: ignored, self._eroot is used instead
-		"""
-		myroot = None
-		if self.vartree.dbapi._categories is not None:
-			self.vartree.dbapi._categories = None
-		reinstall_self = False
-		if self.myroot == "/" and \
-			match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]):
-			inherited = frozenset(self.settings.get('INHERITED', '').split())
-			if not self.vartree.dbapi.cpv_exists(self.mycpv) or \
-				'9999' in self.mycpv or \
-				'git' in inherited or \
-				'git-2' in inherited:
-				reinstall_self = True
-
-		if reinstall_self:
-			# Load lazily referenced portage submodules into memory,
-			# so imports won't fail during portage upgrade/downgrade.
-			portage.proxy.lazyimport._preload_portage_submodules()
-			settings = self.settings
-
-			# Make the temp directory inside $PORTAGE_TMPDIR/portage, since
-			# it's common for /tmp and /var/tmp to be mounted with the
-			# "noexec" option (see bug #346899).
-			build_prefix = os.path.join(settings["PORTAGE_TMPDIR"], "portage")
-			ensure_dirs(build_prefix)
-			base_path_tmp = tempfile.mkdtemp(
-				"", "._portage_reinstall_.", build_prefix)
-			portage.process.atexit_register(shutil.rmtree, base_path_tmp)
-			dir_perms = 0o755
-			for subdir in "bin", "pym":
-				var_name = "PORTAGE_%s_PATH" % subdir.upper()
-				var_orig = settings[var_name]
-				var_new = os.path.join(base_path_tmp, subdir)
-				settings[var_name] = var_new
-				settings.backup_changes(var_name)
-				shutil.copytree(var_orig, var_new, symlinks=True)
-				os.chmod(var_new, dir_perms)
-			portage._bin_path = settings['PORTAGE_BIN_PATH']
-			portage._pym_path = settings['PORTAGE_PYM_PATH']
-			os.chmod(base_path_tmp, dir_perms)
-			# This serves so pre-load the modules.
-			_preload_elog_modules(self.settings)
-
-		return self._merge(mergeroot, inforoot,
-				myebuild=myebuild, cleanup=cleanup,
-				mydbapi=mydbapi, prev_mtimes=prev_mtimes)
-
-	def _merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
-		mydbapi=None, prev_mtimes=None):
-		"""
 		@param myroot: ignored, self._eroot is used instead
 		"""
 		myroot = None

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     1d73babb0e5c1eb735cd8ac16e4950f4d96f3b18
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 26 02:43:12 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 26 02:43:12 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1d73babb

dblink: use self.mycpv in _elog

---
 pym/portage/dbapi/vartree.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index d463c3a..c90d86b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2762,7 +2762,7 @@ class dblink(object):
 		func = getattr(portage.elog.messages, funcname)
 		if self._scheduler is None:
 			for l in lines:
-				func(l, phase=phase, key=self.settings.mycpv)
+				func(l, phase=phase, key=self.mycpv)
 		else:
 			background = self.settings.get("PORTAGE_BACKGROUND") == "1"
 			log_path = None
@@ -2770,7 +2770,7 @@ class dblink(object):
 				log_path = self.settings.get("PORTAGE_LOG_FILE")
 			out = portage.StringIO()
 			for line in lines:
-				func(line, phase=phase, key=self.settings.mycpv, out=out)
+				func(line, phase=phase, key=self.mycpv, out=out)
 			msg = out.getvalue()
 			self._scheduler.output(msg,
 				background=background, log_path=log_path)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     d823364e75eccac43faeaeaa5942183f6b31314f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 26 02:00:16 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 26 02:00:16 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d823364e

MergeProcess: toggle vardbapi._pkgs_changed

---
 pym/portage/dbapi/_MergeProcess.py |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index b5c45b3..809af37 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -67,6 +67,7 @@ class MergeProcess(SpawnProcess):
 			os.close(elog_writer_fd)
 			self._elog_reader_fd = elog_reader_fd
 			self._buf = ""
+			self.vartree.dbapi._pkgs_changed = True
 			portage.process.spawned_pids.append(pid)
 			return [pid]
 

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     176e2c74b368c73b593553773e749b3f25ddad72
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 25 20:52:16 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 20:52:16 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=176e2c74

MergeProcess: handle unicode in elog pipe

---
 pym/portage/dbapi/_MergeProcess.py |    4 ++--
 pym/portage/dbapi/vartree.py       |    2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 5caeef3..b5c45b3 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -7,7 +7,7 @@ import traceback
 import errno
 import fcntl
 import portage
-from portage import os, StringIO
+from portage import os, StringIO, _unicode_decode
 import portage.elog.messages
 from _emerge.PollConstants import PollConstants
 from _emerge.SpawnProcess import SpawnProcess
@@ -32,7 +32,7 @@ class MergeProcess(SpawnProcess):
 				if e.errno not in (errno.EAGAIN, errno.EINTR):
 					raise
 		if output:
-			lines = output.split('\n')
+			lines = _unicode_decode(output).split('\n')
 			if len(lines) == 1:
 				self._buf += lines[0]
 			else:

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 0e60af8..d463c3a 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2802,7 +2802,7 @@ class dblink(object):
 						buffer.append(' '.join(fields))
 						buffer.append('\n')
 			if buffer:
-				os.write(self._pipe, ''.join(buffer))
+				os.write(self._pipe, _unicode_encode(''.join(buffer)))
 
 	def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
 		mydbapi=None, prev_mtimes=None):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     fe6a9433426d284aabdf774376082fbed1741478
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 25 10:39:15 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 10:39:15 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fe6a9433

dblink: fix elog bugs for unmerge phases

---
 pym/portage/dbapi/vartree.py |   40 ++++++++++++++++++++++------------------
 1 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index e900455..0e60af8 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1544,14 +1544,13 @@ class dblink(object):
 		scheduler = self._scheduler
 		retval = os.EX_OK
 		try:
+			builddir_lock = EbuildBuildDir(
+				scheduler=scheduler,
+				settings=self.settings)
+			builddir_lock.lock()
+			prepare_build_dirs(settings=self.settings, cleanup=True)
+			log_path = self.settings.get("PORTAGE_LOG_FILE")
 			if myebuildpath:
-				builddir_lock = EbuildBuildDir(
-					scheduler=scheduler,
-					settings=self.settings)
-				builddir_lock.lock()
-
-				prepare_build_dirs(settings=self.settings, cleanup=True)
-				log_path = self.settings.get("PORTAGE_LOG_FILE")
 				phase = EbuildPhase(background=background,
 					phase=ebuild_phase, scheduler=scheduler,
 					settings=self.settings)
@@ -2754,22 +2753,27 @@ class dblink(object):
 		return 1
 
 	def _eqawarn(self, phase, lines):
-		from portage.elog.messages import eqawarn as _eqawarn
-		if self._scheduler is None:
-			for l in lines:
-				_eqawarn(l, phase=phase, key=self.settings.mycpv)
-		else:
-			self._scheduler.dblinkElog(self,
-				phase, _eqawarn, lines)
+		self._elog("eqawarn", phase, lines)
 
 	def _eerror(self, phase, lines):
-		from portage.elog.messages import eerror as _eerror
+		self._elog("eerror", phase, lines)
+
+	def _elog(self, funcname, phase, lines):
+		func = getattr(portage.elog.messages, funcname)
 		if self._scheduler is None:
 			for l in lines:
-				_eerror(l, phase=phase, key=self.settings.mycpv)
+				func(l, phase=phase, key=self.settings.mycpv)
 		else:
-			self._scheduler.dblinkElog(self,
-				phase, _eerror, lines)
+			background = self.settings.get("PORTAGE_BACKGROUND") == "1"
+			log_path = None
+			if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
+				log_path = self.settings.get("PORTAGE_LOG_FILE")
+			out = portage.StringIO()
+			for line in lines:
+				func(line, phase=phase, key=self.settings.mycpv, out=out)
+			msg = out.getvalue()
+			self._scheduler.output(msg,
+				background=background, log_path=log_path)
 
 	def _elog_process(self, phasefilter=None):
 		cpv = self.mycpv

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     0c4d01435737ba71d9c628ee34849c36ecec140b
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 25 10:12:55 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 10:12:55 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0c4d0143

vartree: remove broken scheduler reference

---
 pym/portage/dbapi/vartree.py |    3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 4f8c2ee..e900455 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3145,8 +3145,7 @@ class dblink(object):
 					# get_owners is slow for large numbers of files, so
 					# don't look them all up.
 					collisions = collisions[:20]
-				owners = self.vartree.dbapi._owners.get_owners(collisions,
-					scheduler=self._scheduler)
+				owners = self.vartree.dbapi._owners.get_owners(collisions)
 				self.vartree.dbapi.flush_cache()
 
 				for pkg, owned_files in owners.items():

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     7ffa0683cd9c40e630488af5783c549bee5cd3c8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 25 09:47:17 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 09:47:17 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7ffa0683

vartree: remove unused scheduler references

Since all the merge code runs inside MergeProcess now, there's no
reason to yield to the scheduler.

---
 pym/portage/dbapi/vartree.py |   40 +++++++---------------------------------
 1 files changed, 7 insertions(+), 33 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 8203de8..4f8c2ee 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -878,7 +878,7 @@ class vardbapi(dbapi):
 		def populate(self):
 			self._populate()
 
-		def _populate(self, scheduler=None):
+		def _populate(self):
 			owners_cache = vardbapi._owners_cache(self._vardb)
 			cached_hashes = set()
 			base_names = self._vardb._aux_cache["owners"]["base_names"]
@@ -902,10 +902,6 @@ class vardbapi(dbapi):
 
 			# Cache any missing packages.
 			for cpv in uncached_pkgs:
-
-				if scheduler is not None:
-					scheduler.scheduleYield()
-
 				owners_cache.add(cpv)
 
 			# Delete any stale cache.
@@ -919,12 +915,12 @@ class vardbapi(dbapi):
 
 			return owners_cache
 
-		def get_owners(self, path_iter, scheduler=None):
+		def get_owners(self, path_iter):
 			"""
 			@return the owners as a dblink -> set(files) mapping.
 			"""
 			owners = {}
-			for owner, f in self.iter_owners(path_iter, scheduler=scheduler):
+			for owner, f in self.iter_owners(path_iter):
 				owned_files = owners.get(owner)
 				if owned_files is None:
 					owned_files = set()
@@ -944,7 +940,7 @@ class vardbapi(dbapi):
 					owner_set.add(pkg_dblink)
 			return file_owners
 
-		def iter_owners(self, path_iter, scheduler=None):
+		def iter_owners(self, path_iter):
 			"""
 			Iterate over tuples of (dblink, path). In order to avoid
 			consuming too many resources for too much time, resources
@@ -956,7 +952,7 @@ class vardbapi(dbapi):
 
 			if not isinstance(path_iter, list):
 				path_iter = list(path_iter)
-			owners_cache = self._populate(scheduler=scheduler)
+			owners_cache = self._populate()
 			vardb = self._vardb
 			root = vardb._eroot
 			hash_pkg = owners_cache._hash_pkg
@@ -1015,23 +1011,19 @@ class vardbapi(dbapi):
 								if dblink(cpv).isowner(path):
 									owners.append((cpv, path))
 
-							if scheduler is not None:
-								scheduler.scheduleYield()
-
 					except StopIteration:
 						path_iter.append(path)
 						del owners[:]
 						dblink_cache.clear()
 						gc.collect()
-						for x in self._iter_owners_low_mem(path_iter,
-							scheduler=scheduler):
+						for x in self._iter_owners_low_mem(path_iter):
 							yield x
 						return
 					else:
 						for cpv, p in owners:
 							yield (dblink(cpv), p)
 
-		def _iter_owners_low_mem(self, path_list, scheduler=None):
+		def _iter_owners_low_mem(self, path_list):
 			"""
 			This implemention will make a short-lived dblink instance (and
 			parse CONTENTS) for every single installed package. This is
@@ -1053,10 +1045,6 @@ class vardbapi(dbapi):
 
 			root = self._vardb._eroot
 			for cpv in self._vardb.cpv_all():
-
-				if scheduler is not None:
-					scheduler.scheduleYield()
-
 				dblnk =  self._vardb._dblink(cpv)
 
 				for path, name, is_basename in path_info_list:
@@ -1197,10 +1185,6 @@ class dblink(object):
 		r')$'
 	)
 
-	# When looping over files for merge/unmerge, temporarily yield to the
-	# scheduler each time this many files are processed.
-	_file_merge_yield_interval = 20
-
 	def __init__(self, cat, pkg, myroot=None, settings=None, treetype=None,
 		vartree=None, blockers=None, scheduler=None, pipe=None):
 		"""
@@ -2586,7 +2570,6 @@ class dblink(object):
 			plib_collisions = {}
 
 			showMessage = self._display_merge
-			scheduler = self._scheduler
 			stopmerge = False
 			collisions = []
 			destroot = self.settings['ROOT']
@@ -2596,10 +2579,6 @@ class dblink(object):
 				if i % 1000 == 0 and i != 0:
 					showMessage(_("%d files checked ...\n") % i)
 
-				if scheduler is not None and \
-					0 == i % self._file_merge_yield_interval:
-					scheduler.scheduleYield()
-
 				dest_path = normalize_path(
 					os.path.join(destroot, f.lstrip(os.path.sep)))
 				try:
@@ -2708,7 +2687,6 @@ class dblink(object):
 		os = _os_merge
 
 		showMessage = self._display_merge
-		scheduler = self._scheduler
 
 		file_paths = set()
 		for dblnk in installed_instances:
@@ -2717,10 +2695,6 @@ class dblink(object):
 		real_paths = set()
 		for i, path in enumerate(file_paths):
 
-			if scheduler is not None and \
-				0 == i % self._file_merge_yield_interval:
-				scheduler.scheduleYield()
-
 			if os is _os_merge:
 				try:
 					_unicode_encode(path,

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     f3c4a464cc38788a4946de5331c2618c183fccac
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 25 08:23:51 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 08:23:51 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f3c4a464

ebuild(1): fix AttributeError for merge phase

  File "pym/portage/dbapi/vartree.py", line 4043, in merge
    merge_task.start()
  File "pym/_emerge/AsynchronousTask.py", line 23, in start
    self._start()
  File "pym/_emerge/SpawnProcess.py", line 118, in _start
    self._reg_id = self.scheduler.register(files.process.fileno(),
AttributeError: 'NoneType' object has no attribute 'register'

---
 pym/portage/dbapi/vartree.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a8888ae..a58521a 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4036,7 +4036,8 @@ def merge(mycat, mypkg, pkgloc, infloc,
 	background = (settings.get('PORTAGE_BACKGROUND') == '1')
 	merge_task = MergeProcess(
 		dblink=dblink, mycat=mycat, mypkg=mypkg, settings=settings,
-		treetype=mytree, vartree=vartree, scheduler=scheduler,
+		treetype=mytree, vartree=vartree,
+		scheduler=(scheduler or PollScheduler().sched_iface),
 		background=background, blockers=blockers, pkgloc=pkgloc,
 		infloc=infloc, myebuild=myebuild, mydbapi=mydbapi,
 		prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE'))

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     981f27f79e607877e7b8e47a904e3972d0e8336e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 25 08:11:45 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 08:11:45 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=981f27f7

MergeProcess: close elog_writer_fd leak

---
 pym/portage/dbapi/_MergeProcess.py |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index a8c3c9d..8f61a2f 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -64,6 +64,7 @@ class MergeProcess(SpawnProcess):
 
 		pid = os.fork()
 		if pid != 0:
+			os.close(elog_writer_fd)
 			self._elog_reader_fd = elog_reader_fd
 			self._buf = ""
 			portage.process.spawned_pids.append(pid)

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     906f42f51dcb10b390732b91409059400c5fb609
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 16 20:58:17 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar 16 20:58:17 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=906f42f5

iter_owners: try the hashed approach in any case

Previously, the slow approach was used whenever more than 10 files were
given as input. However, large numbers of files don't necessarily
mean the hashed approach is a bad choice, so try it in any case.

---
 pym/portage/dbapi/vartree.py |    8 --------
 1 files changed, 0 insertions(+), 8 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 32b4852..bf48b15 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -954,15 +954,7 @@ class vardbapi(dbapi):
 
 			if not isinstance(path_iter, list):
 				path_iter = list(path_iter)
-
-			if len(path_iter) > 10:
-				for x in self._iter_owners_low_mem(path_iter,
-					scheduler=scheduler):
-					yield x
-				return
-
 			owners_cache = self._populate(scheduler=scheduler)
-
 			vardb = self._vardb
 			root = vardb._eroot
 			hash_pkg = owners_cache._hash_pkg

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     dee9b51ee32e9d49794f737d6e3d014ec8a3d683
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Mar  7 17:34:02 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Mar  7 17:34:02 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=dee9b51e

portdbapi: source ebuilds when eclass dir missing

This will fix bug #357581.

---
 pym/portage/dbapi/porttree.py |    2 --
 1 files changed, 0 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 2e5f8ce..bcf64dc 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -464,8 +464,6 @@ class portdbapi(dbapi):
 		if doregen:
 			if myebuild in self._broken_ebuilds:
 				raise KeyError(mycpv)
-			if not self._have_root_eclass_dir:
-				raise KeyError(mycpv)
 
 			self.doebuild_settings.setcpv(mycpv)
 			eapi = None

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     96deeb3f4b83ffed0bef6b22afa7d0014f682f66
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Mar  2 19:46:57 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar  2 19:46:57 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=96deeb3f

Remove redundant CONTENTS parent dir generation.

This code is redunant now, since similar code has been added to
dblink.getcontents() in commit
e0b4048274b2cfc1617f2eff379cbe8435e13ed4.

---
 pym/portage/dbapi/vartree.py |   29 +----------------------------
 1 files changed, 1 insertions(+), 28 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index cd8b1c1..376b382 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -824,23 +824,8 @@ class vardbapi(dbapi):
 				# Empty path is a code used to represent empty contents.
 				self._add_path("", pkg_hash)
 
-			# When adding paths, implicitly add parent directories,
-			# since we can't necessarily assume that they are
-			# explicitly listed in CONTENTS.
-			added_paths = set()
 			for x in contents:
-				x = x[eroot_len:]
-				added_paths.add(x)
-				self._add_path(x, pkg_hash)
-				x_split = x.split(os.sep)
-				x_split.pop()
-				while x_split:
-					parent = os.sep.join(x_split)
-					if parent in added_paths:
-						break
-					added_paths.add(parent)
-					self._add_path(parent, pkg_hash)
-					x_split.pop()
+				self._add_path(x[eroot_len:], pkg_hash)
 
 			self._vardb._aux_cache["modified"].add(cpv)
 
@@ -1894,18 +1879,6 @@ class dblink(object):
 							os = portage.os
 							perf_md5 = portage.checksum.perform_md5
 
-				# Try to unmerge parent directories of everything
-				# listed in CONTENTS, since we can't necessarily
-				# assume that directories are listed in CONTENTS.
-				obj_split = obj.split(os.sep)
-				obj_split.pop()
-				while len(obj_split) > eroot_split_len:
-					parent = os.sep.join(obj_split)
-					if parent in mydirs:
-						break
-					mydirs.add(parent)
-					obj_split.pop()
-
 				file_data = pkgfiles[objkey]
 				file_type = file_data[0]
 				statobj = None

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     e0b4048274b2cfc1617f2eff379cbe8435e13ed4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Mar  2 18:51:41 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar  2 18:51:41 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e0b40482

dblink.getcontents(): generate parent directories

We can't necessarily assume that they are explicitly listed in
CONTENTS, and they are crucial for callers such as dblink.isowner().

---
 pym/portage/dbapi/vartree.py |   16 ++++++++++++++++
 1 files changed, 16 insertions(+), 0 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 04e66ab..cd8b1c1 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1404,6 +1404,9 @@ class dblink(object):
 		myroot = self.settings['ROOT']
 		if myroot == os.path.sep:
 			myroot = None
+		# used to generate parent dir entries
+		dir_entry = (_unicode_decode("dir"),)
+		eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1
 		pos = 0
 		errors = []
 		for pos, line in enumerate(mylines):
@@ -1448,6 +1451,19 @@ class dblink(object):
 			if myroot is not None:
 				path = os.path.join(myroot, path.lstrip(os.path.sep))
 
+			# Implicitly add parent directories, since we can't necessarily
+			# assume that they are explicitly listed in CONTENTS, and it's
+			# useful for callers if they can rely on parent directory entries
+			# being generated here (crucial for things like dblink.isowner()).
+			path_split = path.split(os.sep)
+			path_split.pop()
+			while len(path_split) > eroot_split_len:
+				parent = os.sep.join(path_split)
+				if parent in pkgfiles:
+					break
+				pkgfiles[parent] = dir_entry
+				path_split.pop()
+
 			pkgfiles[path] = data
 
 		if errors:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     6a505e814a0392463bd081b9c433e22001b8380e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Mar  1 21:20:59 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Mar  1 21:20:59 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6a505e81

unmerge: log pre/postrm FAILED messages

---
 pym/portage/dbapi/vartree.py |   17 +++++++++++------
 1 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index af3b5d9..04e66ab 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1519,6 +1519,7 @@ class dblink(object):
 		# Now, don't assume that the name of the ebuild is the same as the
 		# name of the dir; the package may have been moved.
 		myebuildpath = None
+		failures = 0
 		ebuild_phase = "prerm"
 		log_path = None
 		mystuff = os.listdir(self.dbdir)
@@ -1538,16 +1539,18 @@ class dblink(object):
 				doebuild_environment(myebuildpath, "prerm",
 					settings=self.settings, db=self.vartree.dbapi)
 			except UnsupportedAPIException as e:
+				failures += 1
 				# Sometimes this happens due to corruption of the EAPI file.
-				writemsg(_("!!! FAILED prerm: %s\n") % \
-					os.path.join(self.dbdir, "EAPI"), noiselevel=-1)
-				writemsg("%s\n" % str(e), noiselevel=-1)
+				showMessage(_("!!! FAILED prerm: %s\n") % \
+					os.path.join(self.dbdir, "EAPI"),
+					level=logging.ERROR, noiselevel=-1)
+				showMessage("%s\n" % (e,),
+					level=logging.ERROR, noiselevel=-1)
 				myebuildpath = None
 
 		builddir_lock = None
 		scheduler = self._scheduler
 		retval = os.EX_OK
-		failures = 0
 		try:
 			if myebuildpath:
 				builddir_lock = EbuildBuildDir(
@@ -1567,7 +1570,8 @@ class dblink(object):
 				# XXX: Decide how to handle failures here.
 				if retval != os.EX_OK:
 					failures += 1
-					writemsg(_("!!! FAILED prerm: %s\n") % retval, noiselevel=-1)
+					showMessage(_("!!! FAILED prerm: %s\n") % retval,
+						level=logging.ERROR, noiselevel=-1)
 
 			self._unmerge_pkgfiles(pkgfiles, others_in_slot)
 			self._clear_contents_cache()
@@ -1592,7 +1596,8 @@ class dblink(object):
 				# XXX: Decide how to handle failures here.
 				if retval != os.EX_OK:
 					failures += 1
-					writemsg(_("!!! FAILED postrm: %s\n") % retval, noiselevel=-1)
+					showMessage(_("!!! FAILED postrm: %s\n") % retval,
+						level=logging.ERROR, noiselevel=-1)
 
 			# Skip this if another package in the same slot has just been
 			# merged on top of this package, since the other package has

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     fbf86c5d46eafa07a39a18b87cf185486875d294
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Mar  1 21:04:41 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Mar  1 21:04:41 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fbf86c5d

owners_cache: fix EPREFIX miscalculation

This reverts commit 43e0b3ea4289aab9b4258d29d06afac40abd6c01 and
renames the root_len variable to eroot_len (for clarity).

---
 pym/portage/dbapi/vartree.py |    8 +++-----
 1 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 845a5c3..af3b5d9 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -815,11 +815,9 @@ class vardbapi(dbapi):
 
 		def __init__(self, vardb):
 			self._vardb = vardb
-			self._eprefix_split_len = len(x for x in \
-				vardb.settings["EPREFIX"].split(os.sep) if x)
 
 		def add(self, cpv):
-			root_len = len(self._vardb._eroot)
+			eroot_len = len(self._vardb._eroot)
 			contents = self._vardb._dblink(cpv).getcontents()
 			pkg_hash = self._hash_pkg(cpv)
 			if not contents:
@@ -831,12 +829,12 @@ class vardbapi(dbapi):
 			# explicitly listed in CONTENTS.
 			added_paths = set()
 			for x in contents:
-				x = x[root_len:]
+				x = x[eroot_len:]
 				added_paths.add(x)
 				self._add_path(x, pkg_hash)
 				x_split = x.split(os.sep)
 				x_split.pop()
-				while len(x_split) > self._eprefix_split_len:
+				while x_split:
 					parent = os.sep.join(x_split)
 					if parent in added_paths:
 						break

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     43e0b3ea4289aab9b4258d29d06afac40abd6c01
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Mar  1 20:42:56 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Mar  1 20:42:56 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=43e0b3ea

owners_cache: add parent dirs deeper than EPREFIX

---
 pym/portage/dbapi/vartree.py |    4 +++-
 1 files changed, 3 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 77d0256..845a5c3 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -815,6 +815,8 @@ class vardbapi(dbapi):
 
 		def __init__(self, vardb):
 			self._vardb = vardb
+			self._eprefix_split_len = len(x for x in \
+				vardb.settings["EPREFIX"].split(os.sep) if x)
 
 		def add(self, cpv):
 			root_len = len(self._vardb._eroot)
@@ -834,7 +836,7 @@ class vardbapi(dbapi):
 				self._add_path(x, pkg_hash)
 				x_split = x.split(os.sep)
 				x_split.pop()
-				while x_split:
+				while len(x_split) > self._eprefix_split_len:
 					parent = os.sep.join(x_split)
 					if parent in added_paths:
 						break

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4778b4b46260a7f585e30c712a1002361302f4db
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Mar  1 20:07:53 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Mar  1 20:07:53 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4778b4b4

unmerge: implicitly unmerge parent directories

We can't necessarily assume that they are explicitly listed in
CONTENTS.

---
 pym/portage/dbapi/vartree.py |   19 ++++++++++++++++---
 1 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 2e5d873..77d0256 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1785,7 +1785,7 @@ class dblink(object):
 			mykeys.reverse()
 
 			#process symlinks second-to-last, directories last.
-			mydirs = []
+			mydirs = set()
 			ignored_unlink_errnos = (
 				errno.EBUSY, errno.ENOENT,
 				errno.ENOTDIR, errno.EISDIR)
@@ -1847,6 +1847,7 @@ class dblink(object):
 
 			real_root = self.settings['ROOT']
 			real_root_len = len(real_root) - 1
+			eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1
 
 			for i, objkey in enumerate(mykeys):
 
@@ -1872,6 +1873,18 @@ class dblink(object):
 							os = portage.os
 							perf_md5 = portage.checksum.perform_md5
 
+				# Try to unmerge parent directories of everything
+				# listed in CONTENTS, since we can't necessarily
+				# assume that directories are listed in CONTENTS.
+				obj_split = obj.split(os.sep)
+				obj_split.pop()
+				while len(obj_split) > eroot_split_len:
+					parent = os.sep.join(obj_split)
+					if parent in mydirs:
+						break
+					mydirs.add(parent)
+					obj_split.pop()
+
 				file_data = pkgfiles[objkey]
 				file_type = file_data[0]
 				statobj = None
@@ -1940,7 +1953,7 @@ class dblink(object):
 					if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode):
 						show_unmerge("---", unmerge_desc["!dir"], file_type, obj)
 						continue
-					mydirs.append(obj)
+					mydirs.add(obj)
 				elif pkgfiles[objkey][0] == "sym":
 					if not islink:
 						show_unmerge("---", unmerge_desc["!sym"], file_type, obj)
@@ -1992,7 +2005,7 @@ class dblink(object):
 				elif pkgfiles[objkey][0] == "dev":
 					show_unmerge("---", "", file_type, obj)
 
-			mydirs.sort()
+			mydirs = sorted(mydirs)
 			mydirs.reverse()
 
 			for obj in mydirs:

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     4b4699f36bd4809ce4e8765a39b0949ec4864c2f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Mar  1 18:06:01 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Mar  1 18:06:01 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4b4699f3

owners_cache: implicitly add parent directories

We can't necessarily assume that they are explicitly listed in
CONTENTS.

---
 pym/portage/dbapi/vartree.py |   19 ++++++++++++++++++-
 1 files changed, 18 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 2e97fdb..2e5d873 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -823,8 +823,25 @@ class vardbapi(dbapi):
 			if not contents:
 				# Empty path is a code used to represent empty contents.
 				self._add_path("", pkg_hash)
+
+			# When adding paths, implicitly add parent directories,
+			# since we can't necessarily assume that they are
+			# explicitly listed in CONTENTS.
+			added_paths = set()
 			for x in contents:
-				self._add_path(x[root_len:], pkg_hash)
+				x = x[root_len:]
+				added_paths.add(x)
+				self._add_path(x, pkg_hash)
+				x_split = x.split(os.sep)
+				x_split.pop()
+				while x_split:
+					parent = os.sep.join(x_split)
+					if parent in added_paths:
+						break
+					added_paths.add(parent)
+					self._add_path(parent, pkg_hash)
+					x_split.pop()
+
 			self._vardb._aux_cache["modified"].add(cpv)
 
 		def _add_path(self, path, pkg_hash):

[gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/

[Zac Medico]

commit:     334af57e0511842a3b38db091f2aa33e07615729
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 14 03:59:24 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb 14 03:59:24 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=334af57e

Fix unhandled FileNotFound from previous commit.

There is an existing handler for EnvironmentError, so raise that
instead of FileNotFound.

---
 pym/portage/dbapi/bintree.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 78eb429..16b79db 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -807,7 +807,7 @@ class binarytree(object):
 						success = portage.getbinpkg.file_get(url,
 						     tmp_dirname, fcmd=fcmd, filename=tmp_basename)
 						if not success:
-							raise portage.exception.FileNotFound(url)
+							raise EnvironmentError("%s failed" % (setting,))
 						f = open(tmp_filename, 'rb')
 
 				f_dec = codecs.iterdecode(f,