From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <gentoo-commits+bounces-753964-garchives=archives.gentoo.org@lists.gentoo.org>
Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80])
by finch.gentoo.org (Postfix) with ESMTP id 427F11389E2
for <garchives@archives.gentoo.org>; Sun, 7 Dec 2014 23:14:08 +0000 (UTC)
Received: from pigeon.gentoo.org (localhost [127.0.0.1])
by pigeon.gentoo.org (Postfix) with SMTP id 387FDE0C69;
Sun, 7 Dec 2014 23:14:06 +0000 (UTC)
Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183])
(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))
(No client certificate requested)
by pigeon.gentoo.org (Postfix) with ESMTPS id 523DAE0C4A
for <gentoo-commits@lists.gentoo.org>; Sun, 7 Dec 2014 23:14:05 +0000 (UTC)
Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52])
(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))
(No client certificate requested)
by smtp.gentoo.org (Postfix) with ESMTPS id 5DFC6340542
for <gentoo-commits@lists.gentoo.org>; Sun, 7 Dec 2014 23:14:04 +0000 (UTC)
Received: from localhost.localdomain (localhost [127.0.0.1])
by oystercatcher.gentoo.org (Postfix) with ESMTP id 8919EBD57
for <gentoo-commits@lists.gentoo.org>; Sun, 7 Dec 2014 23:14:02 +0000 (UTC)
From: "Zac Medico" <zmedico@gentoo.org>
To: gentoo-commits@lists.gentoo.org
Content-Transfer-Encoding: 8bit
Content-type: text/plain; charset=UTF-8
Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" <zmedico@gentoo.org>
Message-ID: <1417993848.5424b91133b3b155b0e6ddc08fb46ba301d971f8.zmedico@gentoo>
Subject: [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
X-VCS-Repository: proj/portage
X-VCS-Files: pym/portage/dbapi/IndexedVardb.py pym/portage/dbapi/vartree.py
X-VCS-Directories: pym/portage/dbapi/
X-VCS-Committer: zmedico
X-VCS-Committer-Name: Zac Medico
X-VCS-Revision: 5424b91133b3b155b0e6ddc08fb46ba301d971f8
X-VCS-Branch: master
Date: Sun, 7 Dec 2014 23:14:02 +0000 (UTC)
Precedence: bulk
List-Post: <mailto:gentoo-commits@lists.gentoo.org>
List-Help: <mailto:gentoo-commits+help@lists.gentoo.org>
List-Unsubscribe: <mailto:gentoo-commits+unsubscribe@lists.gentoo.org>
List-Subscribe: <mailto:gentoo-commits+subscribe@lists.gentoo.org>
List-Id: Gentoo Linux mail <gentoo-commits.gentoo.org>
X-BeenThere: gentoo-commits@lists.gentoo.org
X-Archives-Salt: c219c45e-2fea-48ae-9b13-611298d8aaed
X-Archives-Hash: e1a6fb95667ad301f56f97fee8f31c59
commit: 5424b91133b3b155b0e6ddc08fb46ba301d971f8
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Nov 1 15:06:01 2014 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec 7 23:10:48 2014 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5424b911
Add IndexedVardb class.
Searching of installed packages is optimized to take advantage of
vardbdbapi._aux_cache, which is backed by vdb_metadata.pickle.
This class only implements a subset of vardbapi functionality that is
useful for searching incrementally. For this reason, the cp_all method
returns an ordered iterator instead of a list, so that search results
can be displayed incrementally.
X-Gentoo-Bug: 525718
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=525718
---
pym/portage/dbapi/IndexedVardb.py | 94 +++++++++++++++++++++++++++++++++++++++
pym/portage/dbapi/vartree.py | 23 +++++++---
2 files changed, 110 insertions(+), 7 deletions(-)
diff --git a/pym/portage/dbapi/IndexedVardb.py b/pym/portage/dbapi/IndexedVardb.py
new file mode 100644
index 0000000..424defc
--- /dev/null
+++ b/pym/portage/dbapi/IndexedVardb.py
@@ -0,0 +1,94 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import portage
+from portage.dep import Atom
+from portage.versions import _pkg_str
+
+class IndexedVardb(object):
+ """
+ A vardbapi interface that sacrifices validation in order to
+ improve performance. It takes advantage of vardbdbapi._aux_cache,
+ which is backed by vdb_metadata.pickle. Since _aux_cache is
+ not updated for every single merge/unmerge (see
+ _aux_cache_threshold), the list of packages is obtained directly
+ from the real vardbapi instance. If a package is missing from
+ _aux_cache, then its metadata is obtained using the normal
+ (validated) vardbapi.aux_get method.
+
+ For performance reasons, the match method only supports package
+ name and version constraints.
+ """
+
+ # Match returns unordered results.
+ match_unordered = True
+
+ _copy_attrs = ('cpv_exists',
+ '_aux_cache_keys', '_cpv_sort_ascending')
+
+ def __init__(self, vardb):
+ self._vardb = vardb
+
+ for k in self._copy_attrs:
+ setattr(self, k, getattr(vardb, k))
+
+ self._cp_map = None
+
+ def cp_all(self):
+ """
+ Returns an ordered iterator instead of a list, so that search
+ results can be displayed incrementally.
+ """
+ if self._cp_map is not None:
+ return iter(sorted(self._cp_map))
+
+ return self._iter_cp_all()
+
+ def _iter_cp_all(self):
+ self._cp_map = cp_map = {}
+ previous_cp = None
+ for cpv in self._vardb._iter_cpv_all(sort = True):
+ cp = portage.cpv_getkey(cpv)
+ if cp is not None:
+ cp_list = cp_map.get(cp)
+ if cp_list is None:
+ cp_list = []
+ cp_map[cp] = cp_list
+ cp_list.append(cpv)
+ if previous_cp is not None and \
+ previous_cp != cp:
+ yield previous_cp
+ previous_cp = cp
+
+ if previous_cp is not None:
+ yield previous_cp
+
+ def match(self, atom):
+ """
+ For performance reasons, only package name and version
+ constraints are supported, and the returned list is
+ unordered.
+ """
+ if not isinstance(atom, Atom):
+ atom = Atom(atom)
+ cp_list = self._cp_map.get(atom.cp)
+ if cp_list is None:
+ return []
+
+ if atom == atom.cp:
+ return cp_list[:]
+ else:
+ return portage.match_from_list(atom, cp_list)
+
+ def aux_get(self, cpv, attrs, myrepo=None):
+ pkg_data = self._vardb._aux_cache["packages"].get(cpv)
+ if not isinstance(pkg_data, tuple) or \
+ len(pkg_data) != 2 or \
+ not isinstance(pkg_data[1], dict):
+ pkg_data = None
+ if pkg_data is None:
+ # It may be missing from _aux_cache due to
+ # _aux_cache_threshold.
+ return self._vardb.aux_get(cpv, attrs)
+ metadata = pkg_data[1]
+ return [metadata.get(k, "") for k in attrs]
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 4840492..9c8b276 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -435,6 +435,9 @@ class vardbapi(dbapi):
(generally this is only necessary in critical sections that
involve merge or unmerge of packages).
"""
+ return list(self._iter_cpv_all(use_cache=use_cache))
+
+ def _iter_cpv_all(self, use_cache=True, sort=False):
returnme = []
basepath = os.path.join(self._eroot, VDB_PATH) + os.path.sep
@@ -451,26 +454,32 @@ class vardbapi(dbapi):
del e
return []
- for x in listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1):
+ catdirs = listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1)
+ if sort:
+ catdirs.sort()
+
+ for x in catdirs:
if self._excluded_dirs.match(x) is not None:
continue
if not self._category_re.match(x):
continue
- for y in listdir(basepath + x, EmptyOnError=1, dirsonly=1):
+
+ pkgdirs = listdir(basepath + x, EmptyOnError=1, dirsonly=1)
+ if sort:
+ pkgdirs.sort()
+
+ for y in pkgdirs:
if self._excluded_dirs.match(y) is not None:
continue
subpath = x + "/" + y
# -MERGING- should never be a cpv, nor should files.
try:
- if catpkgsplit(subpath) is None:
- self.invalidentry(self.getpath(subpath))
- continue
+ subpath = _pkg_str(subpath)
except InvalidData:
self.invalidentry(self.getpath(subpath))
continue
- returnme.append(subpath)
- return returnme
+ yield subpath
def cp_all(self, use_cache=1):
mylist = self.cpv_all(use_cache=use_cache)