From mboxrd@z Thu Jan 1 00:00:00 1970
Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org)
by finch.gentoo.org with esmtp (Exim 4.60)
(envelope-from <gentoo-commits+bounces-344253-garchives=archives.gentoo.org@lists.gentoo.org>)
id 1QJerS-0004qe-VB
for garchives@archives.gentoo.org; Tue, 10 May 2011 04:47:55 +0000
Received: from pigeon.gentoo.org (localhost [127.0.0.1])
by pigeon.gentoo.org (Postfix) with SMTP id C8A401C009;
Tue, 10 May 2011 04:47:45 +0000 (UTC)
Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183])
by pigeon.gentoo.org (Postfix) with ESMTP id 88E881C009
for <gentoo-commits@lists.gentoo.org>; Tue, 10 May 2011 04:47:45 +0000 (UTC)
Received: from pelican.gentoo.org (unknown [66.219.59.40])
(using TLSv1 with cipher ADH-CAMELLIA256-SHA (256/256 bits))
(No client certificate requested)
by smtp.gentoo.org (Postfix) with ESMTPS id 0870C1B4017
for <gentoo-commits@lists.gentoo.org>; Tue, 10 May 2011 04:47:45 +0000 (UTC)
Received: from localhost.localdomain (localhost [127.0.0.1])
by pelican.gentoo.org (Postfix) with ESMTP id D6EAA80505
for <gentoo-commits@lists.gentoo.org>; Tue, 10 May 2011 04:47:42 +0000 (UTC)
From: "Zac Medico" <zmedico@gentoo.org>
To: gentoo-commits@lists.gentoo.org
Content-type: text/plain; charset=UTF-8
Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" <zmedico@gentoo.org>
Message-ID: <928bc527a8c3eab3784df54226df2f2d83c6c85c.zmedico@gentoo>
Subject: [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
X-VCS-Repository: proj/portage
X-VCS-Files: pym/portage/dbapi/_MergeProcess.py pym/portage/dbapi/vartree.py
X-VCS-Directories: pym/portage/dbapi/
X-VCS-Committer: zmedico
X-VCS-Committer-Name: Zac Medico
X-VCS-Revision: 928bc527a8c3eab3784df54226df2f2d83c6c85c
Date: Tue, 10 May 2011 04:47:42 +0000 (UTC)
Precedence: bulk
List-Post: <mailto:gentoo-commits@lists.gentoo.org>
List-Help: <mailto:gentoo-commits+help@lists.gentoo.org>
List-Unsubscribe: <mailto:gentoo-commits+unsubscribe@lists.gentoo.org>
List-Subscribe: <mailto:gentoo-commits+subscribe@lists.gentoo.org>
List-Id: Gentoo Linux mail <gentoo-commits.gentoo.org>
X-BeenThere: gentoo-commits@lists.gentoo.org
Content-Transfer-Encoding: quoted-printable
X-Archives-Salt:
X-Archives-Hash: fa2752719495c2376c7df0daa67f5d8c
commit: 928bc527a8c3eab3784df54226df2f2d83c6c85c
Author: David James <davidjames <AT> chromium <DOT> org>
AuthorDate: Tue May 10 04:11:47 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May 10 04:47:42 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/portage.git;a=
=3Dcommit;h=3D928bc527
Cache counter values, avoiding I/O when it doesn't change.
This improves merge times by up to 25%, since looping over the vardb for
each package install is slow.
TEST=3DEmerge a bunch of packages, notice 25% speed improvement.
BUG=3Dchromium-os:15112
Change-Id: I51dd617219cd1820ceeb702291bd790990995be4
---
pym/portage/dbapi/_MergeProcess.py | 3 +-
pym/portage/dbapi/vartree.py | 71 +++++++++++++++++++-----------=
------
2 files changed, 40 insertions(+), 34 deletions(-)
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_Merg=
eProcess.py
index 12a0baf..05f45d5 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -143,6 +143,7 @@ class MergeProcess(SpawnProcess):
fd_pipes[elog_writer_fd] =3D elog_writer_fd
self._elog_reg_id =3D self.scheduler.register(elog_reader_fd,
self._registered_events, self._elog_output_handler)
+ counter =3D self.vartree.dbapi.counter_tick()
=20
pid =3D os.fork()
if pid !=3D 0:
@@ -196,7 +197,7 @@ class MergeProcess(SpawnProcess):
try:
rval =3D mylink.merge(self.pkgloc, self.infloc,
myebuild=3Dself.myebuild, mydbapi=3Dself.mydbapi,
- prev_mtimes=3Dself.prev_mtimes)
+ prev_mtimes=3Dself.prev_mtimes, counter=3Dcounter)
except SystemExit:
raise
except:
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c81e99b..7c66027 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -166,6 +166,8 @@ class vardbapi(dbapi):
self._linkmap =3D LinkageMap(self)
self._owners =3D self._owners_db(self)
=20
+ self._cached_counter =3D None
+
def getpath(self, mykey, filename=3DNone):
# This is an optimized hotspot, so don't use unicode-wrapped
# os module and don't use os.path.join().
@@ -724,17 +726,6 @@ class vardbapi(dbapi):
@param myroot: ignored, self._eroot is used instead
"""
myroot =3D None
- cp_list =3D self.cp_list
- max_counter =3D 0
- for cp in self.cp_all():
- for cpv in cp_list(cp):
- try:
- counter =3D int(self.aux_get(cpv, ["COUNTER"])[0])
- except (KeyError, OverflowError, ValueError):
- continue
- if counter > max_counter:
- max_counter =3D counter
-
new_vdb =3D False
counter =3D -1
try:
@@ -762,16 +753,27 @@ class vardbapi(dbapi):
writemsg("!!! %s\n" % str(e), noiselevel=3D-1)
del e
=20
- # We must ensure that we return a counter
- # value that is at least as large as the
- # highest one from the installed packages,
- # since having a corrupt value that is too low
- # can trigger incorrect AUTOCLEAN behavior due
- # to newly installed packages having lower
- # COUNTERs than the previous version in the
- # same slot.
- if counter > max_counter:
+ if self._cached_counter =3D=3D counter:
max_counter =3D counter
+ else:
+ # We must ensure that we return a counter
+ # value that is at least as large as the
+ # highest one from the installed packages,
+ # since having a corrupt value that is too low
+ # can trigger incorrect AUTOCLEAN behavior due
+ # to newly installed packages having lower
+ # COUNTERs than the previous version in the
+ # same slot.
+ cp_list =3D self.cp_list
+ max_counter =3D counter
+ for cp in self.cp_all():
+ for cpv in cp_list(cp):
+ try:
+ pkg_counter =3D int(self.aux_get(cpv, ["COUNTER"])[0])
+ except (KeyError, OverflowError, ValueError):
+ continue
+ if pkg_counter > max_counter:
+ max_counter =3D pkg_counter
=20
if counter < 0 and not new_vdb:
writemsg(_("!!! Initializing COUNTER to " \
@@ -789,18 +791,19 @@ class vardbapi(dbapi):
"""
myroot =3D None
mycpv =3D None
-
self.lock()
try:
counter =3D self.get_counter_tick_core() - 1
- if incrementing:
- #increment counter
- counter +=3D 1
- # use same permissions as config._init_dirs()
- ensure_dirs(os.path.dirname(self._counter_path),
- gid=3Dportage_gid, mode=3D0o2750, mask=3D0o2)
- # update new global counter file
- write_atomic(self._counter_path, str(counter))
+ if self._cached_counter !=3D counter:
+ if incrementing:
+ #increment counter
+ counter +=3D 1
+ # use same permissions as config._init_dirs()
+ ensure_dirs(os.path.dirname(self._counter_path),
+ gid=3Dportage_gid, mode=3D0o2750, mask=3D0o2)
+ # update new global counter file
+ write_atomic(self._counter_path, str(counter))
+ self._cached_counter =3D counter
finally:
self.unlock()
=20
@@ -2860,7 +2863,7 @@ class dblink(object):
os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
=20
def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=3D0,
- mydbapi=3DNone, prev_mtimes=3DNone):
+ mydbapi=3DNone, prev_mtimes=3DNone, counter=3DNone):
"""
=09
This function does the following:
@@ -3276,7 +3279,8 @@ class dblink(object):
self.copyfile(inforoot+"/"+x)
=20
# write local package counter for recording
- counter =3D self.vartree.dbapi.counter_tick(mycpv=3Dself.mycpv)
+ if counter is not None:
+ counter =3D self.vartree.dbapi.counter_tick(mycpv=3Dself.mycpv)
codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
encoding=3D_encodings['fs'], errors=3D'strict'),
'w', encoding=3D_encodings['repo.content'], errors=3D'backslashreplac=
e'
@@ -3906,7 +3910,7 @@ class dblink(object):
showMessage(zing + " " + mydest + "\n")
=20
def merge(self, mergeroot, inforoot, myroot=3DNone, myebuild=3DNone, cl=
eanup=3D0,
- mydbapi=3DNone, prev_mtimes=3DNone):
+ mydbapi=3DNone, prev_mtimes=3DNone, counter=3DNone):
"""
@param myroot: ignored, self._eroot is used instead
"""
@@ -3918,7 +3922,8 @@ class dblink(object):
self.vartree.dbapi._bump_mtime(self.mycpv)
try:
retval =3D self.treewalk(mergeroot, myroot, inforoot, myebuild,
- cleanup=3Dcleanup, mydbapi=3Dmydbapi, prev_mtimes=3Dprev_mtimes)
+ cleanup=3Dcleanup, mydbapi=3Dmydbapi, prev_mtimes=3Dprev_mtimes,
+ counter=3Dcounter)
=20
# If PORTAGE_BUILDDIR doesn't exist, then it probably means
# fail-clean is enabled, and the success/die hooks have