From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1QJerS-0004qe-VB for garchives@archives.gentoo.org; Tue, 10 May 2011 04:47:55 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id C8A401C009; Tue, 10 May 2011 04:47:45 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 88E881C009 for ; Tue, 10 May 2011 04:47:45 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-CAMELLIA256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 0870C1B4017 for ; Tue, 10 May 2011 04:47:45 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id D6EAA80505 for ; Tue, 10 May 2011 04:47:42 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <928bc527a8c3eab3784df54226df2f2d83c6c85c.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/ X-VCS-Repository: proj/portage X-VCS-Files: pym/portage/dbapi/_MergeProcess.py pym/portage/dbapi/vartree.py X-VCS-Directories: pym/portage/dbapi/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: 928bc527a8c3eab3784df54226df2f2d83c6c85c Date: Tue, 10 May 2011 04:47:42 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: fa2752719495c2376c7df0daa67f5d8c commit: 928bc527a8c3eab3784df54226df2f2d83c6c85c Author: David James chromium org> AuthorDate: Tue May 10 04:11:47 2011 +0000 Commit: Zac Medico gentoo org> CommitDate: Tue May 10 04:47:42 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/portage.git;a= =3Dcommit;h=3D928bc527 Cache counter values, avoiding I/O when it doesn't change. This improves merge times by up to 25%, since looping over the vardb for each package install is slow. TEST=3DEmerge a bunch of packages, notice 25% speed improvement. BUG=3Dchromium-os:15112 Change-Id: I51dd617219cd1820ceeb702291bd790990995be4 --- pym/portage/dbapi/_MergeProcess.py | 3 +- pym/portage/dbapi/vartree.py | 71 +++++++++++++++++++-----------= ------ 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_Merg= eProcess.py index 12a0baf..05f45d5 100644 --- a/pym/portage/dbapi/_MergeProcess.py +++ b/pym/portage/dbapi/_MergeProcess.py @@ -143,6 +143,7 @@ class MergeProcess(SpawnProcess): fd_pipes[elog_writer_fd] =3D elog_writer_fd self._elog_reg_id =3D self.scheduler.register(elog_reader_fd, self._registered_events, self._elog_output_handler) + counter =3D self.vartree.dbapi.counter_tick() =20 pid =3D os.fork() if pid !=3D 0: @@ -196,7 +197,7 @@ class MergeProcess(SpawnProcess): try: rval =3D mylink.merge(self.pkgloc, self.infloc, myebuild=3Dself.myebuild, mydbapi=3Dself.mydbapi, - prev_mtimes=3Dself.prev_mtimes) + prev_mtimes=3Dself.prev_mtimes, counter=3Dcounter) except SystemExit: raise except: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index c81e99b..7c66027 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -166,6 +166,8 @@ class vardbapi(dbapi): self._linkmap =3D LinkageMap(self) self._owners =3D self._owners_db(self) =20 + self._cached_counter =3D None + def getpath(self, mykey, filename=3DNone): # This is an optimized hotspot, so don't use unicode-wrapped # os module and don't use os.path.join(). @@ -724,17 +726,6 @@ class vardbapi(dbapi): @param myroot: ignored, self._eroot is used instead """ myroot =3D None - cp_list =3D self.cp_list - max_counter =3D 0 - for cp in self.cp_all(): - for cpv in cp_list(cp): - try: - counter =3D int(self.aux_get(cpv, ["COUNTER"])[0]) - except (KeyError, OverflowError, ValueError): - continue - if counter > max_counter: - max_counter =3D counter - new_vdb =3D False counter =3D -1 try: @@ -762,16 +753,27 @@ class vardbapi(dbapi): writemsg("!!! %s\n" % str(e), noiselevel=3D-1) del e =20 - # We must ensure that we return a counter - # value that is at least as large as the - # highest one from the installed packages, - # since having a corrupt value that is too low - # can trigger incorrect AUTOCLEAN behavior due - # to newly installed packages having lower - # COUNTERs than the previous version in the - # same slot. - if counter > max_counter: + if self._cached_counter =3D=3D counter: max_counter =3D counter + else: + # We must ensure that we return a counter + # value that is at least as large as the + # highest one from the installed packages, + # since having a corrupt value that is too low + # can trigger incorrect AUTOCLEAN behavior due + # to newly installed packages having lower + # COUNTERs than the previous version in the + # same slot. + cp_list =3D self.cp_list + max_counter =3D counter + for cp in self.cp_all(): + for cpv in cp_list(cp): + try: + pkg_counter =3D int(self.aux_get(cpv, ["COUNTER"])[0]) + except (KeyError, OverflowError, ValueError): + continue + if pkg_counter > max_counter: + max_counter =3D pkg_counter =20 if counter < 0 and not new_vdb: writemsg(_("!!! Initializing COUNTER to " \ @@ -789,18 +791,19 @@ class vardbapi(dbapi): """ myroot =3D None mycpv =3D None - self.lock() try: counter =3D self.get_counter_tick_core() - 1 - if incrementing: - #increment counter - counter +=3D 1 - # use same permissions as config._init_dirs() - ensure_dirs(os.path.dirname(self._counter_path), - gid=3Dportage_gid, mode=3D0o2750, mask=3D0o2) - # update new global counter file - write_atomic(self._counter_path, str(counter)) + if self._cached_counter !=3D counter: + if incrementing: + #increment counter + counter +=3D 1 + # use same permissions as config._init_dirs() + ensure_dirs(os.path.dirname(self._counter_path), + gid=3Dportage_gid, mode=3D0o2750, mask=3D0o2) + # update new global counter file + write_atomic(self._counter_path, str(counter)) + self._cached_counter =3D counter finally: self.unlock() =20 @@ -2860,7 +2863,7 @@ class dblink(object): os.write(self._pipe, _unicode_encode(''.join(str_buffer))) =20 def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=3D0, - mydbapi=3DNone, prev_mtimes=3DNone): + mydbapi=3DNone, prev_mtimes=3DNone, counter=3DNone): """ =09 This function does the following: @@ -3276,7 +3279,8 @@ class dblink(object): self.copyfile(inforoot+"/"+x) =20 # write local package counter for recording - counter =3D self.vartree.dbapi.counter_tick(mycpv=3Dself.mycpv) + if counter is not None: + counter =3D self.vartree.dbapi.counter_tick(mycpv=3Dself.mycpv) codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), encoding=3D_encodings['fs'], errors=3D'strict'), 'w', encoding=3D_encodings['repo.content'], errors=3D'backslashreplac= e' @@ -3906,7 +3910,7 @@ class dblink(object): showMessage(zing + " " + mydest + "\n") =20 def merge(self, mergeroot, inforoot, myroot=3DNone, myebuild=3DNone, cl= eanup=3D0, - mydbapi=3DNone, prev_mtimes=3DNone): + mydbapi=3DNone, prev_mtimes=3DNone, counter=3DNone): """ @param myroot: ignored, self._eroot is used instead """ @@ -3918,7 +3922,8 @@ class dblink(object): self.vartree.dbapi._bump_mtime(self.mycpv) try: retval =3D self.treewalk(mergeroot, myroot, inforoot, myebuild, - cleanup=3Dcleanup, mydbapi=3Dmydbapi, prev_mtimes=3Dprev_mtimes) + cleanup=3Dcleanup, mydbapi=3Dmydbapi, prev_mtimes=3Dprev_mtimes, + counter=3Dcounter) =20 # If PORTAGE_BUILDDIR doesn't exist, then it probably means # fail-clean is enabled, and the success/die hooks have