From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id E789E1381F3 for ; Thu, 22 Aug 2013 09:01:43 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 9666AE0BB6; Thu, 22 Aug 2013 09:01:43 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 21EACE0BB6 for ; Thu, 22 Aug 2013 09:01:42 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 297F733EAE6 for ; Thu, 22 Aug 2013 09:01:42 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 8EBAEE5465 for ; Thu, 22 Aug 2013 09:01:39 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1377162014.fe1d5ddb27f4937e6dfc7a42334d84af0f291de6.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/distroot/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/overlay/pkgdir/distroot/distroot.py X-VCS-Directories: roverlay/overlay/pkgdir/distroot/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: fe1d5ddb27f4937e6dfc7a42334d84af0f291de6 X-VCS-Branch: master Date: Thu, 22 Aug 2013 09:01:39 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 6babb6df-18cd-488b-a002-88d752cc4cb1 X-Archives-Hash: 1ff655287d4e7f655868a8ac42b790e9 commit: fe1d5ddb27f4937e6dfc7a42334d84af0f291de6 Author: André Erdmann mailerd de> AuthorDate: Thu Aug 22 09:00:14 2013 +0000 Commit: André Erdmann mailerd de> CommitDate: Thu Aug 22 09:00:14 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=fe1d5ddb distroot: calculate distmap digests in a hashpool --- roverlay/overlay/pkgdir/distroot/distroot.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/roverlay/overlay/pkgdir/distroot/distroot.py b/roverlay/overlay/pkgdir/distroot/distroot.py index 609dea3..650ea15 100644 --- a/roverlay/overlay/pkgdir/distroot/distroot.py +++ b/roverlay/overlay/pkgdir/distroot/distroot.py @@ -18,11 +18,14 @@ import tempfile import roverlay.db.distmap import roverlay.overlay.pkgdir.distroot.distdir +import roverlay.util.hashpool class DistrootBase ( object ): """Base class for distroots.""" + HASHPOOL_JOB_COUNT = 2 + def __repr__ ( self ): return "{name}".format ( name = self.__class__.__name__, @@ -295,10 +298,20 @@ class DistrootBase ( object ): if self.distmap is not None: root = self.get_root() distfiles = set() - checkfile = self.distmap.check_integrity + distmap_hashtype = self.distmap.get_hash_type() + checkfile = self.distmap.check_digest_integrity + + self.logger.info ( "calculating file hashes" ) + + hash_pool = roverlay.util.hashpool.HashPool ( + ( distmap_hashtype, ), self.HASHPOOL_JOB_COUNT, use_threads=True + ) for abspath, relpath in self.iter_distfiles ( False ): - status = checkfile ( relpath, abspath ) + hash_pool.add ( relpath, abspath, None ) + + for relpath, hashdict in hash_pool.run_as_completed(): + status = checkfile ( relpath, hashdict[distmap_hashtype] ) if status == 0: self.logger.debug ( @@ -310,7 +323,7 @@ class DistrootBase ( object ): self.logger.info ( "file not in distmap, creating dummy entry: {!r}".format ( relpath ) ) - self.distmap.add_dummy_entry ( relpath, abspath ) + self.distmap.add_dummy_entry ( relpath, hashdict=hashdict ) distfiles.add ( relpath ) elif status == 2: # file in distmap, but not valid - remove it from distmap