From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1QiEJg-0005mS-Be for garchives@archives.gentoo.org; Sat, 16 Jul 2011 23:30:36 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 222C821C0D3; Sat, 16 Jul 2011 23:30:23 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id CFCA621C0D3 for ; Sat, 16 Jul 2011 23:30:22 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 2AF521B4004 for ; Sat, 16 Jul 2011 23:30:22 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id 6957F8003D for ; Sat, 16 Jul 2011 23:30:21 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <785fe0b42a66b1d137955a4b816b3c254a1b785d.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: pym/_emerge/ X-VCS-Repository: proj/portage X-VCS-Files: pym/_emerge/EbuildBuild.py pym/_emerge/EbuildFetcher.py X-VCS-Directories: pym/_emerge/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: 785fe0b42a66b1d137955a4b816b3c254a1b785d Date: Sat, 16 Jul 2011 23:30:21 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: 1562b32def5a2c2a5a6436c790b79c60 commit: 785fe0b42a66b1d137955a4b816b3c254a1b785d Author: Zac Medico gentoo org> AuthorDate: Sat Jul 16 23:30:14 2011 +0000 Commit: Zac Medico gentoo org> CommitDate: Sat Jul 16 23:30:14 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/portage.git;a= =3Dcommit;h=3D785fe0b4 EbuildBuild: skip the fetch queue when possible Since commit f07f8386e945b48358c11c121960e4833c539752, it was possible for EbuildBuild to wait on the fetch queue even in cases in which all required files had been previously fetched. Now this case is optimized to skip the fetch queue, as discribed in bug #375331, comment #2. --- pym/_emerge/EbuildBuild.py | 25 ++++++++- pym/_emerge/EbuildFetcher.py | 120 ++++++++++++++++++++++++++++++++++++= ----- 2 files changed, 129 insertions(+), 16 deletions(-) diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index b6beb49..1776d1e 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -166,12 +166,34 @@ class EbuildBuild(CompositeTask): portage.prepare_build_dirs(self.pkg.root, self.settings, 1) =20 fetcher =3D EbuildFetcher(config_pool=3Dself.config_pool, + ebuild_path=3Dself._ebuild_path, fetchall=3Dself.opts.fetch_all_uri, fetchonly=3Dself.opts.fetchonly, background=3Dself.background, logfile=3Dself.settings.get('PORTAGE_LOG_FILE'), pkg=3Dself.pkg, scheduler=3Dself.scheduler) =20 + try: + already_fetched =3D fetcher.already_fetched(self.settings) + except portage.exception.InvalidDependString as e: + msg_lines =3D [] + msg =3D "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ + (self.pkg.cpv, e) + msg_lines.append(msg) + fetcher._eerror(msg_lines) + portage.elog.elog_process(self.pkg.cpv, self.settings) + self.returncode =3D 1 + self._current_task =3D None + self._unlock_builddir() + self.wait() + return + + if already_fetched: + # This case is optimized to skip the fetch queue. + fetcher =3D None + self._fetch_exit(fetcher) + return + # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) @@ -180,7 +202,8 @@ class EbuildBuild(CompositeTask): =20 def _fetch_exit(self, fetcher): =20 - if self._default_exit(fetcher) !=3D os.EX_OK: + if fetcher is not None and \ + self._default_exit(fetcher) !=3D os.EX_OK: self._fetch_failed() return =20 diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py index c076288..0666787 100644 --- a/pym/_emerge/EbuildFetcher.py +++ b/pym/_emerge/EbuildFetcher.py @@ -14,24 +14,97 @@ from portage import _encodings from portage import _unicode_encode from portage import _unicode_decode from portage.elog.messages import eerror -from portage.package.ebuild.fetch import fetch +from portage.package.ebuild.fetch import _check_distfile, fetch from portage.util._pty import _create_pty_or_pipe =20 class EbuildFetcher(SpawnProcess): =20 - __slots__ =3D ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch= ") + \ + __slots__ =3D ("config_pool", "ebuild_path", "fetchonly", "fetchall", + "pkg", "prefetch") + \ ("_digests", "_settings", "_uri_map") =20 + def already_fetched(self, settings): + """ + Returns True if all files are already exist locally and have correct + digests, otherwise return False. When returning True, appropriate + digest checking messages are produced for display and/or logging. + When returning False, no messages are produced, since we assume + that a fetcher process will later be executed in order to produce + such messages. This will raise InvalidDependString if SRC_URI is + invalid. + """ + + uri_map =3D self._get_uri_map() + if not uri_map: + return True + + digests =3D self._get_digests() + distdir =3D settings["DISTDIR"] + allow_missing =3D "allow-missing-manifests" in settings.features + + for filename in uri_map: + # Use stat rather than lstat since fetch() creates + # symlinks when PORTAGE_RO_DISTDIRS is used. + try: + st =3D os.stat(os.path.join(distdir, filename)) + except OSError: + return False + if st.st_size =3D=3D 0: + return False + expected_size =3D digests.get(filename, {}).get('size') + if expected_size is None: + continue + if st.st_size !=3D expected_size: + return False + + stdout_orig =3D sys.stdout + stderr_orig =3D sys.stderr + global_havecolor =3D portage.output.havecolor + out =3D io.StringIO() + eout =3D portage.output.EOutput() + eout.quiet =3D settings.get("PORTAGE_QUIET") =3D=3D "1" + success =3D True + try: + sys.stdout =3D out + sys.stderr =3D out + if portage.output.havecolor: + portage.output.havecolor =3D not self.background + + for filename in uri_map: + mydigests =3D digests.get(filename) + if mydigests is None: + if not allow_missing: + success =3D False + break + continue + ok, st =3D _check_distfile(os.path.join(distdir, filename), + mydigests, eout, show_errors=3DFalse) + if not ok: + success =3D False + break + finally: + sys.stdout =3D stdout_orig + sys.stderr =3D stderr_orig + portage.output.havecolor =3D global_havecolor + + if success: + # When returning uncessessfully, no messages are produced, since + # we assume that a fetcher process will later be executed in order + # to produce such messages. + msg =3D out.getvalue() + if msg: + self.scheduler.output(msg, log_path=3Dself.logfile) + + return success + def _start(self): =20 root_config =3D self.pkg.root_config portdb =3D root_config.trees["porttree"].dbapi - ebuild_path =3D portdb.findname(self.pkg.cpv, myrepo=3Dself.pkg.repo) - if ebuild_path is None: - raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) + ebuild_path =3D self._get_ebuild_path() =20 try: - uri_map =3D self._get_uri_map(portdb, ebuild_path) + uri_map =3D self._get_uri_map() except portage.exception.InvalidDependString as e: msg_lines =3D [] msg =3D "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ @@ -48,9 +121,6 @@ class EbuildFetcher(SpawnProcess): self.wait() return =20 - self._digests =3D portage.Manifest( - os.path.dirname(ebuild_path), None).getTypeDigests("DIST") - settings =3D self.config_pool.allocate() settings.setcpv(self.pkg) portage.doebuild_environment(ebuild_path, 'fetch', @@ -75,7 +145,6 @@ class EbuildFetcher(SpawnProcess): settings["NOCOLOR"] =3D nocolor =20 self._settings =3D settings - self._uri_map =3D uri_map SpawnProcess._start(self) =20 # Free settings now since it's no longer needed in @@ -110,7 +179,7 @@ class EbuildFetcher(SpawnProcess): allow_missing =3D 'allow-missing-manifests' in self._settings.features try: if fetch(self._uri_map, self._settings, fetchonly=3Dself.fetchonly, - digests=3Dcopy.deepcopy(self._digests), + digests=3Dcopy.deepcopy(self._get_digests()), allow_missing_digests=3Dallow_missing): rval =3D os.EX_OK except SystemExit: @@ -122,16 +191,37 @@ class EbuildFetcher(SpawnProcess): # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval) =20 - def _get_uri_map(self, portdb, ebuild_path): + def _get_ebuild_path(self): + if self.ebuild_path is not None: + return self.ebuild_path + portdb =3D self.pkg.root_config.trees["porttree"].dbapi + self.ebuild_path =3D portdb.findname(self.pkg.cpv, myrepo=3Dself.pkg.r= epo) + if self.ebuild_path is None: + raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) + return self.ebuild_path + + def _get_digests(self): + if self._digests is not None: + return self._digests + self._digests =3D portage.Manifest(os.path.dirname( + self._get_ebuild_path()), None).getTypeDigests("DIST") + return self._digests + + def _get_uri_map(self): """ This can raise InvalidDependString from portdbapi.getFetchMap(). """ - pkgdir =3D os.path.dirname(ebuild_path) + if self._uri_map is not None: + return self._uri_map + pkgdir =3D os.path.dirname(self._get_ebuild_path()) mytree =3D os.path.dirname(os.path.dirname(pkgdir)) use =3D None if not self.fetchall: use =3D self.pkg.use.enabled - return portdb.getFetchMap(self.pkg.cpv, useflags=3Duse, mytree=3Dmytre= e) + portdb =3D self.pkg.root_config.trees["porttree"].dbapi + self._uri_map =3D portdb.getFetchMap(self.pkg.cpv, + useflags=3Duse, mytree=3Dmytree) + return self._uri_map =20 def _prefetch_size_ok(self, uri_map, settings, ebuild_path): distdir =3D settings["DISTDIR"] @@ -148,7 +238,7 @@ class EbuildFetcher(SpawnProcess): return False sizes[filename] =3D st.st_size =20 - digests =3D self._digests + digests =3D self._get_digests() for filename, actual_size in sizes.items(): size =3D digests.get(filename, {}).get('size') if size is None: