From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 513BE13823F for ; Wed, 2 Jan 2013 04:48:29 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 3A4EA21C003; Wed, 2 Jan 2013 04:48:22 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id A4D1321C003 for ; Wed, 2 Jan 2013 04:48:21 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 8CCCF33D929 for ; Wed, 2 Jan 2013 04:48:20 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 03C98E5439 for ; Wed, 2 Jan 2013 04:48:19 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <1357102085.5681942aabf65daa31c5e591f0d1d99e0d2bc70f.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: pym/_emerge/ X-VCS-Repository: proj/portage X-VCS-Files: pym/_emerge/FakeVartree.py pym/_emerge/depgraph.py X-VCS-Directories: pym/_emerge/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: 5681942aabf65daa31c5e591f0d1d99e0d2bc70f X-VCS-Branch: master Date: Wed, 2 Jan 2013 04:48:19 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 98d0fe5f-424e-4f98-b7f4-62b50af373b1 X-Archives-Hash: 050a8dae59f8e327867e043b8777cffd commit: 5681942aabf65daa31c5e591f0d1d99e0d2bc70f Author: Zac Medico gentoo org> AuthorDate: Wed Jan 2 04:48:05 2013 +0000 Commit: Zac Medico gentoo org> CommitDate: Wed Jan 2 04:48:05 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5681942a depgraph: dynamic deps parallel metadata regen --- pym/_emerge/FakeVartree.py | 7 +++++ pym/_emerge/depgraph.py | 60 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py index 31ff65c..058c1c7 100644 --- a/pym/_emerge/FakeVartree.py +++ b/pym/_emerge/FakeVartree.py @@ -167,6 +167,13 @@ class FakeVartree(vartree): perform_global_updates( pkg.cpv, self.dbapi, self._global_updates) + def dynamic_deps_preload(self, pkg, metadata): + if metadata is not None: + metadata = dict((k, metadata.get(k, '')) + for k in self._portdb_keys) + self._apply_dynamic_deps(pkg, metadata) + self._aux_get_history.add(pkg.cpv) + def cpv_discard(self, pkg): """ Discard a package from the fake vardb if it exists. diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index fab87d9..47f1b60 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -41,6 +41,7 @@ from portage.util import ensure_dirs from portage.util import writemsg_level, write_atomic from portage.util.digraph import digraph from portage.util.listdir import _ignorecvs_dirs +from portage.util._async.TaskScheduler import TaskScheduler from portage.versions import catpkgsplit from _emerge.AtomArg import AtomArg @@ -54,6 +55,7 @@ from _emerge.DependencyArg import DependencyArg from _emerge.DepPriority import DepPriority from _emerge.DepPriorityNormalRange import DepPriorityNormalRange from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange +from _emerge.EbuildMetadataPhase import EbuildMetadataPhase from _emerge.FakeVartree import FakeVartree from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps from _emerge.is_valid_package_atom import insert_category_into_atom, \ @@ -551,17 +553,59 @@ class depgraph(object): fakedb = self._dynamic_config._graph_trees[ myroot]["vartree"].dbapi - for pkg in vardb: - self._spinner_update() - if dynamic_deps: - # This causes FakeVartree to update the - # Package instance dependencies via - # PackageVirtualDbapi.aux_update() - vardb.aux_get(pkg.cpv, []) - fakedb.cpv_inject(pkg) + if not dynamic_deps: + for pkg in vardb: + fakedb.cpv_inject(pkg) + else: + max_jobs = self._frozen_config.myopts.get("--jobs") + max_load = self._frozen_config.myopts.get("--load-average") + scheduler = TaskScheduler( + self._dynamic_deps_preload(fake_vartree, fakedb), + max_jobs=max_jobs, + max_load=max_load, + event_loop=fake_vartree._portdb._event_loop) + scheduler.start() + scheduler.wait() self._dynamic_config._vdb_loaded = True + def _dynamic_deps_preload(self, fake_vartree, fakedb): + portdb = fake_vartree._portdb + for pkg in fake_vartree.dbapi: + self._spinner_update() + fakedb.cpv_inject(pkg) + ebuild_path, repo_path = \ + portdb.findname2(pkg.cpv, myrepo=pkg.repo) + if ebuild_path is None: + fake_vartree.dynamic_deps_preload(pkg, None) + continue + metadata, ebuild_hash = portdb._pull_valid_cache( + pkg.cpv, ebuild_path, repo_path) + if metadata is not None: + fake_vartree.dynamic_deps_preload(pkg, metadata) + else: + proc = EbuildMetadataPhase(cpv=pkg.cpv, + ebuild_hash=ebuild_hash, + portdb=portdb, repo_path=repo_path, + settings=portdb.doebuild_settings) + proc.addExitListener( + self._dynamic_deps_proc_exit(pkg, fake_vartree)) + yield proc + + class _dynamic_deps_proc_exit(object): + + __slots__ = ('_pkg', '_fake_vartree') + + def __init__(self, pkg, fake_vartree): + self._pkg = pkg + self._fake_vartree = fake_vartree + + def __call__(self, proc): + metadata = None + if proc.returncode == os.EX_OK: + metadata = proc.metadata + self._fake_vartree.dynamic_deps_preload(self._pkg, metadata) + def _spinner_update(self): if self._frozen_config.spinner: self._frozen_config.spinner.update()