From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1SP0YH-0005hK-GR for garchives@archives.gentoo.org; Tue, 01 May 2012 00:02:45 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id E359DE0AB7; Tue, 1 May 2012 00:02:36 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 9322EE0AB7 for ; Tue, 1 May 2012 00:02:36 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id D30581B4094 for ; Tue, 1 May 2012 00:02:35 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id BA2DFE5403 for ; Tue, 1 May 2012 00:02:33 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1335830528.5fd52e5cf2f85ec872780338fcd2ad165f3123d9.zorry@gentoo> Subject: [gentoo-commits] dev/zorry:master commit in: gobs/pym/ X-VCS-Repository: dev/zorry X-VCS-Files: gobs/pym/Scheduler.py X-VCS-Directories: gobs/pym/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 5fd52e5cf2f85ec872780338fcd2ad165f3123d9 X-VCS-Branch: master Date: Tue, 1 May 2012 00:02:33 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: d1bfebe6-fd36-4474-a487-4da47de75d19 X-Archives-Hash: 5e3f207a800cdba82cebd208491e0fec commit: 5fd52e5cf2f85ec872780338fcd2ad165f3123d9 Author: Magnus Granberg gentoo org> AuthorDate: Tue May 1 00:02:08 2012 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Tue May 1 00:02:08 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Ddev/zorry.git;a=3D= commit;h=3D5fd52e5c Updated Scheduler.py --- gobs/pym/Scheduler.py | 381 +++++++++++++++++++------------------------= ------ 1 files changed, 146 insertions(+), 235 deletions(-) diff --git a/gobs/pym/Scheduler.py b/gobs/pym/Scheduler.py index 005f861..229c595 100644 --- a/gobs/pym/Scheduler.py +++ b/gobs/pym/Scheduler.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 =20 from __future__ import print_function @@ -7,10 +7,8 @@ from collections import deque import gc import gzip import logging -import shutil import signal import sys -import tempfile import textwrap import time import warnings @@ -28,9 +26,12 @@ from portage.output import colorize, create_color_func= , red bad =3D create_color_func("BAD") from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet -from portage.util import writemsg, writemsg_level +from portage.util import ensure_dirs, writemsg, writemsg_level +from portage.util.SlotObject import SlotObject from portage.package.ebuild.digestcheck import digestcheck from portage.package.ebuild.digestgen import digestgen +from portage.package.ebuild.doebuild import (_check_temp_dir, + _prepare_self_update) from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs =20 import _emerge @@ -44,6 +45,7 @@ from _emerge.create_depgraph_params import create_depgr= aph_params from _emerge.create_world_atom import create_world_atom from _emerge.DepPriority import DepPriority from _emerge.depgraph import depgraph, resume_depgraph +from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildFetcher import EbuildFetcher from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog @@ -52,12 +54,9 @@ from _emerge._find_deep_system_runtime_deps import _fi= nd_deep_system_runtime_dep from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo from _emerge.JobStatusDisplay import JobStatusDisplay from _emerge.MergeListItem import MergeListItem -from _emerge.MiscFunctionsProcess import MiscFunctionsProcess from _emerge.Package import Package from _emerge.PackageMerge import PackageMerge from _emerge.PollScheduler import PollScheduler -from _emerge.RootConfig import RootConfig -from _emerge.SlotObject import SlotObject from _emerge.SequentialTaskQueue import SequentialTaskQueue =20 from gobs.build_log import gobs_buildlog @@ -79,17 +78,12 @@ class Scheduler(PollScheduler): frozenset(["--pretend", "--fetchonly", "--fetch-all-uri"]) =20 - _opts_no_restart =3D frozenset(["--buildpkgonly", + _opts_no_self_update =3D frozenset(["--buildpkgonly", "--fetchonly", "--fetch-all-uri", "--pretend"]) =20 - _bad_resume_opts =3D set(["--ask", "--changelog", - "--resume", "--skipfirst"]) - - class _iface_class(SlotObject): + class _iface_class(PollScheduler._sched_iface_class): __slots__ =3D ("fetch", - "output", "register", "schedule", - "scheduleSetup", "scheduleUnpack", "scheduleYield", - "unregister") + "scheduleSetup", "scheduleUnpack") =20 class _fetch_iface_class(SlotObject): __slots__ =3D ("log_file", "schedule") @@ -153,7 +147,7 @@ class Scheduler(PollScheduler): DeprecationWarning, stacklevel=3D2) =20 self.settings =3D settings - self.target_root =3D settings["ROOT"] + self.target_root =3D settings["EROOT"] self.trees =3D trees self.myopts =3D myopts self._spinner =3D spinner @@ -163,7 +157,7 @@ class Scheduler(PollScheduler): self._build_opts =3D self._build_opts_class() =20 for k in self._build_opts.__slots__: - setattr(self._build_opts, k, "--" + k.replace("_", "-") in myopts) + setattr(self._build_opts, k, myopts.get("--" + k.replace("_", "-"))) self._build_opts.buildpkg_exclude =3D InternalPackageSet( \ initial_atoms=3D" ".join(myopts.get("--buildpkg-exclude", [])).split(= ), \ allow_wildcard=3DTrue, allow_repo=3DTrue) @@ -209,10 +203,7 @@ class Scheduler(PollScheduler): if max_jobs is None: max_jobs =3D 1 self._set_max_jobs(max_jobs) - - # The root where the currently running - # portage instance is installed. - self._running_root =3D trees["/"]["root_config"] + self._running_root =3D trees[trees._running_eroot]["root_config"] self.edebug =3D 0 if settings.get("PORTAGE_DEBUG", "") =3D=3D "1": self.edebug =3D 1 @@ -226,13 +217,11 @@ class Scheduler(PollScheduler): fetch_iface =3D self._fetch_iface_class(log_file=3Dself._fetch_log, schedule=3Dself._schedule_fetch) self._sched_iface =3D self._iface_class( - fetch=3Dfetch_iface, output=3Dself._task_output, - register=3Dself._register, - schedule=3Dself._schedule_wait, + fetch=3Dfetch_iface, scheduleSetup=3Dself._schedule_setup, scheduleUnpack=3Dself._schedule_unpack, - scheduleYield=3Dself._schedule_yield, - unregister=3Dself._unregister) + **dict((k, getattr(self.sched_iface, k)) + for k in self.sched_iface.__slots__)) =20 self._prefetchers =3D weakref.WeakValueDictionary() self._pkg_queue =3D [] @@ -296,10 +285,37 @@ class Scheduler(PollScheduler): self._running_portage =3D self._pkg(cpv, "installed", self._running_root, installed=3DTrue) =20 + def _handle_self_update(self): + + if self._opts_no_self_update.intersection(self.myopts): + return os.EX_OK + + for x in self._mergelist: + if not isinstance(x, Package): + continue + if x.operation !=3D "merge": + continue + if x.root !=3D self._running_root.root: + continue + if not portage.dep.match_from_list( + portage.const.PORTAGE_PACKAGE_ATOM, [x]): + continue + if self._running_portage is None or \ + self._running_portage.cpv !=3D x.cpv or \ + '9999' in x.cpv or \ + 'git' in x.inherited or \ + 'git-2' in x.inherited: + rval =3D _check_temp_dir(self.settings) + if rval !=3D os.EX_OK: + return rval + _prepare_self_update(self.settings) + break + + return os.EX_OK + def _terminate_tasks(self): self._status_display.quiet =3D True - while self._running_tasks: - task_id, task =3D self._running_tasks.popitem() + for task in list(self._running_tasks.values()): task.cancel() for q in self._task_queues.values(): q.clear() @@ -311,10 +327,11 @@ class Scheduler(PollScheduler): """ self._set_graph_config(graph_config) self._blocker_db =3D {} + dynamic_deps =3D self.myopts.get("--dynamic-deps", "y") !=3D "n" for root in self.trees: if graph_config is None: fake_vartree =3D FakeVartree(self.trees[root]["root_config"], - pkg_cache=3Dself._pkg_cache) + pkg_cache=3Dself._pkg_cache, dynamic_deps=3Ddynamic_deps) fake_vartree.sync() else: fake_vartree =3D graph_config.trees[root]['vartree'] @@ -331,52 +348,6 @@ class Scheduler(PollScheduler): self._set_graph_config(None) gc.collect() =20 - def _poll(self, timeout=3DNone): - - self._schedule() - - if timeout is None: - while True: - if not self._poll_event_handlers: - self._schedule() - if not self._poll_event_handlers: - raise StopIteration( - "timeout is None and there are no poll() event handlers") - previous_count =3D len(self._poll_event_queue) - PollScheduler._poll(self, timeout=3Dself._max_display_latency) - self._status_display.display() - if previous_count !=3D len(self._poll_event_queue): - break - - elif timeout <=3D self._max_display_latency: - PollScheduler._poll(self, timeout=3Dtimeout) - if timeout =3D=3D 0: - # The display is updated by _schedule() above, so it would be - # redundant to update it here when timeout is 0. - pass - else: - self._status_display.display() - - else: - remaining_timeout =3D timeout - start_time =3D time.time() - while True: - previous_count =3D len(self._poll_event_queue) - PollScheduler._poll(self, - timeout=3Dmin(self._max_display_latency, remaining_timeout)) - self._status_display.display() - if previous_count !=3D len(self._poll_event_queue): - break - elapsed_time =3D time.time() - start_time - if elapsed_time < 0: - # The system clock has changed such that start_time - # is now in the future, so just assume that the - # timeout has already elapsed. - break - remaining_timeout =3D timeout - 1000 * elapsed_time - if remaining_timeout <=3D 0: - break - def _set_max_jobs(self, max_jobs): self._max_jobs =3D max_jobs self._task_queues.jobs.max_jobs =3D max_jobs @@ -388,11 +359,11 @@ class Scheduler(PollScheduler): Check if background mode is enabled and adjust states as necessary. =20 @rtype: bool - @returns: True if background mode is enabled, False otherwise. + @return: True if background mode is enabled, False otherwise. """ background =3D (self._max_jobs is True or \ self._max_jobs > 1 or "--quiet" in self.myopts \ - or "--quiet-build" in self.myopts) and \ + or self.myopts.get("--quiet-build") =3D=3D "y") and \ not bool(self._opts_no_background.intersection(self.myopts)) =20 if background: @@ -405,7 +376,7 @@ class Scheduler(PollScheduler): msg =3D [""] for pkg in interactive_tasks: pkg_str =3D " " + colorize("INFORM", str(pkg.cpv)) - if pkg.root !=3D "/": + if pkg.root_config.settings["ROOT"] !=3D "/": pkg_str +=3D " for " + pkg.root msg.append(pkg_str) msg.append("") @@ -748,7 +719,6 @@ class Scheduler(PollScheduler): self._status_msg("Starting parallel fetch") =20 prefetchers =3D self._prefetchers - getbinpkg =3D "--getbinpkg" in self.myopts =20 for pkg in self._mergelist: # mergelist can contain solved Blocker instances @@ -756,15 +726,13 @@ class Scheduler(PollScheduler): continue prefetcher =3D self._create_prefetcher(pkg) if prefetcher is not None: - self._task_queues.fetch.add(prefetcher) + # This will start the first prefetcher immediately, so that + # self._task() won't discard it. This avoids a case where + # the first prefetcher is discarded, causing the second + # prefetcher to occupy the fetch queue before the first + # fetcher has an opportunity to execute. prefetchers[pkg] =3D prefetcher - - # Start the first prefetcher immediately so that self._task() - # won't discard it. This avoids a case where the first - # prefetcher is discarded, causing the second prefetcher to - # occupy the fetch queue before the first fetcher has an - # opportunity to execute. - self._task_queues.fetch.schedule() + self._task_queues.fetch.add(prefetcher) =20 def _create_prefetcher(self, pkg): """ @@ -792,100 +760,6 @@ class Scheduler(PollScheduler): =20 return prefetcher =20 - def _is_restart_scheduled(self): - """ - Check if the merge list contains a replacement - for the current running instance, that will result - in restart after merge. - @rtype: bool - @returns: True if a restart is scheduled, False otherwise. - """ - if self._opts_no_restart.intersection(self.myopts): - return False - - mergelist =3D self._mergelist - - for i, pkg in enumerate(mergelist): - if self._is_restart_necessary(pkg) and \ - i !=3D len(mergelist) - 1: - return True - - return False - - def _is_restart_necessary(self, pkg): - """ - @return: True if merging the given package - requires restart, False otherwise. - """ - - # Figure out if we need a restart. - if pkg.root =3D=3D self._running_root.root and \ - portage.match_from_list( - portage.const.PORTAGE_PACKAGE_ATOM, [pkg]): - if self._running_portage is None: - return True - elif pkg.cpv !=3D self._running_portage.cpv or \ - '9999' in pkg.cpv or \ - 'git' in pkg.inherited or \ - 'git-2' in pkg.inherited: - return True - return False - - def _restart_if_necessary(self, pkg): - """ - Use execv() to restart emerge. This happens - if portage upgrades itself and there are - remaining packages in the list. - """ - - if self._opts_no_restart.intersection(self.myopts): - return - - if not self._is_restart_necessary(pkg): - return - - if pkg =3D=3D self._mergelist[-1]: - return - - self._main_loop_cleanup() - - logger =3D self._logger - pkg_count =3D self._pkg_count - mtimedb =3D self._mtimedb - bad_resume_opts =3D self._bad_resume_opts - - logger.log(" ::: completed emerge (%s of %s) %s to %s" % \ - (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg.root)) - - logger.log(" *** RESTARTING " + \ - "emerge via exec() after change of " + \ - "portage version.") - - mtimedb["resume"]["mergelist"].remove(list(pkg)) - mtimedb.commit() - portage.run_exitfuncs() - # Don't trust sys.argv[0] here because eselect-python may modify it. - emerge_binary =3D os.path.join(portage.const.PORTAGE_BIN_PATH, 'emerge= ') - mynewargv =3D [emerge_binary, "--resume"] - resume_opts =3D self.myopts.copy() - # For automatic resume, we need to prevent - # any of bad_resume_opts from leaking in - # via EMERGE_DEFAULT_OPTS. - resume_opts["--ignore-default-opts"] =3D True - for myopt, myarg in resume_opts.items(): - if myopt not in bad_resume_opts: - if myarg is True: - mynewargv.append(myopt) - elif isinstance(myarg, list): - # arguments like --exclude that use 'append' action - for x in myarg: - mynewargv.append("%s=3D%s" % (myopt, x)) - else: - mynewargv.append("%s=3D%s" % (myopt, myarg)) - # priority only needs to be adjusted on the first run - os.environ["PORTAGE_NICENESS"] =3D "0" - os.execv(mynewargv[0], mynewargv) - def _run_pkg_pretend(self): """ Since pkg_pretend output may be important, this method sends all @@ -919,11 +793,48 @@ class Scheduler(PollScheduler): root_config =3D x.root_config settings =3D self.pkgsettings[root_config.root] settings.setcpv(x) - tmpdir =3D tempfile.mkdtemp() - tmpdir_orig =3D settings["PORTAGE_TMPDIR"] - settings["PORTAGE_TMPDIR"] =3D tmpdir + + # setcpv/package.env allows for per-package PORTAGE_TMPDIR so we + # have to validate it for each package + rval =3D _check_temp_dir(settings) + if rval !=3D os.EX_OK: + return rval + + build_dir_path =3D os.path.join( + os.path.realpath(settings["PORTAGE_TMPDIR"]), + "portage", x.category, x.pf) + existing_buildir =3D os.path.isdir(build_dir_path) + settings["PORTAGE_BUILDDIR"] =3D build_dir_path + build_dir =3D EbuildBuildDir(scheduler=3Dsched_iface, + settings=3Dsettings) + build_dir.lock() + current_task =3D None =20 try: + + # Clean up the existing build dir, in case pkg_pretend + # checks for available space (bug #390711). + if existing_buildir: + if x.built: + tree =3D "bintree" + infloc =3D os.path.join(build_dir_path, "build-info") + ebuild_path =3D os.path.join(infloc, x.pf + ".ebuild") + else: + tree =3D "porttree" + portdb =3D root_config.trees["porttree"].dbapi + ebuild_path =3D portdb.findname(x.cpv, myrepo=3Dx.repo) + if ebuild_path is None: + raise AssertionError( + "ebuild not found for '%s'" % x.cpv) + portage.package.ebuild.doebuild.doebuild_environment( + ebuild_path, "clean", settings=3Dsettings, + db=3Dself.trees[settings['EROOT']][tree].dbapi) + clean_phase =3D EbuildPhase(background=3DFalse, + phase=3D'clean', scheduler=3Dsched_iface, settings=3Dsettings) + current_task =3D clean_phase + clean_phase.start() + clean_phase.wait() + if x.built: tree =3D "bintree" bintree =3D root_config.trees["bintree"].dbapi.bintree @@ -942,6 +853,7 @@ class Scheduler(PollScheduler): =20 verifier =3D BinpkgVerifier(pkg=3Dx, scheduler=3Dsched_iface) + current_task =3D verifier verifier.start() if verifier.wait() !=3D os.EX_OK: failures +=3D 1 @@ -950,8 +862,8 @@ class Scheduler(PollScheduler): if fetched: bintree.inject(x.cpv, filename=3Dfetched) tbz2_file =3D bintree.getname(x.cpv) - infloc =3D os.path.join(tmpdir, x.category, x.pf, "build-info") - os.makedirs(infloc) + infloc =3D os.path.join(build_dir_path, "build-info") + ensure_dirs(infloc) portage.xpak.tbz2(tbz2_file).unpackinfo(infloc) ebuild_path =3D os.path.join(infloc, x.pf + ".ebuild") settings.configdict["pkg"]["EMERGE_FROM"] =3D "binary" @@ -971,7 +883,8 @@ class Scheduler(PollScheduler): =20 portage.package.ebuild.doebuild.doebuild_environment(ebuild_path, "pretend", settings=3Dsettings, - db=3Dself.trees[settings["ROOT"]][tree].dbapi) + db=3Dself.trees[settings['EROOT']][tree].dbapi) + prepare_build_dirs(root_config.root, settings, cleanup=3D0) =20 vardb =3D root_config.trees['vartree'].dbapi @@ -983,14 +896,21 @@ class Scheduler(PollScheduler): phase=3D"pretend", scheduler=3Dsched_iface, settings=3Dsettings) =20 + current_task =3D pretend_phase pretend_phase.start() ret =3D pretend_phase.wait() if ret !=3D os.EX_OK: failures +=3D 1 portage.elog.elog_process(x.cpv, settings) finally: - shutil.rmtree(tmpdir) - settings["PORTAGE_TMPDIR"] =3D tmpdir_orig + if current_task is not None and current_task.isAlive(): + current_task.cancel() + current_task.wait() + clean_phase =3D EbuildPhase(background=3DFalse, + phase=3D'clean', scheduler=3Dsched_iface, settings=3Dsettings) + clean_phase.start() + clean_phase.wait() + build_dir.unlock() =20 if failures: return 1 @@ -1010,6 +930,10 @@ class Scheduler(PollScheduler): except self._unknown_internal_error: return 1 =20 + rval =3D self._handle_self_update() + if rval !=3D os.EX_OK: + return rval + for root in self.trees: root_config =3D self.trees[root]["root_config"] =20 @@ -1138,12 +1062,9 @@ class Scheduler(PollScheduler): # If only one package failed then just show it's # whole log for easy viewing. failed_pkg =3D self._failed_pkgs_all[-1] - build_dir =3D failed_pkg.build_dir log_file =3D None log_file_real =3D None =20 - log_paths =3D [failed_pkg.build_log] - log_path =3D self._locate_failure_log(failed_pkg) if log_path is not None: try: @@ -1239,9 +1160,6 @@ class Scheduler(PollScheduler): =20 def _locate_failure_log(self, failed_pkg): =20 - build_dir =3D failed_pkg.build_dir - log_file =3D None - log_paths =3D [failed_pkg.build_log] =20 for log_path in log_paths: @@ -1283,7 +1201,7 @@ class Scheduler(PollScheduler): =20 # Skip this if $ROOT !=3D / since it shouldn't matter if there # are unsatisfied system runtime deps in this case. - if pkg.root !=3D '/': + if pkg.root_config.settings["ROOT"] !=3D "/": return =20 completed_tasks =3D self._completed_tasks @@ -1365,8 +1283,6 @@ class Scheduler(PollScheduler): init_buildlog.add_buildlog_main(settings, pkg, trees) return =20 - self._restart_if_necessary(pkg) - # Call mtimedb.commit() after each merge so that # --resume still works after being interrupted # by reboot, sigkill or similar. @@ -1408,7 +1324,8 @@ class Scheduler(PollScheduler): =20 self._failed_pkgs.append(self._failed_pkg( build_dir=3Dbuild_dir, build_log=3Dbuild_log, - pkg=3Dpkg, returncode=3Dbuild.returncode)) + pkg=3Dbuild.pkg, + returncode=3Dbuild.returncode)) if not self._terminated_tasks: self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for") self._status_display.failed =3D len(self._failed_pkgs) @@ -1430,12 +1347,16 @@ class Scheduler(PollScheduler): =20 def _merge(self): =20 + if self._opts_no_background.intersection(self.myopts): + self._set_max_jobs(1) + self._add_prefetchers() self._add_packages() - pkg_queue =3D self._pkg_queue failed_pkgs =3D self._failed_pkgs portage.locks._quiet =3D self._background portage.elog.add_listener(self._elog_listener) + display_timeout_id =3D self.sched_iface.timeout_add( + self._max_display_latency, self._status_display.display) rval =3D os.EX_OK =20 try: @@ -1444,6 +1365,7 @@ class Scheduler(PollScheduler): self._main_loop_cleanup() portage.locks._quiet =3D False portage.elog.remove_listener(self._elog_listener) + self.sched_iface.source_remove(display_timeout_id) if failed_pkgs: rval =3D failed_pkgs[-1].returncode =20 @@ -1524,7 +1446,7 @@ class Scheduler(PollScheduler): merge order @type later: set @rtype: bool - @returns: True if the package is dependent, False otherwise. + @return: True if the package is dependent, False otherwise. """ =20 graph =3D self._digraph @@ -1572,24 +1494,7 @@ class Scheduler(PollScheduler): return temp_settings =20 def _deallocate_config(self, settings): - self._config_pool[settings["ROOT"]].append(settings) - - def _main_loop(self): - - # Only allow 1 job max if a restart is scheduled - # due to portage update. - if self._is_restart_scheduled() or \ - self._opts_no_background.intersection(self.myopts): - self._set_max_jobs(1) - - while self._schedule(): - self._poll_loop() - - while True: - self._schedule() - if not self._is_work_scheduled(): - break - self._poll_loop() + self._config_pool[settings['EROOT']].append(settings) =20 def _keep_scheduling(self): return bool(not self._terminated_tasks and self._pkg_queue and \ @@ -1602,6 +1507,8 @@ class Scheduler(PollScheduler): =20 while True: =20 + state_change =3D 0 + # When the number of jobs and merges drops to zero, # process a single merge from _merge_wait_queue if # it's not empty. We only process one since these are @@ -1612,37 +1519,34 @@ class Scheduler(PollScheduler): not self._task_queues.merge): task =3D self._merge_wait_queue.popleft() task.addExitListener(self._merge_wait_exit_handler) + self._merge_wait_scheduled.append(task) self._task_queues.merge.add(task) self._status_display.merges =3D len(self._task_queues.merge) - self._merge_wait_scheduled.append(task) + state_change +=3D 1 =20 - self._schedule_tasks_imp() - self._status_display.display() + if self._schedule_tasks_imp(): + state_change +=3D 1 =20 - state_change =3D 0 - for q in self._task_queues.values(): - if q.schedule(): - state_change +=3D 1 + self._status_display.display() =20 # Cancel prefetchers if they're the only reason # the main poll loop is still running. if self._failed_pkgs and not self._build_opts.fetchonly and \ not self._is_work_scheduled() and \ self._task_queues.fetch: + # Since this happens asynchronously, it doesn't count in + # state_change (counting it triggers an infinite loop). self._task_queues.fetch.clear() - state_change +=3D 1 =20 if not (state_change or \ (self._merge_wait_queue and not self._jobs and not self._task_queues.merge)): break =20 - return self._keep_scheduling() - def _job_delay(self): """ @rtype: bool - @returns: True if job scheduling should be delayed, False otherwise. + @return: True if job scheduling should be delayed, False otherwise. """ =20 if self._jobs and self._max_load is not None: @@ -1660,7 +1564,7 @@ class Scheduler(PollScheduler): def _schedule_tasks_imp(self): """ @rtype: bool - @returns: True if state changed, False otherwise. + @return: True if state changed, False otherwise. """ =20 state_change =3D 0 @@ -1728,7 +1632,14 @@ class Scheduler(PollScheduler): "installed", pkg.root_config, installed=3DTrue, operation=3D"uninstall") =20 - prefetcher =3D self._prefetchers.pop(pkg, None) + try: + prefetcher =3D self._prefetchers.pop(pkg, None) + except KeyError: + # KeyError observed with PyPy 1.8, despite None given as default. + # Note that PyPy 1.8 has the same WeakValueDictionary code as + # CPython 2.7, so it may be possible for CPython to raise KeyError + # here as well. + prefetcher =3D None if prefetcher is not None and not prefetcher.isAlive(): try: self._task_queues.fetch._task_queue.remove(prefetcher) @@ -1757,7 +1668,7 @@ class Scheduler(PollScheduler): pkg =3D failed_pkg.pkg msg =3D "%s to %s %s" % \ (bad("Failed"), action, colorize("INFORM", pkg.cpv)) - if pkg.root !=3D "/": + if pkg.root_config.settings["ROOT"] !=3D "/": msg +=3D " %s %s" % (preposition, pkg.root) =20 log_path =3D self._locate_failure_log(failed_pkg) @@ -1810,7 +1721,7 @@ class Scheduler(PollScheduler): Use the current resume list to calculate a new one, dropping any packages with unsatisfied deps. @rtype: bool - @returns: True if successful, False otherwise. + @return: True if successful, False otherwise. """ print(colorize("GOOD", "*** Resuming merge...")) =20 @@ -1887,7 +1798,7 @@ class Scheduler(PollScheduler): pkg =3D task msg =3D "emerge --keep-going:" + \ " %s" % (pkg.cpv,) - if pkg.root !=3D "/": + if pkg.root_config.settings["ROOT"] !=3D "/": msg +=3D " for %s" % (pkg.root,) msg +=3D " dropped due to unsatisfied dependency." for line in textwrap.wrap(msg, msg_width):