From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 30F4F1381F3 for ; Fri, 7 Dec 2012 14:58:41 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id E7F7121C024; Fri, 7 Dec 2012 14:58:33 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 5DD2E21C024 for ; Fri, 7 Dec 2012 14:58:33 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 49BD533DAFE for ; Fri, 7 Dec 2012 14:58:32 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id E267FE5436 for ; Fri, 7 Dec 2012 14:58:30 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1354892217.7ed42c1b0bd9448bbddd94646c7b70bfb3a967d7.zorry@gentoo> Subject: [gentoo-commits] dev/zorry:master commit in: gobs/pym/ X-VCS-Repository: dev/zorry X-VCS-Files: gobs/pym/Scheduler.py gobs/pym/actions.py X-VCS-Directories: gobs/pym/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 7ed42c1b0bd9448bbddd94646c7b70bfb3a967d7 X-VCS-Branch: master Date: Fri, 7 Dec 2012 14:58:30 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 94cca11a-1ff4-46d2-a2f9-4bb0525f6b4c X-Archives-Hash: 0a874145297c3a47b9b9324689b0a0a9 commit: 7ed42c1b0bd9448bbddd94646c7b70bfb3a967d7 Author: Magnus Granberg gentoo org> AuthorDate: Fri Dec 7 14:56:57 2012 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Fri Dec 7 14:56:57 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=dev/zorry.git;a=commit;h=7ed42c1b Update the Scheduler.py --- gobs/pym/Scheduler.py | 118 +++++++++++++++++++++++++++++++++++-------------- gobs/pym/actions.py | 2 +- 2 files changed, 85 insertions(+), 35 deletions(-) diff --git a/gobs/pym/Scheduler.py b/gobs/pym/Scheduler.py index 229c595..9614503 100644 --- a/gobs/pym/Scheduler.py +++ b/gobs/pym/Scheduler.py @@ -28,6 +28,8 @@ from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet from portage.util import ensure_dirs, writemsg, writemsg_level from portage.util.SlotObject import SlotObject +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.EventLoop import EventLoop from portage.package.ebuild.digestcheck import digestcheck from portage.package.ebuild.digestgen import digestgen from portage.package.ebuild.doebuild import (_check_temp_dir, @@ -50,6 +52,7 @@ from _emerge.EbuildFetcher import EbuildFetcher from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog from _emerge.FakeVartree import FakeVartree +from _emerge.getloadavg import getloadavg from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo from _emerge.JobStatusDisplay import JobStatusDisplay @@ -59,13 +62,16 @@ from _emerge.PackageMerge import PackageMerge from _emerge.PollScheduler import PollScheduler from _emerge.SequentialTaskQueue import SequentialTaskQueue -from gobs.build_log import gobs_buildlog +from gobs.build_log import add_buildlog_main if sys.hexversion >= 0x3000000: basestring = str class Scheduler(PollScheduler): + # max time between loadavg checks (milliseconds) + _loadavg_latency = 30000 + # max time between display status updates (milliseconds) _max_display_latency = 3000 @@ -81,7 +87,7 @@ class Scheduler(PollScheduler): _opts_no_self_update = frozenset(["--buildpkgonly", "--fetchonly", "--fetch-all-uri", "--pretend"]) - class _iface_class(PollScheduler._sched_iface_class): + class _iface_class(SchedulerInterface): __slots__ = ("fetch", "scheduleSetup", "scheduleUnpack") @@ -138,7 +144,7 @@ class Scheduler(PollScheduler): def __init__(self, settings, trees, mtimedb, myopts, spinner, mergelist=None, favorites=None, graph_config=None): - PollScheduler.__init__(self) + PollScheduler.__init__(self, main=True) if mergelist is not None: warnings.warn("The mergelist parameter of the " + \ @@ -217,14 +223,15 @@ class Scheduler(PollScheduler): fetch_iface = self._fetch_iface_class(log_file=self._fetch_log, schedule=self._schedule_fetch) self._sched_iface = self._iface_class( + self._event_loop, + is_background=self._is_background, fetch=fetch_iface, scheduleSetup=self._schedule_setup, - scheduleUnpack=self._schedule_unpack, - **dict((k, getattr(self.sched_iface, k)) - for k in self.sched_iface.__slots__)) + scheduleUnpack=self._schedule_unpack) self._prefetchers = weakref.WeakValueDictionary() self._pkg_queue = [] + self._jobs = 0 self._running_tasks = {} self._completed_tasks = set() @@ -243,9 +250,7 @@ class Scheduler(PollScheduler): # The load average takes some time to respond when new # jobs are added, so we need to limit the rate of adding # new jobs. - self._job_delay_max = 10 - self._job_delay_factor = 1.0 - self._job_delay_exp = 1.5 + self._job_delay_max = 5 self._previous_job_start_time = None # This is used to memoize the _choose_pkg() result when @@ -300,15 +305,10 @@ class Scheduler(PollScheduler): if not portage.dep.match_from_list( portage.const.PORTAGE_PACKAGE_ATOM, [x]): continue - if self._running_portage is None or \ - self._running_portage.cpv != x.cpv or \ - '9999' in x.cpv or \ - 'git' in x.inherited or \ - 'git-2' in x.inherited: - rval = _check_temp_dir(self.settings) - if rval != os.EX_OK: - return rval - _prepare_self_update(self.settings) + rval = _check_temp_dir(self.settings) + if rval != os.EX_OK: + return rval + _prepare_self_update(self.settings) break return os.EX_OK @@ -328,10 +328,13 @@ class Scheduler(PollScheduler): self._set_graph_config(graph_config) self._blocker_db = {} dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_operator_deps = self.myopts.get( + "--ignore-built-slot-operator-deps", "n") == "y" for root in self.trees: if graph_config is None: fake_vartree = FakeVartree(self.trees[root]["root_config"], - pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps) + pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps, + ignore_built_slot_operator_deps=ignore_built_slot_operator_deps) fake_vartree.sync() else: fake_vartree = graph_config.trees[root]['vartree'] @@ -653,10 +656,11 @@ class Scheduler(PollScheduler): if value and value.strip(): continue msg = _("%(var)s is not set... " - "Are you missing the '%(configroot)setc/make.profile' symlink? " + "Are you missing the '%(configroot)s%(profile_path)s' symlink? " "Is the symlink correct? " "Is your portage tree complete?") % \ - {"var": var, "configroot": settings["PORTAGE_CONFIGROOT"]} + {"var": var, "configroot": settings["PORTAGE_CONFIGROOT"], + "profile_path": portage.const.PROFILE_PATH} out = portage.output.EOutput() for line in textwrap.wrap(msg, 70): @@ -769,10 +773,10 @@ class Scheduler(PollScheduler): failures = 0 - # Use a local PollScheduler instance here, since we don't + # Use a local EventLoop instance here, since we don't # want tasks here to trigger the usual Scheduler callbacks # that handle job scheduling and status display. - sched_iface = PollScheduler().sched_iface + sched_iface = SchedulerInterface(EventLoop(main=False)) for x in self._mergelist: if not isinstance(x, Package): @@ -1249,7 +1253,6 @@ class Scheduler(PollScheduler): pkg = merge.merge.pkg settings = merge.merge.settings trees = self.trees - init_buildlog = gobs_buildlog() if merge.returncode != os.EX_OK: build_dir = settings.get("PORTAGE_BUILDDIR") build_log = settings.get("PORTAGE_LOG_FILE") @@ -1261,7 +1264,7 @@ class Scheduler(PollScheduler): if not self._terminated_tasks: self._failed_pkg_msg(self._failed_pkgs[-1], "install", "to") self._status_display.failed = len(self._failed_pkgs) - init_buildlog.add_buildlog_main(settings, pkg, trees) + add_buildlog_main(settings, pkg, trees) return self._task_complete(pkg) @@ -1280,7 +1283,6 @@ class Scheduler(PollScheduler): self._pkg_cache.pop(pkg_to_replace, None) if pkg.installed: - init_buildlog.add_buildlog_main(settings, pkg, trees) return # Call mtimedb.commit() after each merge so that @@ -1291,7 +1293,6 @@ class Scheduler(PollScheduler): if not mtimedb["resume"]["mergelist"]: del mtimedb["resume"] mtimedb.commit() - init_buildlog.add_buildlog_main(settings, pkg, trees) def _build_exit(self, build): self._running_tasks.pop(id(build), None) @@ -1318,7 +1319,6 @@ class Scheduler(PollScheduler): settings = build.settings trees = self.trees pkg=build.pkg - init_buildlog = gobs_buildlog() build_dir = settings.get("PORTAGE_BUILDDIR") build_log = settings.get("PORTAGE_LOG_FILE") @@ -1330,7 +1330,7 @@ class Scheduler(PollScheduler): self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for") self._status_display.failed = len(self._failed_pkgs) self._deallocate_config(build.settings) - init_buildlog.add_buildlog_main(settings, pkg, trees) + add_buildlog_main(settings, pkg, trees) self._jobs -= 1 self._status_display.running = self._jobs self._schedule() @@ -1345,6 +1345,38 @@ class Scheduler(PollScheduler): blocker_db = self._blocker_db[pkg.root] blocker_db.discardBlocker(pkg) + def _main_loop(self): + term_check_id = self._event_loop.idle_add(self._termination_check) + loadavg_check_id = None + if self._max_load is not None and \ + self._loadavg_latency is not None and \ + (self._max_jobs is True or self._max_jobs > 1): + # We have to schedule periodically, in case the load + # average has changed since the last call. + loadavg_check_id = self._event_loop.timeout_add( + self._loadavg_latency, self._schedule) + + try: + # Populate initial event sources. Unless we're scheduling + # based on load average, we only need to do this once + # here, since it can be called during the loop from within + # event handlers. + self._schedule() + + # Loop while there are jobs to be scheduled. + while self._keep_scheduling(): + self._event_loop.iteration() + + # Clean shutdown of previously scheduled jobs. In the + # case of termination, this allows for basic cleanup + # such as flushing of buffered output to logs. + while self._is_work_scheduled(): + self._event_loop.iteration() + finally: + self._event_loop.source_remove(term_check_id) + if loadavg_check_id is not None: + self._event_loop.source_remove(loadavg_check_id) + def _merge(self): if self._opts_no_background.intersection(self.myopts): @@ -1355,8 +1387,10 @@ class Scheduler(PollScheduler): failed_pkgs = self._failed_pkgs portage.locks._quiet = self._background portage.elog.add_listener(self._elog_listener) - display_timeout_id = self.sched_iface.timeout_add( - self._max_display_latency, self._status_display.display) + display_timeout_id = None + if self._status_display._isatty and not self._status_display.quiet: + display_timeout_id = self._event_loop.timeout_add( + self._max_display_latency, self._status_display.display) rval = os.EX_OK try: @@ -1365,7 +1399,8 @@ class Scheduler(PollScheduler): self._main_loop_cleanup() portage.locks._quiet = False portage.elog.remove_listener(self._elog_listener) - self.sched_iface.source_remove(display_timeout_id) + if display_timeout_id is not None: + self._event_loop.source_remove(display_timeout_id) if failed_pkgs: rval = failed_pkgs[-1].returncode @@ -1503,6 +1538,9 @@ class Scheduler(PollScheduler): def _is_work_scheduled(self): return bool(self._running_tasks) + def _running_job_count(self): + return self._jobs + def _schedule_tasks(self): while True: @@ -1552,15 +1590,27 @@ class Scheduler(PollScheduler): if self._jobs and self._max_load is not None: current_time = time.time() + try: + avg1, avg5, avg15 = getloadavg() + except OSError: + return False - delay = self._job_delay_factor * self._jobs ** self._job_delay_exp + delay = self._job_delay_max * avg1 / self._max_load if delay > self._job_delay_max: delay = self._job_delay_max - if (current_time - self._previous_job_start_time) < delay: + elapsed_seconds = current_time - self._previous_job_start_time + # elapsed_seconds < 0 means the system clock has been adjusted + if elapsed_seconds > 0 and elapsed_seconds < delay: + self._event_loop.timeout_add( + 1000 * (delay - elapsed_seconds), self._schedule_once) return True return False + def _schedule_once(self): + self._schedule() + return False + def _schedule_tasks_imp(self): """ @rtype: bool diff --git a/gobs/pym/actions.py b/gobs/pym/actions.py index e848859..4b48408 100644 --- a/gobs/pym/actions.py +++ b/gobs/pym/actions.py @@ -70,7 +70,7 @@ from _emerge.MetadataRegen import MetadataRegen from _emerge.Package import Package from _emerge.ProgressHandler import ProgressHandler from _emerge.RootConfig import RootConfig -from _emerge.Scheduler import Scheduler +from gobs..Scheduler import Scheduler from _emerge.search import search from _emerge.SetArg import SetArg from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice