From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id B60E81389E2 for ; Tue, 23 Dec 2014 20:46:12 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 88D0DE092E; Tue, 23 Dec 2014 20:46:11 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id E9277E091E for ; Tue, 23 Dec 2014 20:46:10 +0000 (UTC) Received: from x51r2.gaikai.org (unknown [100.42.98.7]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) (Authenticated sender: zmedico) by smtp.gentoo.org (Postfix) with ESMTPSA id 19F493405DB; Tue, 23 Dec 2014 20:46:10 +0000 (UTC) From: Zac Medico To: gentoo-portage-dev@lists.gentoo.org Cc: Zac Medico Subject: [gentoo-portage-dev] [PATCH] emerge: add --changed-deps/--binpkg-changed-deps (282927) Date: Tue, 23 Dec 2014 12:45:36 -0800 Message-Id: <1419367536-30393-1-git-send-email-zmedico@gentoo.org> X-Mailer: git-send-email 2.0.5 Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org X-Archives-Salt: 51d72c4e-9f37-441a-9ecf-76b4f402da11 X-Archives-Hash: 65b2e660ca7a1fe4310dea71b6efcb14 The @changed-deps set is useful, but it has limitations similar to the @installed set (see bug #387059), which can make it unsuitable for use when updating the whole system. Therefore, implement two new options that are analogous to --newuse and --binpkg-respect-use, called --changed-deps and --binpkg-changed-deps. The rationale for having a separate --binpkg-* option is the same in both cases: depending on the situation, people may want different behavior for binary packages. For example, just like ---binpkg-respect-use is automatically enabled if the user has not specified --usepkgonly, so is --binpkg-changed-deps (though the user can explicitly override the automatic behavior). In both cases, inconsistencies in dependencies are automatically avoided, increasing the probability of a successful dependency calculation. X-Gentoo-Bug: 282927 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=282927 --- man/emerge.1 | 22 +++- pym/_emerge/create_depgraph_params.py | 16 +++ pym/_emerge/depgraph.py | 138 ++++++++++++++++++++++-- pym/_emerge/main.py | 26 +++++ pym/portage/dep/_slot_operator.py | 13 +++ pym/portage/tests/resolver/test_changed_deps.py | 120 +++++++++++++++++++++ 6 files changed, 323 insertions(+), 12 deletions(-) create mode 100644 pym/portage/tests/resolver/test_changed_deps.py diff --git a/man/emerge.1 b/man/emerge.1 index faa1f33..7eb30a5 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -386,9 +386,20 @@ Specifies an integer number of times to backtrack if dependency calculation fails due to a conflict or an unsatisfied dependency (default: \'10\'). .TP +.BR "\-\-binpkg\-changed\-deps [ y | n ]" +Tells emerge to ignore binary packages for which the corresponding +ebuild dependencies have changed since the packages were built. +In order to help avoid issues with resolving inconsistent dependencies, +this option is automatically enabled unless the \fB\-\-usepkgonly\fR +option is enabled. Behavior with respect to changed build\-time +dependencies is controlled by the \fB\-\-with\-bdeps\fR option. +.TP .BR "\-\-binpkg\-respect\-use [ y | n ]" -Tells emerge to ignore binary packages if their use flags -don't match the current configuration. (default: \'n\') +Tells emerge to ignore binary packages if their USE flags +don't match the current configuration. In order to help avoid issues +with resolving inconsistent USE flag settings, this option is +automatically enabled unless the \fB\-\-usepkgonly\fR option +is enabled. .TP .BR "\-\-buildpkg [ y | n ] (\-b short option)" Tells emerge to build binary packages for all ebuilds processed in @@ -410,6 +421,13 @@ Creates binary packages for all ebuilds processed without actually merging the packages. This comes with the caveat that all build-time dependencies must already be emerged on the system. .TP +.BR "\-\-changed\-deps [ y | n ]" +Tells emerge to replace installed packages for which the corresponding +ebuild dependencies have changed since the packages were built. This +option also implies the \fB\-\-selective\fR option. Behavior with +respect to changed build\-time dependencies is controlled by the +\fB\-\-with\-bdeps\fR option. +.TP .BR "\-\-changed\-use " (\fB\-U\fR) Tells emerge to include installed packages where USE flags have changed since installation. This option also implies the diff --git a/pym/_emerge/create_depgraph_params.py b/pym/_emerge/create_depgraph_params.py index 6f74de7..11e20f4 100644 --- a/pym/_emerge/create_depgraph_params.py +++ b/pym/_emerge/create_depgraph_params.py @@ -22,6 +22,8 @@ def create_depgraph_params(myopts, myaction): # ignore_built_slot_operator_deps: ignore the slot/sub-slot := operator parts # of dependencies that have been recorded when packages where built # with_test_deps: pull in test deps for packages matched by arguments + # changed_deps: rebuild installed packages with outdated deps + # binpkg_changed_deps: reject binary packages with outdated deps myparams = {"recurse" : True} bdeps = myopts.get("--with-bdeps") @@ -51,6 +53,7 @@ def create_depgraph_params(myopts, myaction): "--newuse" in myopts or \ "--reinstall" in myopts or \ "--noreplace" in myopts or \ + myopts.get("--changed-deps", "n") != "n" or \ myopts.get("--selective", "n") != "n": myparams["selective"] = True @@ -99,6 +102,19 @@ def create_depgraph_params(myopts, myaction): # have been specified. myparams['binpkg_respect_use'] = 'auto' + binpkg_changed_deps = myopts.get('--binpkg-changed-deps') + if binpkg_changed_deps is not None: + myparams['binpkg_changed_deps'] = binpkg_changed_deps + elif '--usepkgonly' not in myopts: + # In order to avoid dependency resolution issues due to changed + # dependencies, enable this automatically, as long as it doesn't + # strongly conflict with other options that have been specified. + myparams['binpkg_changed_deps'] = 'auto' + + changed_deps = myopts.get('--changed-deps') + if changed_deps is not None: + myparams['changed_deps'] = changed_deps + if myopts.get("--selective") == "n": # --selective=n can be used to remove selective # behavior that may have been implied by some diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 28abea4..fae117d 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -24,7 +24,8 @@ from portage.dbapi._similar_name_search import similar_name_search from portage.dep import Atom, best_match_to_list, extract_affecting_use, \ check_required_use, human_readable_required_use, match_from_list, \ _repo_separator -from portage.dep._slot_operator import ignore_built_slot_operator_deps +from portage.dep._slot_operator import (ignore_built_slot_operator_deps, + strip_slots) from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use, \ _get_eapi_attrs from portage.exception import (InvalidAtom, InvalidData, InvalidDependString, @@ -794,14 +795,12 @@ class depgraph(object): match the user's config. """ if not self._dynamic_config.ignored_binaries \ - or '--quiet' in self._frozen_config.myopts \ - or self._dynamic_config.myparams.get( - "binpkg_respect_use") in ("y", "n"): + or '--quiet' in self._frozen_config.myopts: return - for pkg in list(self._dynamic_config.ignored_binaries): + ignored_binaries = {} - selected_pkg = list() + for pkg in list(self._dynamic_config.ignored_binaries): for selected_pkg in self._dynamic_config._package_tracker.match( pkg.root, pkg.slot_atom): @@ -819,15 +818,38 @@ class depgraph(object): self._dynamic_config.ignored_binaries.pop(pkg) break - if not self._dynamic_config.ignored_binaries: + else: + for reason, info in self._dynamic_config.\ + ignored_binaries[pkg].items(): + ignored_binaries.setdefault(reason, {})[pkg] = info + + if self._dynamic_config.myparams.get( + "binpkg_respect_use") in ("y", "n"): + ignored_binaries.pop("respect_use", None) + + if self._dynamic_config.myparams.get( + "binpkg_changed_deps") in ("y", "n"): + ignored_binaries.pop("changed_deps", None) + + if not ignored_binaries: return self._show_merge_list() + if ignored_binaries.get("respect_use"): + self._show_ignored_binaries_respect_use( + ignored_binaries["respect_use"]) + + if ignored_binaries.get("changed_deps"): + self._show_ignored_binaries_changed_deps( + ignored_binaries["changed_deps"]) + + def _show_ignored_binaries_respect_use(self, respect_use): + writemsg("\n!!! The following binary packages have been ignored " + \ "due to non matching USE:\n\n", noiselevel=-1) - for pkg, flags in self._dynamic_config.ignored_binaries.items(): + for pkg, flags in respect_use.items(): flag_display = [] for flag in sorted(flags): if flag not in pkg.use.enabled: @@ -852,6 +874,30 @@ class depgraph(object): line = colorize("INFORM", line) writemsg(line + "\n", noiselevel=-1) + def _show_ignored_binaries_changed_deps(self, changed_deps): + + writemsg("\n!!! The following binary packages have been " + "ignored due to changed dependencies:\n\n", + noiselevel=-1) + + for pkg in changed_deps: + msg = " %s%s%s" % (pkg.cpv, _repo_separator, pkg.repo) + if pkg.root_config.settings["ROOT"] != "/": + msg += " for %s" % pkg.root + writemsg("%s\n" % msg, noiselevel=-1) + + msg = [ + "", + "NOTE: The --binpkg-changed-deps=n option will prevent emerge", + " from ignoring these binary packages if possible.", + " Using --binpkg-changed-deps=y will silence this warning." + ] + + for line in msg: + if line: + line = colorize("INFORM", line) + writemsg(line + "\n", noiselevel=-1) + def _get_missed_updates(self): # In order to minimize noise, show only the highest @@ -2173,6 +2219,52 @@ class depgraph(object): return flags return None + def _changed_deps(self, pkg): + + ebuild = None + try: + ebuild = self._pkg(pkg.cpv, "ebuild", + pkg.root_config, myrepo=pkg.repo) + except PackageNotFound: + # Use first available instance of the same version. + for ebuild in self._iter_match_pkgs( + pkg.root_config, "ebuild", Atom("=" + pkg.cpv)): + break + + if ebuild is None: + changed = False + else: + if self._dynamic_config.myparams.get("bdeps", "n") == "y": + depvars = Package._dep_keys + else: + depvars = Package._runtime_keys + + # Use _raw_metadata, in order to avoid interaction + # with --dynamic-deps. + try: + built_deps = [] + for k in depvars: + dep_struct = portage.dep.use_reduce( + pkg._raw_metadata[k], uselist=pkg.use.enabled, + eapi=pkg.eapi, token_class=Atom) + strip_slots(dep_struct) + built_deps.append(dep_struct) + except InvalidDependString: + changed = True + else: + unbuilt_deps = [] + for k in depvars: + dep_struct = portage.dep.use_reduce( + ebuild._raw_metadata[k], + uselist=pkg.use.enabled, + eapi=ebuild.eapi, token_class=Atom) + strip_slots(dep_struct) + unbuilt_deps.append(dep_struct) + + changed = built_deps != unbuilt_deps + + return changed + def _create_graph(self, allow_unsatisfied=False): dep_stack = self._dynamic_config._dep_stack dep_disjunctive_stack = self._dynamic_config._dep_disjunctive_stack @@ -4595,8 +4687,14 @@ class depgraph(object): mreasons = ["need to rebuild from source"] elif pkg.installed and root_slot in self._rebuild.reinstall_list: mreasons = ["need to rebuild from source"] - elif pkg.built and not mreasons: + elif (pkg.built and not mreasons and + self._dynamic_config.ignored_binaries.get( + pkg, {}).get("respect_use")): mreasons = ["use flag configuration mismatch"] + elif (pkg.built and not mreasons and + self._dynamic_config.ignored_binaries.get( + pkg, {}).get("changed_deps")): + mreasons = ["changed deps"] masked_packages.append( (root_config, pkgsettings, cpv, repo, metadata, mreasons)) @@ -5693,6 +5791,12 @@ class depgraph(object): # reject the built package if necessary. reinstall_use = ("--newuse" in self._frozen_config.myopts or \ "--reinstall" in self._frozen_config.myopts) + changed_deps = ( + self._dynamic_config.myparams.get( + "changed_deps", "n") != "n") + binpkg_changed_deps = ( + self._dynamic_config.myparams.get( + "binpkg_changed_deps", "n") != "n") respect_use = self._dynamic_config.myparams.get("binpkg_respect_use") in ("y", "auto") if built and not useoldpkg and \ (not installed or matched_packages) and \ @@ -5719,8 +5823,22 @@ class depgraph(object): forced_flags, old_use, iuses, now_use, cur_iuse) if reinstall_for_flags: if not pkg.installed: - self._dynamic_config.ignored_binaries.setdefault(pkg, set()).update(reinstall_for_flags) + self._dynamic_config.\ + ignored_binaries.setdefault( + pkg, {}).setdefault( + "respect_use", set()).update( + reinstall_for_flags) break + + if (((installed and changed_deps) or + (not installed and binpkg_changed_deps)) and + self._changed_deps(pkg)): + if not installed: + self._dynamic_config.\ + ignored_binaries.setdefault( + pkg, {})["changed_deps"] = True + break + # Compare current config to installed package # and do not reinstall if possible. if not installed and not useoldpkg and cpv in vardb.match(atom): diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 7c707f9..ecbbdb0 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -129,7 +129,9 @@ def insert_optional_args(args): '--autounmask-keep-masks': y_or_n, '--autounmask-unrestricted-atoms' : y_or_n, '--autounmask-write' : y_or_n, + '--binpkg-changed-deps' : y_or_n, '--buildpkg' : y_or_n, + '--changed-deps' : y_or_n, '--complete-graph' : y_or_n, '--deep' : valid_integers, '--depclean-lib-check' : y_or_n, @@ -353,6 +355,12 @@ def parse_opts(tmpcmdline, silent=False): "action" : "store" }, + "--binpkg-changed-deps": { + "help" : ("reject binary packages with outdated " + "dependencies"), + "choices" : true_y_or_n + }, + "--buildpkg": { "shortopt" : "-b", "help" : "build binary packages", @@ -367,6 +375,12 @@ def parse_opts(tmpcmdline, silent=False): "action" : "append" }, + "--changed-deps": { + "help" : ("replace installed packages with " + "outdated dependencies"), + "choices" : true_y_or_n + }, + "--config-root": { "help":"specify the location for portage configuration files", "action":"store" @@ -722,6 +736,12 @@ def parse_opts(tmpcmdline, silent=False): if myoptions.autounmask_write in true_y: myoptions.autounmask_write = True + if myoptions.binpkg_changed_deps is not None: + if myoptions.binpkg_changed_deps in true_y: + myoptions.binpkg_changed_deps = 'y' + else: + myoptions.binpkg_changed_deps = 'n' + if myoptions.buildpkg in true_y: myoptions.buildpkg = True @@ -731,6 +751,12 @@ def parse_opts(tmpcmdline, silent=False): parser.error("Invalid Atom(s) in --buildpkg-exclude parameter: '%s'\n" % \ (",".join(bad_atoms),)) + if myoptions.changed_deps is not None: + if myoptions.changed_deps in true_y: + myoptions.changed_deps = 'y' + else: + myoptions.changed_deps = 'n' + if myoptions.changed_use is not False: myoptions.reinstall = "changed-use" myoptions.changed_use = False diff --git a/pym/portage/dep/_slot_operator.py b/pym/portage/dep/_slot_operator.py index 8b67fc5..8ce570d 100644 --- a/pym/portage/dep/_slot_operator.py +++ b/pym/portage/dep/_slot_operator.py @@ -8,6 +8,19 @@ from portage.eapi import _get_eapi_attrs from portage.exception import InvalidData from _emerge.Package import Package +def strip_slots(dep_struct): + """ + Search dep_struct for any slot := operators and remove the + slot/sub-slot part, while preserving the operator. The result + is suitable for --changed-deps comparisons. + """ + for i, x in enumerate(dep_struct): + if isinstance(x, list): + strip_slots(x) + elif (isinstance(x, Atom) and + x.slot_operator == "=" and x.slot is not None): + dep_struct[i] = x.with_slot("=") + def find_built_slot_operator_atoms(pkg): atoms = {} for k in Package._dep_keys: diff --git a/pym/portage/tests/resolver/test_changed_deps.py b/pym/portage/tests/resolver/test_changed_deps.py new file mode 100644 index 0000000..2421c53 --- /dev/null +++ b/pym/portage/tests/resolver/test_changed_deps.py @@ -0,0 +1,120 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ( + ResolverPlayground, ResolverPlaygroundTestCase) + +class ChangedDepsTestCase(TestCase): + + def testChangedDeps(self): + + ebuilds = { + "app-misc/A-0": { + "DEPEND": "app-misc/B", + "RDEPEND": "app-misc/B", + }, + "app-misc/B-0": { + } + } + + binpkgs = { + "app-misc/A-0": {}, + } + + installed = { + "app-misc/A-0": {}, + } + + world= ( + "app-misc/A", + ) + + test_cases = ( + + # --dynamic-deps=n causes the original deps to be respected + ResolverPlaygroundTestCase( + ["@world"], + success = True, + options = { + "--update": True, + "--deep": True, + "--dynamic-deps": "n", + "--usepkg": True, + }, + mergelist = [] + ), + + # --dynamic-deps causes app-misc/B to get pulled in + ResolverPlaygroundTestCase( + ["@world"], + success = True, + options = { + "--update": True, + "--deep": True, + "--usepkg": True, + }, + mergelist = ["app-misc/B-0"] + ), + + # --changed-deps causes app-misc/A to be rebuilt + ResolverPlaygroundTestCase( + ["@world"], + success = True, + options = { + "--update": True, + "--deep": True, + "--changed-deps": "y", + "--usepkg": True, + }, + mergelist = ["app-misc/B-0", "app-misc/A-0"] + ), + + # --usepkgonly prevents automatic --binpkg-changed-deps + ResolverPlaygroundTestCase( + ["app-misc/A"], + success = True, + options = { + "--changed-deps": "y", + "--usepkgonly": True, + }, + mergelist = ["[binary]app-misc/A-0"] + ), + + # Test automatic --binpkg-changed-deps, which cases the + # binpkg with stale deps to be ignored (with warning + # message) + ResolverPlaygroundTestCase( + ["app-misc/A"], + success = True, + options = { + "--usepkg": True, + }, + mergelist = ["app-misc/B-0", "app-misc/A-0"] + ), + ) + test_cases = ( + + # Forcibly disable --binpkg-changed-deps, which causes + # --changed-deps to be overridden by --binpkg-changed-deps + ResolverPlaygroundTestCase( + ["app-misc/A"], + success = True, + options = { + "--binpkg-changed-deps": "n", + "--changed-deps": "y", + "--usepkg": True, + }, + mergelist = ["[binary]app-misc/A-0"] + ), + ) + + playground = ResolverPlayground(debug=False, ebuilds=ebuilds, + binpkgs=binpkgs, installed=installed, world=world) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, + True, test_case.fail_msg) + finally: + playground.cleanup() -- 2.0.5