* [gentoo-portage-dev] [PATCH 1/2] Move and rename check_reverse_dependencies
@ 2022-11-25 3:36 Pin-yen Lin
2022-11-25 3:36 ` [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies Pin-yen Lin
0 siblings, 1 reply; 4+ messages in thread
From: Pin-yen Lin @ 2022-11-25 3:36 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Pin-yen Lin
Move the function outside the original scope and rename the function to
_slot_operator_check_reverse_dependencies to accommodate the coding
style.
Signed-off-by: Pin-yen Lin <treapking@chromium.org>
---
lib/_emerge/depgraph.py | 232 +++++++++++++++++++++-------------------
1 file changed, 119 insertions(+), 113 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index c976f8205..ce6cabcc1 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -2240,6 +2240,117 @@ class depgraph:
return None
+ def _slot_operator_check_reverse_dependencies(
+ self, existing_pkg, candidate_pkg, replacement_parent=None
+ ):
+ """
+ Check if candidate_pkg satisfies all of existing_pkg's non-
+ slot operator parents.
+ """
+ debug = "--debug" in self._frozen_config.myopts
+ built_slot_operator_parents = set()
+ for parent, atom in self._dynamic_config._parent_atoms.get(
+ existing_pkg, []
+ ):
+ if atom.soname or atom.slot_operator_built:
+ built_slot_operator_parents.add(parent)
+
+ for parent, atom in self._dynamic_config._parent_atoms.get(
+ existing_pkg, []
+ ):
+ if isinstance(parent, Package):
+ if parent in built_slot_operator_parents:
+ if hasattr(atom, "_orig_atom"):
+ # If atom is the result of virtual expansion, then
+ # derefrence it to _orig_atom so that it will be correctly
+ # handled as a built slot operator dependency when
+ # appropriate (see bug 764764).
+ atom = atom._orig_atom
+ # This parent may need to be rebuilt, therefore
+ # discard its soname and built slot operator
+ # dependency components which are not necessarily
+ # relevant.
+ if atom.soname:
+ continue
+ elif atom.package and atom.slot_operator_built:
+ # This discards the slot/subslot component.
+ atom = atom.with_slot("=")
+
+ if replacement_parent is not None and (
+ replacement_parent.slot_atom == parent.slot_atom
+ or replacement_parent.cpv == parent.cpv
+ ):
+ # This parent is irrelevant because we intend to
+ # replace it with replacement_parent.
+ continue
+
+ if any(
+ pkg is not parent
+ and (pkg.slot_atom == parent.slot_atom or pkg.cpv == parent.cpv)
+ for pkg in self._dynamic_config._package_tracker.match(
+ parent.root, Atom(parent.cp)
+ )
+ ):
+ # This parent may need to be eliminated due to a
+ # slot conflict, so its dependencies aren't
+ # necessarily relevant.
+ continue
+
+ if not self._too_deep(
+ parent.depth
+ ) and not self._frozen_config.excluded_pkgs.findAtomForPackage(
+ parent, modified_use=self._pkg_use_enabled(parent)
+ ):
+ # Check for common reasons that the parent's
+ # dependency might be irrelevant.
+ if self._upgrade_available(parent):
+ # This parent could be replaced by
+ # an upgrade (bug 584626).
+ continue
+ if parent.installed and self._in_blocker_conflict(parent):
+ # This parent could be uninstalled in order
+ # to solve a blocker conflict (bug 612772).
+ continue
+ if self._dynamic_config.digraph.has_edge(parent, existing_pkg):
+ # There is a direct circular dependency between
+ # parent and existing_pkg. This type of
+ # relationship tends to prevent updates
+ # of packages (bug 612874). Since candidate_pkg
+ # is available, we risk a missed update if we
+ # don't try to eliminate this parent from the
+ # graph. Therefore, we give candidate_pkg a
+ # chance, and assume that it will be masked
+ # by backtracking if necessary.
+ continue
+
+ atom_set = InternalPackageSet(initial_atoms=(atom,), allow_repo=True)
+ if not atom_set.findAtomForPackage(
+ candidate_pkg, modified_use=self._pkg_use_enabled(candidate_pkg)
+ ):
+ if debug:
+ parent_atoms = []
+ for (
+ other_parent,
+ other_atom,
+ ) in self._dynamic_config._parent_atoms.get(existing_pkg, []):
+ if other_parent is parent:
+ parent_atoms.append(other_atom)
+ msg = (
+ "",
+ "",
+ "_slot_operator_check_reverse_dependencies:",
+ " candidate package does not match atom '%s': %s"
+ % (atom, candidate_pkg),
+ " parent: %s" % parent,
+ " parent atoms: %s" % " ".join(parent_atoms),
+ "",
+ )
+ writemsg_level(
+ "\n".join(msg), noiselevel=-1, level=logging.DEBUG
+ )
+ return False
+ return True
+
def _slot_operator_update_probe(
self, dep, new_child_slot=False, slot_conflict=False, autounmask_level=None
):
@@ -2274,116 +2385,6 @@ class depgraph:
want_downgrade = None
want_downgrade_parent = None
- def check_reverse_dependencies(
- existing_pkg, candidate_pkg, replacement_parent=None
- ):
- """
- Check if candidate_pkg satisfies all of existing_pkg's non-
- slot operator parents.
- """
- built_slot_operator_parents = set()
- for parent, atom in self._dynamic_config._parent_atoms.get(
- existing_pkg, []
- ):
- if atom.soname or atom.slot_operator_built:
- built_slot_operator_parents.add(parent)
-
- for parent, atom in self._dynamic_config._parent_atoms.get(
- existing_pkg, []
- ):
- if isinstance(parent, Package):
- if parent in built_slot_operator_parents:
- if hasattr(atom, "_orig_atom"):
- # If atom is the result of virtual expansion, then
- # derefrence it to _orig_atom so that it will be correctly
- # handled as a built slot operator dependency when
- # appropriate (see bug 764764).
- atom = atom._orig_atom
- # This parent may need to be rebuilt, therefore
- # discard its soname and built slot operator
- # dependency components which are not necessarily
- # relevant.
- if atom.soname:
- continue
- elif atom.package and atom.slot_operator_built:
- # This discards the slot/subslot component.
- atom = atom.with_slot("=")
-
- if replacement_parent is not None and (
- replacement_parent.slot_atom == parent.slot_atom
- or replacement_parent.cpv == parent.cpv
- ):
- # This parent is irrelevant because we intend to
- # replace it with replacement_parent.
- continue
-
- if any(
- pkg is not parent
- and (pkg.slot_atom == parent.slot_atom or pkg.cpv == parent.cpv)
- for pkg in self._dynamic_config._package_tracker.match(
- parent.root, Atom(parent.cp)
- )
- ):
- # This parent may need to be eliminated due to a
- # slot conflict, so its dependencies aren't
- # necessarily relevant.
- continue
-
- if not self._too_deep(
- parent.depth
- ) and not self._frozen_config.excluded_pkgs.findAtomForPackage(
- parent, modified_use=self._pkg_use_enabled(parent)
- ):
- # Check for common reasons that the parent's
- # dependency might be irrelevant.
- if self._upgrade_available(parent):
- # This parent could be replaced by
- # an upgrade (bug 584626).
- continue
- if parent.installed and self._in_blocker_conflict(parent):
- # This parent could be uninstalled in order
- # to solve a blocker conflict (bug 612772).
- continue
- if self._dynamic_config.digraph.has_edge(parent, existing_pkg):
- # There is a direct circular dependency between
- # parent and existing_pkg. This type of
- # relationship tends to prevent updates
- # of packages (bug 612874). Since candidate_pkg
- # is available, we risk a missed update if we
- # don't try to eliminate this parent from the
- # graph. Therefore, we give candidate_pkg a
- # chance, and assume that it will be masked
- # by backtracking if necessary.
- continue
-
- atom_set = InternalPackageSet(initial_atoms=(atom,), allow_repo=True)
- if not atom_set.findAtomForPackage(
- candidate_pkg, modified_use=self._pkg_use_enabled(candidate_pkg)
- ):
- if debug:
- parent_atoms = []
- for (
- other_parent,
- other_atom,
- ) in self._dynamic_config._parent_atoms.get(existing_pkg, []):
- if other_parent is parent:
- parent_atoms.append(other_atom)
- msg = (
- "",
- "",
- "check_reverse_dependencies:",
- " candidate package does not match atom '%s': %s"
- % (atom, candidate_pkg),
- " parent: %s" % parent,
- " parent atoms: %s" % " ".join(parent_atoms),
- "",
- )
- writemsg_level(
- "\n".join(msg), noiselevel=-1, level=logging.DEBUG
- )
- return False
- return True
-
for replacement_parent in self._iter_similar_available(
dep.parent, dep.parent.slot_atom, autounmask_level=autounmask_level
):
@@ -2397,7 +2398,9 @@ class depgraph:
if not want_downgrade_parent:
continue
- if not check_reverse_dependencies(dep.parent, replacement_parent):
+ if not self._slot_operator_check_reverse_dependencies(
+ dep.parent, replacement_parent
+ ):
continue
selected_atoms = None
@@ -2510,8 +2513,11 @@ class depgraph:
if atom_not_selected:
break
- if not insignificant and check_reverse_dependencies(
- dep.child, pkg, replacement_parent=replacement_parent
+ if (
+ not insignificant
+ and self._slot_operator_check_reverse_dependencies(
+ dep.child, pkg, replacement_parent=replacement_parent
+ )
):
candidate_pkg_atoms.append((pkg, unevaluated_atom or atom))
--
2.38.1.584.g0f3c55d4c2-goog
^ permalink raw reply related [flat|nested] 4+ messages in thread* [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies
2022-11-25 3:36 [gentoo-portage-dev] [PATCH 1/2] Move and rename check_reverse_dependencies Pin-yen Lin
@ 2022-11-25 3:36 ` Pin-yen Lin
2022-11-25 17:08 ` Matt Turner
2022-11-28 15:47 ` Zac Medico
0 siblings, 2 replies; 4+ messages in thread
From: Pin-yen Lin @ 2022-11-25 3:36 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Pin-yen Lin
Add lru_cache to speed up the running time of "Calculating
dependencies".
In a ChromeOS use case, this patch decreases the running time from
311s to 197s with almost no memory usage increase.
Signed-off-by: Pin-yen Lin <treapking@chromium.org>
---
lib/_emerge/depgraph.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index ce6cabcc1..9649bb2a8 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -2240,6 +2240,7 @@ class depgraph:
return None
+ @functools.lru_cache(maxsize=100)
def _slot_operator_check_reverse_dependencies(
self, existing_pkg, candidate_pkg, replacement_parent=None
):
--
2.38.1.584.g0f3c55d4c2-goog
^ permalink raw reply related [flat|nested] 4+ messages in thread* Re: [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies
2022-11-25 3:36 ` [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies Pin-yen Lin
@ 2022-11-25 17:08 ` Matt Turner
2022-11-28 15:47 ` Zac Medico
1 sibling, 0 replies; 4+ messages in thread
From: Matt Turner @ 2022-11-25 17:08 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Pin-yen Lin
On Thu, Nov 24, 2022 at 10:36 PM Pin-yen Lin <treapking@chromium.org> wrote:
>
> Add lru_cache to speed up the running time of "Calculating
> dependencies".
>
> In a ChromeOS use case, this patch decreases the running time from
> 311s to 197s with almost no memory usage increase.
>
> Signed-off-by: Pin-yen Lin <treapking@chromium.org>
Thank you!
With recent subslot rebuilds (icu, boost, poppler), I measure an
improvement of 19%!
Benchmark 1: emerge @world -vuNDp
Time (mean ± σ): 42.668 s ± 0.555 s [User: 42.095 s, System: 0.315 s]
Range (min … max): 41.572 s … 43.342 s 10 runs
Benchmark 2: emerge @world -vuNDp
Time (mean ± σ): 35.991 s ± 0.154 s [User: 35.409 s, System: 0.332 s]
Range (min … max): 35.831 s … 36.306 s 10 runs
Summary
'emerge @world -vuNDp' ran
1.19 ± 0.02 times faster than 'emerge @world -vuNDp'
^ permalink raw reply [flat|nested] 4+ messages in thread* Re: [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies
2022-11-25 3:36 ` [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies Pin-yen Lin
2022-11-25 17:08 ` Matt Turner
@ 2022-11-28 15:47 ` Zac Medico
1 sibling, 0 replies; 4+ messages in thread
From: Zac Medico @ 2022-11-28 15:47 UTC (permalink / raw
To: gentoo-portage-dev, Pin-yen Lin
On 11/24/22 19:36, Pin-yen Lin wrote:
> Add lru_cache to speed up the running time of "Calculating
> dependencies".
>
> In a ChromeOS use case, this patch decreases the running time from
> 311s to 197s with almost no memory usage increase.
>
> Signed-off-by: Pin-yen Lin <treapking@chromium.org>
> ---
> lib/_emerge/depgraph.py | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
> index ce6cabcc1..9649bb2a8 100644
> --- a/lib/_emerge/depgraph.py
> +++ b/lib/_emerge/depgraph.py
> @@ -2240,6 +2240,7 @@ class depgraph:
>
> return None
>
> + @functools.lru_cache(maxsize=100)
> def _slot_operator_check_reverse_dependencies(
> self, existing_pkg, candidate_pkg, replacement_parent=None
> ):
Merged. Thank you!
https://gitweb.gentoo.org/proj/portage.git/commit/?id=0c42cc962e1926ecbdc83d903a2804f9e037f2a9
https://gitweb.gentoo.org/proj/portage.git/commit/?id=839ab46be1777e5886da28b98b53a462b992c5bf
--
Thanks,
Zac
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2022-11-28 15:47 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-11-25 3:36 [gentoo-portage-dev] [PATCH 1/2] Move and rename check_reverse_dependencies Pin-yen Lin
2022-11-25 3:36 ` [gentoo-portage-dev] [PATCH 2/2] Add caching to _slot_operator_check_reverse_dependencies Pin-yen Lin
2022-11-25 17:08 ` Matt Turner
2022-11-28 15:47 ` Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox