[gentoo-portage-dev] [PATCH] depgraph: fix bug #526160

[gentoo-portage-dev] [PATCH] depgraph: fix bug #526160

[zmedico]

From: Zac Medico <zmedico@gentoo.org>

This fixes _dep_check_composite_db to mask packages that aren't the
highest visible match, but only if an update is desirable. This causes
desirable updates to get pulled in for cases like bug #526160. The
included unit test simulates the virtual/pypy update that triggered
the bug.

X-Gentoo-Bug: 526160
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=526160
---
 pym/_emerge/depgraph.py                         | 21 ++++++++
 pym/portage/tests/resolver/test_virtual_slot.py | 66 +++++++++++++++++++++++++
 2 files changed, 87 insertions(+)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 5180db5..7dc37f7 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -527,6 +527,8 @@ class depgraph(object):
 		self._event_loop = (portage._internal_caller and
 			global_event_loop() or EventLoop(main=False))
 
+		self._select_atoms_parent = None
+
 		self.query = UserQuery(myopts).query
 
 	def _load_vdb(self):
@@ -4062,11 +4064,13 @@ class depgraph(object):
 			self._dynamic_config._autounmask = False
 			# backup state for restoration, in case of recursive
 			# calls to this method
+			backup_parent = self._select_atoms_parent
 			backup_state = mytrees.copy()
 			try:
 				# clear state from previous call, in case this
 				# call is recursive (we have a backup, that we
 				# will use to restore it later)
+				self._select_atoms_parent = None
 				mytrees.pop("pkg_use_enabled", None)
 				mytrees.pop("parent", None)
 				mytrees.pop("atom_graph", None)
@@ -4074,6 +4078,7 @@ class depgraph(object):
 
 				mytrees["pkg_use_enabled"] = self._pkg_use_enabled
 				if parent is not None:
+					self._select_atoms_parent = parent
 					mytrees["parent"] = parent
 					mytrees["atom_graph"] = atom_graph
 				if priority is not None:
@@ -4085,6 +4090,7 @@ class depgraph(object):
 			finally:
 				# restore state
 				self._dynamic_config._autounmask = _autounmask_backup
+				self._select_atoms_parent = backup_parent
 				mytrees.pop("pkg_use_enabled", None)
 				mytrees.pop("parent", None)
 				mytrees.pop("atom_graph", None)
@@ -8529,6 +8535,21 @@ class _dep_check_composite_db(dbapi):
 				elif not self._depgraph._equiv_ebuild_visible(pkg):
 					return False
 
+		want_update = False
+		if self._depgraph._select_atoms_parent is not None:
+			want_update = \
+				self._depgraph._want_update_pkg(
+				self._depgraph._select_atoms_parent, pkg)
+
+		if want_update:
+			for new_child in self._depgraph._iter_similar_available(
+				pkg, next(iter(atom_set))):
+				if not self._depgraph._virt_deps_visible(
+					new_child, ignore_use=True):
+					continue
+				if pkg < new_child:
+					return False
+
 		in_graph = next(self._depgraph._dynamic_config._package_tracker.match(
 			self._root, pkg.slot_atom, installed=False), None)
 
diff --git a/pym/portage/tests/resolver/test_virtual_slot.py b/pym/portage/tests/resolver/test_virtual_slot.py
index 1b19d77..2e5ca7f 100644
--- a/pym/portage/tests/resolver/test_virtual_slot.py
+++ b/pym/portage/tests/resolver/test_virtual_slot.py
@@ -92,6 +92,72 @@ class VirtualSlotResolverTestCase(TestCase):
 		finally:
 			playground.cleanup()
 
+	def testVirtualSubslotUpdate(self):
+
+		ebuilds = {
+			"virtual/pypy-2.3.1" : {
+				"EAPI": "5",
+				"SLOT": "0/2.3",
+				"RDEPEND": "|| ( >=dev-python/pypy-2.3.1:0/2.3 >=dev-python/pypy-bin-2.3.1:0/2.3 ) "
+			},
+			"virtual/pypy-2.4.0" : {
+				"EAPI": "5",
+				"SLOT": "0/2.4",
+				"RDEPEND": "|| ( >=dev-python/pypy-2.4.0:0/2.4 >=dev-python/pypy-bin-2.4.0:0/2.4 ) "
+			},
+			"dev-python/pypy-2.3.1": {
+				"EAPI": "5",
+				"SLOT": "0/2.3"
+			},
+			"dev-python/pypy-2.4.0": {
+				"EAPI": "5",
+				"SLOT": "0/2.4"
+			},
+			"dev-python/pygments-1.6_p20140324-r1": {
+				"EAPI": "5",
+				"DEPEND": "virtual/pypy:="
+			}
+		}
+
+		installed = {
+			"virtual/pypy-2.3.1" : {
+				"EAPI": "5",
+				"SLOT": "0/2.3",
+				"RDEPEND": "|| ( >=dev-python/pypy-2.3.1:0/2.3 >=dev-python/pypy-bin-2.3.1:0/2.3 ) "
+			},
+			"dev-python/pypy-2.3.1": {
+				"EAPI": "5",
+				"SLOT": "0/2.3"
+			},
+			"dev-python/pygments-1.6_p20140324-r1": {
+				"EAPI": "5",
+				"DEPEND": "virtual/pypy:0/2.3=",
+				"RDEPEND": "virtual/pypy:0/2.3=",
+			}
+		}
+
+		world = ["dev-python/pygments"]
+
+		test_cases = (
+			# bug 526160 - test for missed pypy sub-slot update
+			ResolverPlaygroundTestCase(
+				["@world"],
+				options = {"--update": True, "--deep": True},
+				success=True,
+				mergelist = ['dev-python/pypy-2.4.0',
+					'virtual/pypy-2.4.0',
+					'dev-python/pygments-1.6_p20140324-r1']),
+		)
+
+		playground = ResolverPlayground(debug=False, ebuilds=ebuilds,
+			installed=installed, world=world)
+		try:
+			for test_case in test_cases:
+				playground.run_TestCase(test_case)
+				self.assertEqual(test_case.test_success, True, test_case.fail_msg)
+		finally:
+			playground.cleanup()
+
 	def testVirtualSlotDepclean(self):
 
 		ebuilds = {
-- 
2.0.4

[gentoo-portage-dev] [PATCH] depgraph: fix bug #526160

[Zac Medico]

This fixes _dep_check_composite_db to mask packages that aren't the
highest visible match, but only if an update is desirable. This causes
desirable updates to get pulled in for cases like bug #526160. The
included unit test simulates the virtual/pypy update that triggered
the bug.

X-Gentoo-Bug: 526160
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=526160
---
This updated patch fixes the new logic so that it only applies to
virtual packages, since that's the only case where it is currently
needed.

 pym/_emerge/depgraph.py                         | 24 +++++++++
 pym/portage/tests/resolver/test_virtual_slot.py | 66 +++++++++++++++++++++++++
 2 files changed, 90 insertions(+)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 5180db5..78b9236 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -527,6 +527,8 @@ class depgraph(object):
 		self._event_loop = (portage._internal_caller and
 			global_event_loop() or EventLoop(main=False))
 
+		self._select_atoms_parent = None
+
 		self.query = UserQuery(myopts).query
 
 	def _load_vdb(self):
@@ -4062,11 +4064,13 @@ class depgraph(object):
 			self._dynamic_config._autounmask = False
 			# backup state for restoration, in case of recursive
 			# calls to this method
+			backup_parent = self._select_atoms_parent
 			backup_state = mytrees.copy()
 			try:
 				# clear state from previous call, in case this
 				# call is recursive (we have a backup, that we
 				# will use to restore it later)
+				self._select_atoms_parent = None
 				mytrees.pop("pkg_use_enabled", None)
 				mytrees.pop("parent", None)
 				mytrees.pop("atom_graph", None)
@@ -4074,6 +4078,7 @@ class depgraph(object):
 
 				mytrees["pkg_use_enabled"] = self._pkg_use_enabled
 				if parent is not None:
+					self._select_atoms_parent = parent
 					mytrees["parent"] = parent
 					mytrees["atom_graph"] = atom_graph
 				if priority is not None:
@@ -4085,6 +4090,7 @@ class depgraph(object):
 			finally:
 				# restore state
 				self._dynamic_config._autounmask = _autounmask_backup
+				self._select_atoms_parent = backup_parent
 				mytrees.pop("pkg_use_enabled", None)
 				mytrees.pop("parent", None)
 				mytrees.pop("atom_graph", None)
@@ -8529,6 +8535,24 @@ class _dep_check_composite_db(dbapi):
 				elif not self._depgraph._equiv_ebuild_visible(pkg):
 					return False
 
+		if pkg.cp.startswith("virtual/"):
+			# Force virtual updates to be pulled in when appropriate
+			# for bug #526160.
+			want_update = False
+			if self._depgraph._select_atoms_parent is not None:
+				want_update = \
+					self._depgraph._want_update_pkg(
+					self._depgraph._select_atoms_parent, pkg)
+
+			if want_update:
+				for new_child in self._depgraph._iter_similar_available(
+					pkg, next(iter(atom_set))):
+					if not self._depgraph._virt_deps_visible(
+						new_child, ignore_use=True):
+						continue
+					if pkg < new_child:
+						return False
+
 		in_graph = next(self._depgraph._dynamic_config._package_tracker.match(
 			self._root, pkg.slot_atom, installed=False), None)
 
diff --git a/pym/portage/tests/resolver/test_virtual_slot.py b/pym/portage/tests/resolver/test_virtual_slot.py
index 1b19d77..2e5ca7f 100644
--- a/pym/portage/tests/resolver/test_virtual_slot.py
+++ b/pym/portage/tests/resolver/test_virtual_slot.py
@@ -92,6 +92,72 @@ class VirtualSlotResolverTestCase(TestCase):
 		finally:
 			playground.cleanup()
 
+	def testVirtualSubslotUpdate(self):
+
+		ebuilds = {
+			"virtual/pypy-2.3.1" : {
+				"EAPI": "5",
+				"SLOT": "0/2.3",
+				"RDEPEND": "|| ( >=dev-python/pypy-2.3.1:0/2.3 >=dev-python/pypy-bin-2.3.1:0/2.3 ) "
+			},
+			"virtual/pypy-2.4.0" : {
+				"EAPI": "5",
+				"SLOT": "0/2.4",
+				"RDEPEND": "|| ( >=dev-python/pypy-2.4.0:0/2.4 >=dev-python/pypy-bin-2.4.0:0/2.4 ) "
+			},
+			"dev-python/pypy-2.3.1": {
+				"EAPI": "5",
+				"SLOT": "0/2.3"
+			},
+			"dev-python/pypy-2.4.0": {
+				"EAPI": "5",
+				"SLOT": "0/2.4"
+			},
+			"dev-python/pygments-1.6_p20140324-r1": {
+				"EAPI": "5",
+				"DEPEND": "virtual/pypy:="
+			}
+		}
+
+		installed = {
+			"virtual/pypy-2.3.1" : {
+				"EAPI": "5",
+				"SLOT": "0/2.3",
+				"RDEPEND": "|| ( >=dev-python/pypy-2.3.1:0/2.3 >=dev-python/pypy-bin-2.3.1:0/2.3 ) "
+			},
+			"dev-python/pypy-2.3.1": {
+				"EAPI": "5",
+				"SLOT": "0/2.3"
+			},
+			"dev-python/pygments-1.6_p20140324-r1": {
+				"EAPI": "5",
+				"DEPEND": "virtual/pypy:0/2.3=",
+				"RDEPEND": "virtual/pypy:0/2.3=",
+			}
+		}
+
+		world = ["dev-python/pygments"]
+
+		test_cases = (
+			# bug 526160 - test for missed pypy sub-slot update
+			ResolverPlaygroundTestCase(
+				["@world"],
+				options = {"--update": True, "--deep": True},
+				success=True,
+				mergelist = ['dev-python/pypy-2.4.0',
+					'virtual/pypy-2.4.0',
+					'dev-python/pygments-1.6_p20140324-r1']),
+		)
+
+		playground = ResolverPlayground(debug=False, ebuilds=ebuilds,
+			installed=installed, world=world)
+		try:
+			for test_case in test_cases:
+				playground.run_TestCase(test_case)
+				self.assertEqual(test_case.test_success, True, test_case.fail_msg)
+		finally:
+			playground.cleanup()
+
 	def testVirtualSlotDepclean(self):
 
 		ebuilds = {
-- 
2.0.4

Re: [gentoo-portage-dev] [PATCH] depgraph: fix bug #526160

[Alexander Berntsen]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

On 27/10/14 08:33, Zac Medico wrote:
> +					if not self._depgraph._virt_deps_visible( +						new_child,
> ignore_use=True): +						continue +					if pkg < new_child: +
> return False
Seems needlessly bit obscure when you could just "if
self.depgraph._virt_deps_visible(new_child, ignore_use=True) and pkg <
new_child" or some variant thereof. But it's a minor qualm.

Otherwise LGTM, go ahead & push.

- -- 
Alexander
bernalex@gentoo.org
https://secure.plaimi.net/~alexander
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/

iF4EAREIAAYFAlROALYACgkQRtClrXBQc7WY4QEAgp2lH+WOHmAcGBoxVIfr0cGc
mYc5xIpc17eu4OxvPUIBAIaMsRyieA9OZSq7mUQL27of9zrirjH0PGPMjSuxZQxB
=1fXi
-----END PGP SIGNATURE-----

Re: [gentoo-portage-dev] [PATCH] depgraph: fix bug #526160

[Alexander Berntsen]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

On 27/10/14 09:22, Alexander Berntsen wrote:
> On 27/10/14 08:33, Zac Medico wrote:
>> +					if not self._depgraph._virt_deps_visible( +						new_child, 
>> ignore_use=True): +						continue +					if pkg < new_child: + 
>> return False
> Seems needlessly bit obscure when you could just "if 
> self.depgraph._virt_deps_visible(new_child, ignore_use=True) and pkg 
> < new_child" or some variant thereof. But it's a minor qualm.
> 
> Otherwise LGTM, go ahead & push.
Thunderbird fucked up wrapping. I am referring to this part:

On 27/10/14 08:33, Zac Medico wrote:> +				for new_child in self._depgraph._iter_similar_available(
> +					pkg, next(iter(atom_set))):
> +					if not self._depgraph._virt_deps_visible(
> +						new_child, ignore_use=True):
> +						continue
> +					if pkg < new_child:
> +						return False
- -- 
Alexander
bernalex@gentoo.org
https://secure.plaimi.net/~alexander
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/

iF4EAREIAAYFAlROAP4ACgkQRtClrXBQc7XIhAD8CStcQaeCbFzibcHf0CAHN/71
xyUKhSWcoky/LUt0gB8A/2WC8wwngghQjbhwEJ0WFryygLmvvTdhGBl2K7/9Xq7Q
=Diyd
-----END PGP SIGNATURE-----

Re: [gentoo-portage-dev] [PATCH] depgraph: fix bug #526160

[Alexander Berntsen]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

I give up. You'll figure it out.
- -- 
Alexander
bernalex@gentoo.org
https://secure.plaimi.net/~alexander
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/

iF4EAREIAAYFAlROARcACgkQRtClrXBQc7X90AD/XqL9YziqEAQEILESp48jD+HJ
Px+/5flPE74lV/oZ25wA/08ZujUDso/kNB3Q8ApydhX8hoOIr0uL9G+VFZ0MVQdq
=rlDo
-----END PGP SIGNATURE-----

Re: [gentoo-portage-dev] [PATCH] depgraph: fix bug #526160

[Zac Medico]

On 10/27/2014 01:22 AM, Alexander Berntsen wrote:
> On 27/10/14 08:33, Zac Medico wrote:
>> +					if not self._depgraph._virt_deps_visible( +						new_child,
>> ignore_use=True): +						continue +					if pkg < new_child: +
>> return False
> Seems needlessly bit obscure when you could just "if
> self.depgraph._virt_deps_visible(new_child, ignore_use=True) and pkg <
> new_child" or some variant thereof. But it's a minor qualm.

I prefer not to group them into a single expression because they are
separate heuristics. At some point in the future, we may find that we
need to adjust one or both heuristics, add more heuristics, or remove
some heuristics. Keeping the heuristics as separate expressions makes it
easier to do such modifications. It also provides emphasis to the reader
that these are separate heuristics.

> Otherwise LGTM, go ahead & push.

Thanks, it's pushed:

https://github.com/gentoo/portage/commit/d3be49fe6827aa1974856dffe6d5a1aca80a7bed
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] depgraph: fix bug #526160

[Brian Dolbec]

On Sun, 26 Oct 2014 18:05:54 -0700
zmedico@gentoo.org wrote:

> From: Zac Medico <zmedico@gentoo.org>
> 
> This fixes _dep_check_composite_db to mask packages that aren't the
> highest visible match, but only if an update is desirable. This causes
> desirable updates to get pulled in for cases like bug #526160. The
> included unit test simulates the virtual/pypy update that triggered
> the bug.
> 
> X-Gentoo-Bug: 526160
> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=526160
> ---
>  pym/_emerge/depgraph.py                         | 21 ++++++++
>  pym/portage/tests/resolver/test_virtual_slot.py | 66
> +++++++++++++++++++++++++ 2 files changed, 87 insertions(+)
> 

Looks good, merge away


-- 
Brian Dolbec <dolsen>