[gentoo-portage-dev] [PATCH] Support @profile package set for bug #532224

[Zac Medico <zmedico@gentoo.org>]

Add support for a new @profile set which allows the profile to pull
in additional packages that do not belong to the @system set.

The motivation to have @profile separate from @system is that
@system packages may have incomplete dependency specifications
(due to long-standing Gentoo policy), and incomplete dependency
specifications have deleterious effects on the ability of emerge
--jobs to parallelize builds. So, unlike @system, packages added to
@profile do not hurt emerge --jobs parallelization.

Packages are added to the @profile set in the same way that they are
added to the @system set, except that atoms in the @profile set are
not preceded with a '*' character. Also, the @profile package set
is only supported when 'profile-set' is listed in the layout.conf
profile-formats field of the containing repository.

X-Gentoo-Bug: 532224
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532224
---
 cnf/sets/portage.conf                              |   5 +-
 doc/config/sets.docbook                            |  19 +++-
 man/emerge.1                                       |  14 ++-
 man/portage.5                                      |  33 ++++--
 pym/_emerge/actions.py                             |  14 ++-
 pym/portage/_sets/ProfilePackageSet.py             |  33 ++++++
 pym/portage/_sets/__init__.py                      |   2 +-
 pym/portage/package/ebuild/config.py               |   4 +-
 pym/portage/repository/config.py                   |   2 +-
 .../tests/resolver/test_profile_package_set.py     | 123 +++++++++++++++++++++
 10 files changed, 225 insertions(+), 24 deletions(-)
 create mode 100644 pym/portage/_sets/ProfilePackageSet.py
 create mode 100644 pym/portage/tests/resolver/test_profile_package_set.py

diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
index 8ffcedc..cff7488 100644
--- a/cnf/sets/portage.conf
+++ b/cnf/sets/portage.conf
@@ -7,7 +7,10 @@
 # Not much that could be changed for world, so better leave it alone
 [world]
 class = portage.sets.base.DummyPackageSet
-packages = @selected @system
+packages = @profile @selected @system
+
+[profile]
+class = portage.sets.ProfilePackageSet.ProfilePackageSet
 
 [selected]
 class = portage.sets.files.WorldSelectedSet
diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook
index d3aa147..a557113 100644
--- a/doc/config/sets.docbook
+++ b/doc/config/sets.docbook
@@ -296,7 +296,23 @@
 			</para>
 			</sect3>
 		</sect2>
-		
+
+		<sect2 id='config-set-classes-ProfilePackageSet'>
+		<title>portage.sets.ProfilePackageSet.ProfilePackageSet</title>
+		<para>
+		This class implements the <parameter>profile</parameter> set, based on the
+		<filename>packages</filename> files in the profile.
+		There is no reason to use this in a user configuration as it is already
+		confgured by default and doesn't support any options.
+		</para>
+			<sect3>
+			<title>Single Set Configuration</title>
+			<para>
+			This class doesn't support any extra options.
+			</para>
+			</sect3>
+		</sect2>
+
 		<sect2 id='config-set-classes-SecuritySet' xreflabel='SecuritySet'>
 		<title>portage.sets.security.SecuritySet</title>
 		<para>
@@ -601,6 +617,7 @@
 	</para>
 	<itemizedlist>
 	<listitem><para><varname>world</varname>: uses <classname>DummySet</classname></para></listitem>
+	<listitem><para><varname>profile</varname>: uses <classname>ProfilePackageSet</classname></para></listitem>
 	<listitem><para><varname>selected</varname>: uses <classname>WorldSelectedSet</classname></para></listitem>
 	<listitem><para><varname>system</varname>: uses <classname>PackagesSystemSet</classname></para></listitem>
 	<listitem><para><varname>security</varname>: uses <classname>NewAffectedSet</classname> with default options</para></listitem>
diff --git a/man/emerge.1 b/man/emerge.1
index f64fd1b..b36f59c 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -61,15 +61,17 @@ would like to query the owners of one or more files or directories.
 .TP
 .BR set
 A \fIset\fR is a convenient shorthand for a large group of
-packages. Five sets are currently always available: \fBselected-packages\fR,
-\fBselected-sets\fR, \fBselected\fR, \fBsystem\fR and \fBworld\fR.
+packages. Six sets are currently always available: \fBselected-packages\fR,
+\fBselected-sets\fR, \fBselected\fR, \fBsystem\fR, \fBprofile\fR, and \fBworld\fR.
 \fBselected-packages\fR contains the user-selected "world" packages that
 are listed in \fB/var/lib/portage/world\fR, while \fBselected-sets\fR
 contains the nested sets that may be listed in \fB/var/lib/portage/world_sets\fR.
-\fBsystem\fR refers to a set of packages deemed necessary for your system
-to run properly. \fBselected\fR encompasses both the \fBselected-packages\fR
-and \fBselected-sets\fR sets, while \fBworld\fR encompasses the \fBselected\fR
-and \fBsystem\fR sets. [See
+\fBsystem\fR and \fBprofile\fR both refer to sets of packages deemed
+necessary for your system to run properly (the differences between these
+two sets are documented in \fBportage\fR(5)).
+\fBselected\fR encompasses both the \fBselected-packages\fR
+and \fBselected-sets\fR sets, while \fBworld\fR encompasses the \fBselected\fR,
+\fBsystem\fR and \fBprofile\fR sets. [See
 \fBFILES\fR below for more information.] Other sets can exist depending
 on the current configuration. The default set configuration is located
 in the \fB/usr/share/portage/config/sets\fR directory.
diff --git a/man/portage.5 b/man/portage.5
index 2fa699c..88cf3bb 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1,4 +1,4 @@
-.TH "PORTAGE" "5" "Feb 2014" "Portage VERSION" "Portage"
+.TH "PORTAGE" "5" "December 2014" "Portage VERSION" "Portage"
 .SH NAME
 portage \- the heart of Gentoo
 .SH "DESCRIPTION"
@@ -331,14 +331,25 @@ Special USE flags which may be needed when bootstrapping from stage1 to stage2.
 .PD 1
 .TP
 .BR packages
-Provides the list of packages that compose the special \fIsystem\fR set.
+Provides the list of packages that compose the \fI@system\fR and
+\fI@profile\fR  package sets. The motivation to have \fI@profile\fR
+separate from \fI@system\fR is that \fI@system\fR packages may have
+incomplete dependency specifications (due to long-standing Gentoo
+policy), and incomplete dependency specifications have deleterious
+effects on the ability of \fBemerge\fR to parallelize builds. So,
+unlike \fI@system\fR, packages included in \fI@profile\fR do not
+hurt \fBemerge\fR's ability to parallelize.
 
 .I Format:
 .nf
 \- comments begin with # (no inline comments)
 \- one DEPEND atom per line
-\- packages to be added to the system set begin with a *
-\- atoms without * only appear for legacy reasons
+\- packages to be added to the @system set begin with a *
+\- packages to be added to the @profile set do not begin with a *
+\- packages may only be added to the @profile set if the containing
+  repository's layout.conf has 'profile-set' listed in the
+  profile-formats field. Otherwise, packages that do not begin with
+  '*' will simply be ignored for legacy reasons
 .fi
 .I Note:
 In a cascading profile setup, you can remove packages in children
@@ -348,12 +359,14 @@ a '\-'.
 .I Example:
 .nf
 # i am a comment !
-# pull in a version of glibc less than 2.3
+# pull a version of glibc less than 2.3 into @system
 *<sys\-libs/glibc\-2.3
-# pull in any version of bash
+# pull any version of bash into @system
 *app\-shells/bash
-# pull in a version of readline earlier than 4.2
+# pull a version of readline earlier than 4.2 into @system
 *<sys\-libs/readline\-4.2
+# pull vim into @profile
+app-editors/vim
 .fi
 .TP
 .BR packages.build
@@ -1101,13 +1114,15 @@ The default setting for repoman's --echangelog option.
 The cache formats supported in the metadata tree.  There is the old "pms" format
 and the newer/faster "md5-dict" format.  Default is to detect dirs.
 .TP
-.BR profile\-formats " = [pms|portage-1|portage-2|profile-bashrcs]"
+.BR profile\-formats " = [pms|portage-1|portage-2|profile-bashrcs|profile-set]"
 Control functionality available to profiles in this repo such as which files
 may be dirs, or the syntax available in parent files.  Use "portage-2" if you're
 unsure.  The default is "portage-1-compat" mode which is meant to be compatible
 with old profiles, but is not allowed to be opted into directly.
 Setting profile-bashrcs will enable the per-profile bashrc mechanism
-\fBpackage.bashrc\fR.
+\fBpackage.bashrc\fR. Setting profile-set enables support for using the
+profile \fBpackages\fR file to add atoms to the @profile package set.
+See the profile \fBpackages\fR section for more information.
 .RE
 .RE
 
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index c7246a9..0ae2c16 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -650,7 +650,7 @@ def action_depclean(settings, trees, ldpath_mtimes,
 		return rval
 
 	set_atoms = {}
-	for k in ("system", "selected"):
+	for k in ("profile", "system", "selected"):
 		try:
 			set_atoms[k] = root_config.setconfig.getSetAtoms(k)
 		except portage.exception.PackageSetNotFound:
@@ -660,6 +660,8 @@ def action_depclean(settings, trees, ldpath_mtimes,
 	print("Packages installed:   " + str(len(vardb.cpv_all())))
 	print("Packages in world:    %d" % len(set_atoms["selected"]))
 	print("Packages in system:   %d" % len(set_atoms["system"]))
+	if set_atoms["profile"]:
+		print("Packages in profile:  %d" % len(set_atoms["profile"]))
 	print("Required packages:    "+str(req_pkg_count))
 	if "--pretend" in myopts:
 		print("Number to remove:     "+str(len(cleanlist)))
@@ -693,20 +695,24 @@ def calc_depclean(settings, trees, ldpath_mtimes,
 	system_set = psets["system"]
 
 	set_atoms = {}
-	for k in ("system", "selected"):
+	for k in ("profile", "system", "selected"):
 		try:
 			set_atoms[k] = root_config.setconfig.getSetAtoms(k)
 		except portage.exception.PackageSetNotFound:
 			# A nested set could not be resolved, so ignore nested sets.
 			set_atoms[k] = root_config.sets[k].getAtoms()
 
-	if not set_atoms["system"] or not set_atoms["selected"]:
+	if (not set_atoms["system"] or
+		not (set_atoms["selected"] or set_atoms["profile"])):
 
 		if not set_atoms["system"]:
 			writemsg_level("!!! You have no system list.\n",
 				level=logging.ERROR, noiselevel=-1)
 
-		if not set_atoms["selected"]:
+		# Skip this warning if @profile is non-empty, in order to
+		# support using @profile as an alternative to @selected
+		# for building a stage 4.
+		if not (set_atoms["selected"] or set_atoms["profile"]):
 			writemsg_level("!!! You have no world file.\n",
 					level=logging.WARNING, noiselevel=-1)
 
diff --git a/pym/portage/_sets/ProfilePackageSet.py b/pym/portage/_sets/ProfilePackageSet.py
new file mode 100644
index 0000000..c2f5fee
--- /dev/null
+++ b/pym/portage/_sets/ProfilePackageSet.py
@@ -0,0 +1,33 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage import os
+from portage.util import grabfile_package, stack_lists
+from portage._sets.base import PackageSet
+
+class ProfilePackageSet(PackageSet):
+	_operations = ["merge"]
+
+	def __init__(self, profiles, debug=False):
+		super(ProfilePackageSet, self).__init__()
+		self._profiles = profiles
+		if profiles:
+			desc_profile = profiles[-1]
+			if desc_profile.user_config and len(profiles) > 1:
+				desc_profile = profiles[-2]
+			description = desc_profile.location
+		else:
+			description = None
+		self.description = "Profile packages for profile %s" % description
+
+	def load(self):
+		self._setAtoms(x for x in stack_lists(
+			[grabfile_package(os.path.join(y.location, "packages"),
+			verify_eapi=True) for y in self._profiles
+			if "profile-set" in y.profile_formats],
+			incremental=1) if x[:1] != "*")
+
+	def singleBuilder(self, options, settings, trees):
+		return ProfilePackageSet(
+			settings._locations_manager.profiles_complex)
+	singleBuilder = classmethod(singleBuilder)
diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py
index a652227..d53387a 100644
--- a/pym/portage/_sets/__init__.py
+++ b/pym/portage/_sets/__init__.py
@@ -115,7 +115,7 @@ class SetConfig(object):
 		parser.remove_section("world")
 		parser.add_section("world")
 		parser.set("world", "class", "portage.sets.base.DummyPackageSet")
-		parser.set("world", "packages", "@selected @system")
+		parser.set("world", "packages", "@profile @selected @system")
 
 		parser.remove_section("selected")
 		parser.add_section("selected")
diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
index 59e239b..1a0377e 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -512,7 +512,6 @@ class config(object):
 				if v is not None:
 					portdir_sync = v
 
-			known_repos = frozenset(known_repos)
 			self["PORTDIR"] = portdir
 			self["PORTDIR_OVERLAY"] = portdir_overlay
 			if portdir_sync:
@@ -523,6 +522,9 @@ class config(object):
 			else:
 				self.repositories = repositories
 
+			known_repos.extend(repo.location for repo in self.repositories)
+			known_repos = frozenset(known_repos)
+
 			self['PORTAGE_REPOSITORIES'] = self.repositories.config_string()
 			self.backup_changes('PORTAGE_REPOSITORIES')
 
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index f45684b..9096d73 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -41,7 +41,7 @@ if sys.hexversion >= 0x3000000:
 _invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]')
 
 _valid_profile_formats = frozenset(
-	['pms', 'portage-1', 'portage-2', 'profile-bashrcs'])
+	['pms', 'portage-1', 'portage-2', 'profile-bashrcs', 'profile-set'])
 
 _portage1_profiles_allow_directories = frozenset(
 	["portage-1-compat", "portage-1", 'portage-2'])
diff --git a/pym/portage/tests/resolver/test_profile_package_set.py b/pym/portage/tests/resolver/test_profile_package_set.py
new file mode 100644
index 0000000..88a2a82
--- /dev/null
+++ b/pym/portage/tests/resolver/test_profile_package_set.py
@@ -0,0 +1,123 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import unicode_literals
+
+import io
+
+from portage import os, _encodings
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+	ResolverPlayground, ResolverPlaygroundTestCase)
+from portage.util import ensure_dirs
+
+class ProfilePackageSetTestCase(TestCase):
+
+	def testProfilePackageSet(self):
+
+		repo_configs = {
+			"test_repo": {
+				"layout.conf": ("profile-formats = profile-set",),
+			}
+		}
+
+		profiles = (
+			(
+				'default/linux',
+				{
+					"eapi": ("5",),
+					"packages": (
+						"*sys-libs/A",
+						"app-misc/A",
+						"app-misc/B",
+						"app-misc/C",
+					),
+				}
+			),
+			(
+				'default/linux/x86',
+				{
+					"eapi": ("5",),
+					"packages": (
+						"-app-misc/B",
+					),
+					"parent": ("..",)
+				}
+			),
+		)
+
+		ebuilds = {
+			"sys-libs/A-1": {
+				"EAPI": "5",
+			},
+			"app-misc/A-1": {
+				"EAPI": "5",
+			},
+			"app-misc/B-1": {
+				"EAPI": "5",
+			},
+			"app-misc/C-1": {
+				"EAPI": "5",
+			},
+		}
+
+		installed = {
+			"sys-libs/A-1": {
+				"EAPI": "5",
+			},
+			"app-misc/A-1": {
+				"EAPI": "5",
+			},
+			"app-misc/B-1": {
+				"EAPI": "5",
+			},
+			"app-misc/C-1": {
+				"EAPI": "5",
+			},
+		}
+
+		test_cases = (
+
+			ResolverPlaygroundTestCase(
+				["@world"],
+				options={"--update": True, "--deep": True},
+				mergelist = [],
+				success = True,
+			),
+
+			ResolverPlaygroundTestCase(
+				[],
+				options={"--depclean": True},
+				success=True,
+				cleanlist=["app-misc/B-1"]
+			),
+
+		)
+
+		playground = ResolverPlayground(debug=False, ebuilds=ebuilds,
+			installed=installed, repo_configs=repo_configs)
+		try:
+			repo_dir = (playground.settings.repositories.
+				get_location_for_name("test_repo"))
+			profile_root = os.path.join(repo_dir, "profiles")
+
+			for p, data in profiles:
+				prof_path = os.path.join(profile_root, p)
+				ensure_dirs(prof_path)
+				for k, v in data.items():
+					with io.open(os.path.join(prof_path, k), mode="w",
+						encoding=_encodings["repo.content"]) as f:
+						for line in v:
+							f.write("%s\n" % line)
+
+			# The config must be reloaded in order to account
+			# for the above profile customizations.
+			playground.reload_config()
+
+			for test_case in test_cases:
+				playground.run_TestCase(test_case)
+				self.assertEqual(test_case.test_success, True,
+					test_case.fail_msg)
+
+		finally:
+			playground.cleanup()
-- 
2.0.4