[gentoo-portage-dev] [PATCH 0/3] thin manifest support

[gentoo-portage-dev] [PATCH 0/3] thin manifest support

[Brian Harring]

This patch series adds thin manifest support.  While mini-manifest feature
exists in funtoo, that functionality is a global toggle- either all are mini,
or none.

This series makes thin controllable per repository via layout.conf; if
the file exists and has 'thin-manifest = true' in it, the manifest rules
switch to thin- *just* for that repository.

This should allow git vcs overlays to use thin, while the mainline rsync
repo continues to use full manifest2.

Finally, some of the api work here is kind of ugly.  I went for consistancy
with the surrounding code; that said, refactoring of the RepoConfig api's
likely would be beneficial (that's outside the scope of my intent however).

Brian Harring (3):
  Bind all manifest access through repoconfigs
  add thin manifest support to the Manifest class
  add layout.conf awareness of thin-manifests

 bin/ebuild                                |    5 +-
 bin/repoman                               |    8 +-
 pym/_emerge/EbuildFetcher.py              |    6 +-
 pym/_emerge/search.py                     |    4 +-
 pym/portage/dbapi/porttree.py             |    8 +-
 pym/portage/manifest.py                   |  149 ++++++++++++++++++----------
 pym/portage/package/ebuild/digestcheck.py |    6 +-
 pym/portage/package/ebuild/digestgen.py   |    3 +-
 pym/portage/package/ebuild/doebuild.py    |    4 +-
 pym/portage/package/ebuild/fetch.py       |    3 +-
 pym/portage/repository/config.py          |   14 +++-
 11 files changed, 142 insertions(+), 68 deletions(-)

-- 
1.7.6.1

[gentoo-portage-dev] [PATCH 1/3] Bind all manifest access through repoconfigs

[Brian Harring]

This enables controling the behaviour (creation and validation) per
repo, and while mildly ugly, refactors in the right direction.
---
 bin/ebuild                                |    5 +++--
 bin/repoman                               |    8 ++++++--
 pym/_emerge/EbuildFetcher.py              |    6 ++++--
 pym/_emerge/search.py                     |    4 +++-
 pym/portage/dbapi/porttree.py             |    8 ++++++--
 pym/portage/package/ebuild/digestcheck.py |    4 +++-
 pym/portage/package/ebuild/digestgen.py   |    3 ++-
 pym/portage/package/ebuild/doebuild.py    |    4 +++-
 pym/portage/package/ebuild/fetch.py       |    3 ++-
 pym/portage/repository/config.py          |    7 ++++++-
 10 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/bin/ebuild b/bin/ebuild
index db7e5e3..92105bb 100755
--- a/bin/ebuild
+++ b/bin/ebuild
@@ -200,8 +200,9 @@ def discard_digests(myebuild, mysettings, mydbapi):
 		portage._doebuild_manifest_exempt_depend += 1
 		pkgdir = os.path.dirname(myebuild)
 		fetchlist_dict = portage.FetchlistDict(pkgdir, mysettings, mydbapi)
-		from portage.manifest import Manifest
-		mf = Manifest(pkgdir, mysettings["DISTDIR"],
+		mf = mysettings.repositories.get_repo_for_location(
+			os.path.dirname(os.path.dirname(pkgdir)))
+		mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"],
 			fetchlist_dict=fetchlist_dict, manifest1_compat=False)
 		mf.create(requiredDistfiles=None,
 			assumeDistHashesSometimes=True, assumeDistHashesAlways=True)
diff --git a/bin/repoman b/bin/repoman
index 6ec84e5..5f81a0f 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -1104,7 +1104,9 @@ for x in scanlist:
 			portage._doebuild_manifest_exempt_depend += 1
 			try:
 				distdir = repoman_settings['DISTDIR']
-				mf = portage.manifest.Manifest(checkdir, distdir,
+				mf = repoman_settings.repositories.get_repo_for_location(
+					os.path.dirname(os.path.dirname(checkdir)))
+				mf = mf.load_manifest(checkdir, distdir,
 					fetchlist_dict=fetchlist_dict)
 				mf.create(requiredDistfiles=None,
 					assumeDistHashesAlways=True)
@@ -1314,7 +1316,9 @@ for x in scanlist:
 				raise
 			continue
 
-	mf = Manifest(checkdir, repoman_settings["DISTDIR"])
+	mf = repoman_settings.repositories.get_repo_for_location(
+		os.path.dirname(os.path.dirname(checkdir)))
+	mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
 	mydigests=mf.getTypeDigests("DIST")
 
 	fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index feb68d0..4389f84 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -206,8 +206,10 @@ class EbuildFetcher(SpawnProcess):
 	def _get_digests(self):
 		if self._digests is not None:
 			return self._digests
-		self._digests = portage.Manifest(os.path.dirname(
-			self._get_ebuild_path()), None).getTypeDigests("DIST")
+		pkgdir = os.path.dirname(self._get_ebuild_path())
+		mf = self.pkg.root_config.settings.repositories.get_repo_for_location(
+			os.path.dirname(os.path.dirname(pkgdir)))
+		self._digests = mf.load_manifest(pkgdir, None).getTypeDigests("DIST")
 		return self._digests
 
 	def _get_uri_map(self):
diff --git a/pym/_emerge/search.py b/pym/_emerge/search.py
index 096b384..4a4183d 100644
--- a/pym/_emerge/search.py
+++ b/pym/_emerge/search.py
@@ -317,7 +317,9 @@ class search(object):
 							installed=False, metadata=metadata,
 							root_config=self.root_config, type_name="ebuild")
 						pkgdir = os.path.dirname(myebuild)
-						mf = Manifest(
+						mf = self.settings.repositories.get_repo_for_location(
+							os.path.dirname(os.path.dirname(pkgdir)))
+						mf = mf.load_manifest(
 							pkgdir, self.settings["DISTDIR"])
 						try:
 							uri_map = _parse_uri_map(mycpv, metadata,
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..1172798 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -576,7 +576,9 @@ class portdbapi(dbapi):
 		if myebuild is None:
 			raise AssertionError(_("ebuild not found for '%s'") % mypkg)
 		pkgdir = os.path.dirname(myebuild)
-		mf = Manifest(pkgdir, self.settings["DISTDIR"])
+		mf = self.repositories.get_repo_for_location(
+			os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
+				pkgdir, self.settings["DISTDIR"])
 		checksums = mf.getDigests()
 		if not checksums:
 			if debug: 
@@ -644,7 +646,9 @@ class portdbapi(dbapi):
 		if myebuild is None:
 			raise AssertionError(_("ebuild not found for '%s'") % mypkg)
 		pkgdir = os.path.dirname(myebuild)
-		mf = Manifest(pkgdir, self.settings["DISTDIR"])
+		mf = self.repositories.get_repo_for_location(
+			os.path.dirname(os.path.dirname(pkgdir)))
+		mf = mf.load_manifest(pkgdir, self.settings["DISTDIR"])
 		mysums = mf.getDigests()
 
 		failures = {}
diff --git a/pym/portage/package/ebuild/digestcheck.py b/pym/portage/package/ebuild/digestcheck.py
index 1e34b14..d184301 100644
--- a/pym/portage/package/ebuild/digestcheck.py
+++ b/pym/portage/package/ebuild/digestcheck.py
@@ -41,7 +41,9 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
 		else:
 			return 1
 	if mf is None:
-		mf = Manifest(pkgdir, mysettings["DISTDIR"])
+		mf = mysettings.repositories.get_repo_for_location(
+			os.path.dirname(os.path.dirname(pkgdir)))
+		mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"])
 	manifest_empty = True
 	for d in mf.fhashdict.values():
 		if d:
diff --git a/pym/portage/package/ebuild/digestgen.py b/pym/portage/package/ebuild/digestgen.py
index eb7210e..6051512 100644
--- a/pym/portage/package/ebuild/digestgen.py
+++ b/pym/portage/package/ebuild/digestgen.py
@@ -53,7 +53,8 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None):
 				return 0
 		mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
 		manifest1_compat = False
-		mf = Manifest(mysettings["O"], mysettings["DISTDIR"],
+		mf = mysettings.repositories.get_repo_for_location(mytree)
+		mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"],
 			fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat)
 		# Don't require all hashes since that can trigger excessive
 		# fetches when sufficient digests already exist.  To ease transition
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index 7b3561e..7c276ba 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -508,7 +508,9 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
 				out.eerror(_("Manifest not found for '%s'") % (myebuild,))
 				_doebuild_broken_ebuilds.add(myebuild)
 				return 1
-			mf = Manifest(pkgdir, mysettings["DISTDIR"])
+			mf = mysettings.repositories.get_repo_for_location(
+				os.path.dirname(os.path.dirname(pkgdir)))
+			mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"])
 
 		else:
 			mf = _doebuild_manifest_cache
diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
index 5cbbf87..11c4c01 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -356,7 +356,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 		allow_missing_digests = True
 	pkgdir = mysettings.get("O")
 	if digests is None and not (pkgdir is None or skip_manifest):
-		mydigests = Manifest(
+		mydigests = mysettings.repositories.get_repo_for_location(
+			os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
 			pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST")
 	elif digests is None or skip_manifest:
 		# no digests because fetch was not called for a specific package
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index a12bd7b..014ec25 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -21,6 +21,7 @@ from portage.util import normalize_path, writemsg, writemsg_level, shlex_split
 from portage.localization import _
 from portage import _unicode_encode
 from portage import _encodings
+from portage import manifest
 
 _repo_name_sub_re = re.compile(r'[^\w-]')
 
@@ -41,7 +42,7 @@ class RepoConfig(object):
 	"""Stores config of one repository"""
 
 	__slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo',
-		'missing_repo_name', 'name', 'priority', 'sync', 'format']
+		'missing_repo_name', 'name', 'priority', 'sync', 'format', 'load_manifest']
 
 	def __init__(self, name, repo_opts):
 		"""Build a RepoConfig with options in repo_opts
@@ -110,6 +111,7 @@ class RepoConfig(object):
 			missing = False
 		self.name = name
 		self.missing_repo_name = missing
+		self.load_manifest = manifest.Manifest
 
 	def update(self, new_repo):
 		"""Update repository with options in another RepoConfig"""
@@ -498,6 +500,9 @@ class RepoConfigLoader(object):
 			return None
 		return self.treemap[repo_name]
 
+	def get_repo_for_location(self, location):
+		return self.prepos[self.get_name_for_location(location)]
+
 	def __getitem__(self, repo_name):
 		return self.prepos[repo_name]
 
-- 
1.7.6.1

[gentoo-portage-dev] [PATCH 2/3] add thin manifest support to the Manifest class

[Brian Harring]

'thin' is just distfiles.  This is primarily useful when the ebuild
lives in a vcs- git for example, which already has it's own checksums
to rely on.
---
 pym/portage/manifest.py                   |  149 ++++++++++++++++++----------
 pym/portage/package/ebuild/digestcheck.py |    2 +-
 2 files changed, 97 insertions(+), 54 deletions(-)

diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 13efab7..ef1a552 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -49,6 +49,12 @@ def guessManifestFileType(filename):
 	else:
 		return "DIST"
 
+def guessThinManifestFileType(filename):
+	type = guessManifestFileType(filename)
+	if type != "DIST":
+		return None
+	return "DIST"
+
 def parseManifest2(mysplit):
 	myentry = None
 	if len(mysplit) > 4 and mysplit[0] in portage.const.MANIFEST2_IDENTIFIERS:
@@ -93,12 +99,14 @@ class Manifest2Entry(ManifestEntry):
 class Manifest(object):
 	parsers = (parseManifest2,)
 	def __init__(self, pkgdir, distdir, fetchlist_dict=None,
-		manifest1_compat=False, from_scratch=False):
+		manifest1_compat=False, from_scratch=False, thin=False):
 		""" create new Manifest instance for package in pkgdir
 		    and add compability entries for old portage versions if manifest1_compat == True.
 		    Do not parse Manifest file if from_scratch == True (only for internal use)
 			The fetchlist_dict parameter is required only for generation of
-			a Manifest (not needed for parsing and checking sums)."""
+			a Manifest (not needed for parsing and checking sums).
+			If thin is specified, then the manifest carries only info for
+			distfiles."""
 		self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
 		self.fhashdict = {}
 		self.hashes = set()
@@ -120,7 +128,11 @@ class Manifest(object):
 		else:
 			self.fetchlist_dict = {}
 		self.distdir = distdir
-		self.guessType = guessManifestFileType
+		self.thin = thin
+		if thin:
+			self.guessType = guessThinManifestFileType
+		else:
+			self.guessType = guessManifestFileType
 
 	def getFullname(self):
 		""" Returns the absolute path to the Manifest file for this instance """
@@ -313,64 +325,20 @@ class Manifest(object):
 			distfilehashes = {}
 		self.__init__(self.pkgdir, self.distdir,
 			fetchlist_dict=self.fetchlist_dict, from_scratch=True,
-			manifest1_compat=False)
-		cpvlist = []
+			manifest1_compat=False, thin=self.thin)
 		pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
 		cat = self._pkgdir_category()
 
 		pkgdir = self.pkgdir
+		if self.thin:
+			cpvlist = self._update_thin_pkgdir(cat, pn, pkgdir)
+		else:
+			cpvlist = self._update_thick_pkgdir(cat, pn, pkgdir)
 
-		for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
-			break
-		for f in pkgdir_files:
-			try:
-				f = _unicode_decode(f,
-					encoding=_encodings['fs'], errors='strict')
-			except UnicodeDecodeError:
-				continue
-			if f[:1] == ".":
-				continue
-			pf = None
-			if f[-7:] == '.ebuild':
-				pf = f[:-7]
-			if pf is not None:
-				mytype = "EBUILD"
-				ps = portage.versions._pkgsplit(pf)
-				cpv = "%s/%s" % (cat, pf)
-				if not ps:
-					raise PortagePackageException(
-						_("Invalid package name: '%s'") % cpv)
-				if ps[0] != pn:
-					raise PortagePackageException(
-						_("Package name does not "
-						"match directory name: '%s'") % cpv)
-				cpvlist.append(cpv)
-			elif manifest2MiscfileFilter(f):
-				mytype = "MISC"
-			else:
-				continue
-			self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes)
-		recursive_files = []
-
-		pkgdir = self.pkgdir
-		cut_len = len(os.path.join(pkgdir, "files") + os.sep)
-		for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
-			for f in files:
-				try:
-					f = _unicode_decode(f,
-						encoding=_encodings['fs'], errors='strict')
-				except UnicodeDecodeError:
-					continue
-				full_path = os.path.join(parentdir, f)
-				recursive_files.append(full_path[cut_len:])
-		for f in recursive_files:
-			if not manifest2AuxfileFilter(f):
-				continue
-			self.fhashdict["AUX"][f] = perform_multiple_checksums(
-				os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes)
 		distlist = set()
 		for cpv in cpvlist:
 			distlist.update(self._getCpvDistfiles(cpv))
+
 		if requiredDistfiles is None:
 			# This allows us to force removal of stale digests for the
 			# ebuild --force digest option (no distfiles are required).
@@ -404,6 +372,81 @@ class Manifest(object):
 					if f in requiredDistfiles:
 						raise
 
+	def _is_cpv(self, cat, pn, filename):
+		if not filename.endswith(".ebuild"):
+			return None
+		pf = filename[:-7]
+		if pf is None:
+			return None
+		ps = portage.versions._pkgsplit(pf)
+		cpv = "%s/%s" % (cat, pf)
+		if not ps:
+			raise PortagePackageException(
+				_("Invalid package name: '%s'") % cpv)
+		if ps[0] != pn:
+			raise PortagePackageException(
+				_("Package name does not "
+				"match directory name: '%s'") % cpv)
+		return cpv
+
+	def _update_thin_pkgdir(self, cat, pn, pkgdir):
+		for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
+			break
+		cpvlist = []
+		for f in pkgdir_files:
+			try:
+				f = _unicode_decode(f,
+					encoding=_encodings['fs'], errors='strict')
+			except UnicodeDecodeError:
+				continue
+			if f[:1] == '.':
+				continue
+			pf = self._is_cpv(cat, pn, f)
+			if pf is not None:
+				cpvlist.append(pf)
+		return cpvlist
+
+	def _update_thick_pkgdir(self, cat, pn, pkgdir):
+		cpvlist = []
+		for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
+			break
+		for f in pkgdir_files:
+			try:
+				f = _unicode_decode(f,
+					encoding=_encodings['fs'], errors='strict')
+			except UnicodeDecodeError:
+				continue
+			if f[:1] == ".":
+				continue
+			pf = self._is_cpv(cat, pn, f)
+			if pf is not None:
+				mytype = "EBUILD"
+				cpvlist.append(pf)
+			elif manifest2MiscfileFilter(f):
+				mytype = "MISC"
+			else:
+				continue
+			self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes)
+		recursive_files = []
+
+		pkgdir = self.pkgdir
+		cut_len = len(os.path.join(pkgdir, "files") + os.sep)
+		for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
+			for f in files:
+				try:
+					f = _unicode_decode(f,
+						encoding=_encodings['fs'], errors='strict')
+				except UnicodeDecodeError:
+					continue
+				full_path = os.path.join(parentdir, f)
+				recursive_files.append(full_path[cut_len:])
+		for f in recursive_files:
+			if not manifest2AuxfileFilter(f):
+				continue
+			self.fhashdict["AUX"][f] = perform_multiple_checksums(
+				os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes)
+		return cpvlist
+
 	def _pkgdir_category(self):
 		return self.pkgdir.rstrip(os.sep).split(os.sep)[-2]
 
diff --git a/pym/portage/package/ebuild/digestcheck.py b/pym/portage/package/ebuild/digestcheck.py
index d184301..466fd05 100644
--- a/pym/portage/package/ebuild/digestcheck.py
+++ b/pym/portage/package/ebuild/digestcheck.py
@@ -92,7 +92,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
 		writemsg(_("!!! Got: %s\n") % e.value[2], noiselevel=-1)
 		writemsg(_("!!! Expected: %s\n") % e.value[3], noiselevel=-1)
 		return 0
-	if allow_missing:
+	if allow_missing or mf.thin:
 		# In this case we ignore any missing digests that
 		# would otherwise be detected below.
 		return 1
-- 
1.7.6.1

[gentoo-portage-dev] [PATCH 3/3] add layout.conf awareness of thin-manifests

[Brian Harring]

For any repo that wants thin (just src_uri digests), they just need to add

thin-manifests = true

to their layout.conf.  Again, this should only be used in repositories
were the backing vcs provides checksums for the ebuild data.
---
 pym/portage/repository/config.py |   11 +++++++++--
 1 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 014ec25..0731223 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -42,7 +42,7 @@ class RepoConfig(object):
 	"""Stores config of one repository"""
 
 	__slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo',
-		'missing_repo_name', 'name', 'priority', 'sync', 'format', 'load_manifest']
+		'missing_repo_name', 'name', 'priority', 'sync', 'format', 'thin_manifest']
 
 	def __init__(self, name, repo_opts):
 		"""Build a RepoConfig with options in repo_opts
@@ -111,7 +111,11 @@ class RepoConfig(object):
 			missing = False
 		self.name = name
 		self.missing_repo_name = missing
-		self.load_manifest = manifest.Manifest
+		self.thin_manifest = False
+
+	def load_manifest(self, *args, **kwds):
+		kwds['thin'] = self.thin_manifest
+		return manifest.Manifest(*args, **kwds)
 
 	def update(self, new_repo):
 		"""Update repository with options in another RepoConfig"""
@@ -331,6 +335,9 @@ class RepoConfigLoader(object):
 					aliases.extend(repo.aliases)
 				repo.aliases = tuple(sorted(set(aliases)))
 
+			if layout_data.get('thin-manifests', '').lower() == 'true':
+				repo.thin_manifest = True
+
 		#Take aliases into account.
 		new_prepos = {}
 		for repo_name, repo in prepos.items():
-- 
1.7.6.1