* [gentoo-portage-dev] [PATCH 1/3] Bind all manifest access through repoconfigs
2011-09-01 22:07 [gentoo-portage-dev] [PATCH 0/3] thin manifest support Brian Harring
@ 2011-09-01 22:07 ` Brian Harring
2011-09-01 22:07 ` [gentoo-portage-dev] [PATCH 2/3] add thin manifest support to the Manifest class Brian Harring
2011-09-01 22:07 ` [gentoo-portage-dev] [PATCH 3/3] add layout.conf awareness of thin-manifests Brian Harring
2 siblings, 0 replies; 4+ messages in thread
From: Brian Harring @ 2011-09-01 22:07 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Brian Harring
This enables controling the behaviour (creation and validation) per
repo, and while mildly ugly, refactors in the right direction.
---
bin/ebuild | 5 +++--
bin/repoman | 8 ++++++--
pym/_emerge/EbuildFetcher.py | 6 ++++--
pym/_emerge/search.py | 4 +++-
pym/portage/dbapi/porttree.py | 8 ++++++--
pym/portage/package/ebuild/digestcheck.py | 4 +++-
pym/portage/package/ebuild/digestgen.py | 3 ++-
pym/portage/package/ebuild/doebuild.py | 4 +++-
pym/portage/package/ebuild/fetch.py | 3 ++-
pym/portage/repository/config.py | 7 ++++++-
10 files changed, 38 insertions(+), 14 deletions(-)
diff --git a/bin/ebuild b/bin/ebuild
index db7e5e3..92105bb 100755
--- a/bin/ebuild
+++ b/bin/ebuild
@@ -200,8 +200,9 @@ def discard_digests(myebuild, mysettings, mydbapi):
portage._doebuild_manifest_exempt_depend += 1
pkgdir = os.path.dirname(myebuild)
fetchlist_dict = portage.FetchlistDict(pkgdir, mysettings, mydbapi)
- from portage.manifest import Manifest
- mf = Manifest(pkgdir, mysettings["DISTDIR"],
+ mf = mysettings.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(pkgdir)))
+ mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict, manifest1_compat=False)
mf.create(requiredDistfiles=None,
assumeDistHashesSometimes=True, assumeDistHashesAlways=True)
diff --git a/bin/repoman b/bin/repoman
index 6ec84e5..5f81a0f 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -1104,7 +1104,9 @@ for x in scanlist:
portage._doebuild_manifest_exempt_depend += 1
try:
distdir = repoman_settings['DISTDIR']
- mf = portage.manifest.Manifest(checkdir, distdir,
+ mf = repoman_settings.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(checkdir)))
+ mf = mf.load_manifest(checkdir, distdir,
fetchlist_dict=fetchlist_dict)
mf.create(requiredDistfiles=None,
assumeDistHashesAlways=True)
@@ -1314,7 +1316,9 @@ for x in scanlist:
raise
continue
- mf = Manifest(checkdir, repoman_settings["DISTDIR"])
+ mf = repoman_settings.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(checkdir)))
+ mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
mydigests=mf.getTypeDigests("DIST")
fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index feb68d0..4389f84 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -206,8 +206,10 @@ class EbuildFetcher(SpawnProcess):
def _get_digests(self):
if self._digests is not None:
return self._digests
- self._digests = portage.Manifest(os.path.dirname(
- self._get_ebuild_path()), None).getTypeDigests("DIST")
+ pkgdir = os.path.dirname(self._get_ebuild_path())
+ mf = self.pkg.root_config.settings.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(pkgdir)))
+ self._digests = mf.load_manifest(pkgdir, None).getTypeDigests("DIST")
return self._digests
def _get_uri_map(self):
diff --git a/pym/_emerge/search.py b/pym/_emerge/search.py
index 096b384..4a4183d 100644
--- a/pym/_emerge/search.py
+++ b/pym/_emerge/search.py
@@ -317,7 +317,9 @@ class search(object):
installed=False, metadata=metadata,
root_config=self.root_config, type_name="ebuild")
pkgdir = os.path.dirname(myebuild)
- mf = Manifest(
+ mf = self.settings.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(pkgdir)))
+ mf = mf.load_manifest(
pkgdir, self.settings["DISTDIR"])
try:
uri_map = _parse_uri_map(mycpv, metadata,
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..1172798 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -576,7 +576,9 @@ class portdbapi(dbapi):
if myebuild is None:
raise AssertionError(_("ebuild not found for '%s'") % mypkg)
pkgdir = os.path.dirname(myebuild)
- mf = Manifest(pkgdir, self.settings["DISTDIR"])
+ mf = self.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
+ pkgdir, self.settings["DISTDIR"])
checksums = mf.getDigests()
if not checksums:
if debug:
@@ -644,7 +646,9 @@ class portdbapi(dbapi):
if myebuild is None:
raise AssertionError(_("ebuild not found for '%s'") % mypkg)
pkgdir = os.path.dirname(myebuild)
- mf = Manifest(pkgdir, self.settings["DISTDIR"])
+ mf = self.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(pkgdir)))
+ mf = mf.load_manifest(pkgdir, self.settings["DISTDIR"])
mysums = mf.getDigests()
failures = {}
diff --git a/pym/portage/package/ebuild/digestcheck.py b/pym/portage/package/ebuild/digestcheck.py
index 1e34b14..d184301 100644
--- a/pym/portage/package/ebuild/digestcheck.py
+++ b/pym/portage/package/ebuild/digestcheck.py
@@ -41,7 +41,9 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
else:
return 1
if mf is None:
- mf = Manifest(pkgdir, mysettings["DISTDIR"])
+ mf = mysettings.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(pkgdir)))
+ mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"])
manifest_empty = True
for d in mf.fhashdict.values():
if d:
diff --git a/pym/portage/package/ebuild/digestgen.py b/pym/portage/package/ebuild/digestgen.py
index eb7210e..6051512 100644
--- a/pym/portage/package/ebuild/digestgen.py
+++ b/pym/portage/package/ebuild/digestgen.py
@@ -53,7 +53,8 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None):
return 0
mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
manifest1_compat = False
- mf = Manifest(mysettings["O"], mysettings["DISTDIR"],
+ mf = mysettings.repositories.get_repo_for_location(mytree)
+ mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat)
# Don't require all hashes since that can trigger excessive
# fetches when sufficient digests already exist. To ease transition
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index 7b3561e..7c276ba 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -508,7 +508,9 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
out.eerror(_("Manifest not found for '%s'") % (myebuild,))
_doebuild_broken_ebuilds.add(myebuild)
return 1
- mf = Manifest(pkgdir, mysettings["DISTDIR"])
+ mf = mysettings.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(pkgdir)))
+ mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"])
else:
mf = _doebuild_manifest_cache
diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
index 5cbbf87..11c4c01 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -356,7 +356,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
allow_missing_digests = True
pkgdir = mysettings.get("O")
if digests is None and not (pkgdir is None or skip_manifest):
- mydigests = Manifest(
+ mydigests = mysettings.repositories.get_repo_for_location(
+ os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST")
elif digests is None or skip_manifest:
# no digests because fetch was not called for a specific package
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index a12bd7b..014ec25 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -21,6 +21,7 @@ from portage.util import normalize_path, writemsg, writemsg_level, shlex_split
from portage.localization import _
from portage import _unicode_encode
from portage import _encodings
+from portage import manifest
_repo_name_sub_re = re.compile(r'[^\w-]')
@@ -41,7 +42,7 @@ class RepoConfig(object):
"""Stores config of one repository"""
__slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo',
- 'missing_repo_name', 'name', 'priority', 'sync', 'format']
+ 'missing_repo_name', 'name', 'priority', 'sync', 'format', 'load_manifest']
def __init__(self, name, repo_opts):
"""Build a RepoConfig with options in repo_opts
@@ -110,6 +111,7 @@ class RepoConfig(object):
missing = False
self.name = name
self.missing_repo_name = missing
+ self.load_manifest = manifest.Manifest
def update(self, new_repo):
"""Update repository with options in another RepoConfig"""
@@ -498,6 +500,9 @@ class RepoConfigLoader(object):
return None
return self.treemap[repo_name]
+ def get_repo_for_location(self, location):
+ return self.prepos[self.get_name_for_location(location)]
+
def __getitem__(self, repo_name):
return self.prepos[repo_name]
--
1.7.6.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [gentoo-portage-dev] [PATCH 2/3] add thin manifest support to the Manifest class
2011-09-01 22:07 [gentoo-portage-dev] [PATCH 0/3] thin manifest support Brian Harring
2011-09-01 22:07 ` [gentoo-portage-dev] [PATCH 1/3] Bind all manifest access through repoconfigs Brian Harring
@ 2011-09-01 22:07 ` Brian Harring
2011-09-01 22:07 ` [gentoo-portage-dev] [PATCH 3/3] add layout.conf awareness of thin-manifests Brian Harring
2 siblings, 0 replies; 4+ messages in thread
From: Brian Harring @ 2011-09-01 22:07 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Brian Harring
'thin' is just distfiles. This is primarily useful when the ebuild
lives in a vcs- git for example, which already has it's own checksums
to rely on.
---
pym/portage/manifest.py | 149 ++++++++++++++++++----------
pym/portage/package/ebuild/digestcheck.py | 2 +-
2 files changed, 97 insertions(+), 54 deletions(-)
diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 13efab7..ef1a552 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -49,6 +49,12 @@ def guessManifestFileType(filename):
else:
return "DIST"
+def guessThinManifestFileType(filename):
+ type = guessManifestFileType(filename)
+ if type != "DIST":
+ return None
+ return "DIST"
+
def parseManifest2(mysplit):
myentry = None
if len(mysplit) > 4 and mysplit[0] in portage.const.MANIFEST2_IDENTIFIERS:
@@ -93,12 +99,14 @@ class Manifest2Entry(ManifestEntry):
class Manifest(object):
parsers = (parseManifest2,)
def __init__(self, pkgdir, distdir, fetchlist_dict=None,
- manifest1_compat=False, from_scratch=False):
+ manifest1_compat=False, from_scratch=False, thin=False):
""" create new Manifest instance for package in pkgdir
and add compability entries for old portage versions if manifest1_compat == True.
Do not parse Manifest file if from_scratch == True (only for internal use)
The fetchlist_dict parameter is required only for generation of
- a Manifest (not needed for parsing and checking sums)."""
+ a Manifest (not needed for parsing and checking sums).
+ If thin is specified, then the manifest carries only info for
+ distfiles."""
self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
self.fhashdict = {}
self.hashes = set()
@@ -120,7 +128,11 @@ class Manifest(object):
else:
self.fetchlist_dict = {}
self.distdir = distdir
- self.guessType = guessManifestFileType
+ self.thin = thin
+ if thin:
+ self.guessType = guessThinManifestFileType
+ else:
+ self.guessType = guessManifestFileType
def getFullname(self):
""" Returns the absolute path to the Manifest file for this instance """
@@ -313,64 +325,20 @@ class Manifest(object):
distfilehashes = {}
self.__init__(self.pkgdir, self.distdir,
fetchlist_dict=self.fetchlist_dict, from_scratch=True,
- manifest1_compat=False)
- cpvlist = []
+ manifest1_compat=False, thin=self.thin)
pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
cat = self._pkgdir_category()
pkgdir = self.pkgdir
+ if self.thin:
+ cpvlist = self._update_thin_pkgdir(cat, pn, pkgdir)
+ else:
+ cpvlist = self._update_thick_pkgdir(cat, pn, pkgdir)
- for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
- break
- for f in pkgdir_files:
- try:
- f = _unicode_decode(f,
- encoding=_encodings['fs'], errors='strict')
- except UnicodeDecodeError:
- continue
- if f[:1] == ".":
- continue
- pf = None
- if f[-7:] == '.ebuild':
- pf = f[:-7]
- if pf is not None:
- mytype = "EBUILD"
- ps = portage.versions._pkgsplit(pf)
- cpv = "%s/%s" % (cat, pf)
- if not ps:
- raise PortagePackageException(
- _("Invalid package name: '%s'") % cpv)
- if ps[0] != pn:
- raise PortagePackageException(
- _("Package name does not "
- "match directory name: '%s'") % cpv)
- cpvlist.append(cpv)
- elif manifest2MiscfileFilter(f):
- mytype = "MISC"
- else:
- continue
- self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes)
- recursive_files = []
-
- pkgdir = self.pkgdir
- cut_len = len(os.path.join(pkgdir, "files") + os.sep)
- for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
- for f in files:
- try:
- f = _unicode_decode(f,
- encoding=_encodings['fs'], errors='strict')
- except UnicodeDecodeError:
- continue
- full_path = os.path.join(parentdir, f)
- recursive_files.append(full_path[cut_len:])
- for f in recursive_files:
- if not manifest2AuxfileFilter(f):
- continue
- self.fhashdict["AUX"][f] = perform_multiple_checksums(
- os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes)
distlist = set()
for cpv in cpvlist:
distlist.update(self._getCpvDistfiles(cpv))
+
if requiredDistfiles is None:
# This allows us to force removal of stale digests for the
# ebuild --force digest option (no distfiles are required).
@@ -404,6 +372,81 @@ class Manifest(object):
if f in requiredDistfiles:
raise
+ def _is_cpv(self, cat, pn, filename):
+ if not filename.endswith(".ebuild"):
+ return None
+ pf = filename[:-7]
+ if pf is None:
+ return None
+ ps = portage.versions._pkgsplit(pf)
+ cpv = "%s/%s" % (cat, pf)
+ if not ps:
+ raise PortagePackageException(
+ _("Invalid package name: '%s'") % cpv)
+ if ps[0] != pn:
+ raise PortagePackageException(
+ _("Package name does not "
+ "match directory name: '%s'") % cpv)
+ return cpv
+
+ def _update_thin_pkgdir(self, cat, pn, pkgdir):
+ for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
+ break
+ cpvlist = []
+ for f in pkgdir_files:
+ try:
+ f = _unicode_decode(f,
+ encoding=_encodings['fs'], errors='strict')
+ except UnicodeDecodeError:
+ continue
+ if f[:1] == '.':
+ continue
+ pf = self._is_cpv(cat, pn, f)
+ if pf is not None:
+ cpvlist.append(pf)
+ return cpvlist
+
+ def _update_thick_pkgdir(self, cat, pn, pkgdir):
+ cpvlist = []
+ for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
+ break
+ for f in pkgdir_files:
+ try:
+ f = _unicode_decode(f,
+ encoding=_encodings['fs'], errors='strict')
+ except UnicodeDecodeError:
+ continue
+ if f[:1] == ".":
+ continue
+ pf = self._is_cpv(cat, pn, f)
+ if pf is not None:
+ mytype = "EBUILD"
+ cpvlist.append(pf)
+ elif manifest2MiscfileFilter(f):
+ mytype = "MISC"
+ else:
+ continue
+ self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes)
+ recursive_files = []
+
+ pkgdir = self.pkgdir
+ cut_len = len(os.path.join(pkgdir, "files") + os.sep)
+ for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
+ for f in files:
+ try:
+ f = _unicode_decode(f,
+ encoding=_encodings['fs'], errors='strict')
+ except UnicodeDecodeError:
+ continue
+ full_path = os.path.join(parentdir, f)
+ recursive_files.append(full_path[cut_len:])
+ for f in recursive_files:
+ if not manifest2AuxfileFilter(f):
+ continue
+ self.fhashdict["AUX"][f] = perform_multiple_checksums(
+ os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes)
+ return cpvlist
+
def _pkgdir_category(self):
return self.pkgdir.rstrip(os.sep).split(os.sep)[-2]
diff --git a/pym/portage/package/ebuild/digestcheck.py b/pym/portage/package/ebuild/digestcheck.py
index d184301..466fd05 100644
--- a/pym/portage/package/ebuild/digestcheck.py
+++ b/pym/portage/package/ebuild/digestcheck.py
@@ -92,7 +92,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
writemsg(_("!!! Got: %s\n") % e.value[2], noiselevel=-1)
writemsg(_("!!! Expected: %s\n") % e.value[3], noiselevel=-1)
return 0
- if allow_missing:
+ if allow_missing or mf.thin:
# In this case we ignore any missing digests that
# would otherwise be detected below.
return 1
--
1.7.6.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [gentoo-portage-dev] [PATCH 3/3] add layout.conf awareness of thin-manifests
2011-09-01 22:07 [gentoo-portage-dev] [PATCH 0/3] thin manifest support Brian Harring
2011-09-01 22:07 ` [gentoo-portage-dev] [PATCH 1/3] Bind all manifest access through repoconfigs Brian Harring
2011-09-01 22:07 ` [gentoo-portage-dev] [PATCH 2/3] add thin manifest support to the Manifest class Brian Harring
@ 2011-09-01 22:07 ` Brian Harring
2 siblings, 0 replies; 4+ messages in thread
From: Brian Harring @ 2011-09-01 22:07 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: Brian Harring
For any repo that wants thin (just src_uri digests), they just need to add
thin-manifests = true
to their layout.conf. Again, this should only be used in repositories
were the backing vcs provides checksums for the ebuild data.
---
pym/portage/repository/config.py | 11 +++++++++--
1 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 014ec25..0731223 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -42,7 +42,7 @@ class RepoConfig(object):
"""Stores config of one repository"""
__slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo',
- 'missing_repo_name', 'name', 'priority', 'sync', 'format', 'load_manifest']
+ 'missing_repo_name', 'name', 'priority', 'sync', 'format', 'thin_manifest']
def __init__(self, name, repo_opts):
"""Build a RepoConfig with options in repo_opts
@@ -111,7 +111,11 @@ class RepoConfig(object):
missing = False
self.name = name
self.missing_repo_name = missing
- self.load_manifest = manifest.Manifest
+ self.thin_manifest = False
+
+ def load_manifest(self, *args, **kwds):
+ kwds['thin'] = self.thin_manifest
+ return manifest.Manifest(*args, **kwds)
def update(self, new_repo):
"""Update repository with options in another RepoConfig"""
@@ -331,6 +335,9 @@ class RepoConfigLoader(object):
aliases.extend(repo.aliases)
repo.aliases = tuple(sorted(set(aliases)))
+ if layout_data.get('thin-manifests', '').lower() == 'true':
+ repo.thin_manifest = True
+
#Take aliases into account.
new_prepos = {}
for repo_name, repo in prepos.items():
--
1.7.6.1
^ permalink raw reply related [flat|nested] 4+ messages in thread