[gentoo-commits] proj/portage:master commit in: bin/, pym/portage/repository/, pym/portage/

[gentoo-commits] proj/portage:master commit in: bin/, pym/portage/repository/, pym/portage/

[Zac Medico]

commit:     6d8d0c02457c2e94c759fe89db0bef196b78158a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 23 22:43:04 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Sep 23 22:43:04 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6d8d0c02

RepoConfig: add find_invalid_path_char method

This binds filename validation to the RepoConfig, so that eventually
we'll be able to control it via a layout.conf setting as discussed in
bug #435934.

---
 bin/repoman                      |   23 ++++++++++-------------
 pym/portage/manifest.py          |   24 ++++++++++++------------
 pym/portage/repository/config.py |   22 +++++++++++++++++++++-
 3 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/bin/repoman b/bin/repoman
index bc2ac9b..b463cbe 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -71,8 +71,6 @@ from portage import cvstree, normalize_path
 from portage import util
 from portage.exception import (FileNotFound, MissingParameter,
 	ParseError, PermissionDenied)
-from portage.manifest import _prohibited_filename_chars_re as \
-	disallowed_filename_chars_re
 from portage.process import find_binary, spawn
 from portage.output import bold, create_color_func, \
 	green, nocolor, red
@@ -1431,19 +1429,19 @@ for x in effective_scanlist:
 	ebuildlist = [pkg.pf for pkg in ebuildlist]
 
 	for y in checkdirlist:
-		m = disallowed_filename_chars_re.search(y.strip(os.sep))
-		if m is not None:
+		index = repo_config.find_invalid_path_char(y)
+		if index != -1:
 			y_relative = os.path.join(checkdir_relative, y)
 			if vcs is not None and not vcs_new_changed(y_relative):
 				# If the file isn't in the VCS new or changed set, then
 				# assume that it's an irrelevant temporary file (Manifest
 				# entries are not generated for file names containing
 				# prohibited characters). See bug #406877.
-				m = None
-		if m is not None:
+				index = -1
+		if index != -1:
 			stats["file.name"] += 1
 			fails["file.name"].append("%s/%s: char '%s'" % \
-				(checkdir, y, m.group(0)))
+				(checkdir, y, y[index]))
 
 		if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
 			continue
@@ -1609,20 +1607,19 @@ for x in effective_scanlist:
 				stats["file.size"] += 1
 				fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
 
-			m = disallowed_filename_chars_re.search(
-				os.path.basename(y.rstrip(os.sep)))
-			if m is not None:
+			index = repo_config.find_invalid_path_char(y)
+			if index != -1:
 				y_relative = os.path.join(checkdir_relative, "files", y)
 				if vcs is not None and not vcs_new_changed(y_relative):
 					# If the file isn't in the VCS new or changed set, then
 					# assume that it's an irrelevant temporary file (Manifest
 					# entries are not generated for file names containing
 					# prohibited characters). See bug #406877.
-					m = None
-			if m is not None:
+					index = -1
+			if index != -1:
 				stats["file.name"] += 1
 				fails["file.name"].append("%s/files/%s: char '%s'" % \
-					(checkdir, y, m.group(0)))
+					(checkdir, y, y[index]))
 	del mydigests
 
 	if check_changelog and "ChangeLog" not in checkdirlist:

diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 25886bb..b81b580 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -3,7 +3,6 @@
 
 import errno
 import io
-import re
 import sys
 import warnings
 
@@ -11,6 +10,7 @@ import portage
 portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
 		'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes',
+	'portage.repository.config:_find_invalid_path_char',
 	'portage.util:write_atomic',
 )
 
@@ -30,9 +30,6 @@ if sys.hexversion >= 0x3000000:
 else:
 	_unicode = unicode
 
-# Characters prohibited by repoman's file.name check.
-_prohibited_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]')
-
 class FileNotInManifestException(PortageException):
 	pass
 
@@ -44,14 +41,10 @@ def manifest2AuxfileFilter(filename):
 	for x in mysplit:
 		if x[:1] == '.':
 			return False
-		if _prohibited_filename_chars_re.search(x) is not None:
-			return False
 	return not filename[:7] == 'digest-'
 
 def manifest2MiscfileFilter(filename):
 	filename = filename.strip(os.sep)
-	if _prohibited_filename_chars_re.search(filename) is not None:
-		return False
 	return not (filename in ["CVS", ".svn", "files", "Manifest"] or filename.endswith(".ebuild"))
 
 def guessManifestFileType(filename):
@@ -126,7 +119,8 @@ class Manifest(object):
 	parsers = (parseManifest2,)
 	def __init__(self, pkgdir, distdir, fetchlist_dict=None,
 		manifest1_compat=DeprecationWarning, from_scratch=False, thin=False,
-			allow_missing=False, allow_create=True, hashes=None):
+		allow_missing=False, allow_create=True, hashes=None,
+		find_invalid_path_char=None):
 		""" Create new Manifest instance for package in pkgdir.
 		    Do not parse Manifest file if from_scratch == True (only for internal use)
 			The fetchlist_dict parameter is required only for generation of
@@ -139,6 +133,9 @@ class Manifest(object):
 				"portage.manifest.Manifest constructor is deprecated.",
 				DeprecationWarning, stacklevel=2)
 
+		if find_invalid_path_char is None:
+			find_invalid_path_char = _find_invalid_path_char
+		self._find_invalid_path_char = find_invalid_path_char
 		self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
 		self.fhashdict = {}
 		self.hashes = set()
@@ -380,7 +377,8 @@ class Manifest(object):
 		self.__init__(self.pkgdir, self.distdir,
 			fetchlist_dict=self.fetchlist_dict, from_scratch=True,
 			thin=self.thin, allow_missing=self.allow_missing,
-			allow_create=self.allow_create, hashes=self.hashes)
+			allow_create=self.allow_create, hashes=self.hashes,
+			find_invalid_path_char=self._find_invalid_path_char)
 		pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
 		cat = self._pkgdir_category()
 
@@ -475,7 +473,8 @@ class Manifest(object):
 			if pf is not None:
 				mytype = "EBUILD"
 				cpvlist.append(pf)
-			elif manifest2MiscfileFilter(f):
+			elif self._find_invalid_path_char(f) == -1 and \
+				manifest2MiscfileFilter(f):
 				mytype = "MISC"
 			else:
 				continue
@@ -494,7 +493,8 @@ class Manifest(object):
 				full_path = os.path.join(parentdir, f)
 				recursive_files.append(full_path[cut_len:])
 		for f in recursive_files:
-			if not manifest2AuxfileFilter(f):
+			if self._find_invalid_path_char(f) != -1 or \
+				not manifest2AuxfileFilter(f):
 				continue
 			self.fhashdict["AUX"][f] = perform_multiple_checksums(
 				os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes)

diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 77b016d..83018b8 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -28,6 +28,9 @@ from portage import _unicode_encode
 from portage import _encodings
 from portage import manifest
 
+# Characters prohibited by repoman's file.name check.
+_invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]')
+
 _valid_profile_formats = frozenset(
 	['pms', 'portage-1', 'portage-2'])
 
@@ -49,12 +52,27 @@ def _gen_valid_repo(name):
 		name = None
 	return name
 
+def _find_invalid_path_char(path, pos=0, endpos=None):
+	"""
+	Returns the position of the first invalid character found in basename,
+	or -1 if no invalid characters are found.
+	"""
+	if endpos is None:
+		endpos = len(path)
+
+	m = _invalid_path_char_re.search(path, pos=pos, endpos=endpos)
+	if m is not None:
+		return m.start()
+
+	return -1
+
 class RepoConfig(object):
 	"""Stores config of one repository"""
 
 	__slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual',
 		'cache_formats', 'create_manifest', 'disable_manifest', 'eapi',
-		'eclass_db', 'eclass_locations', 'eclass_overrides', 'format', 'location',
+		'eclass_db', 'eclass_locations', 'eclass_overrides',
+		'find_invalid_path_char', 'format', 'location',
 		'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
 		'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority',
 		'profile_formats', 'sign_commit', 'sign_manifest', 'sync',
@@ -138,6 +156,7 @@ class RepoConfig(object):
 		self.cache_formats = None
 		self.portage1_profiles = True
 		self.portage1_profiles_compat = False
+		self.find_invalid_path_char = _find_invalid_path_char
 
 		# Parse layout.conf.
 		if self.location:
@@ -211,6 +230,7 @@ class RepoConfig(object):
 		kwds['hashes'] = self.manifest_hashes
 		if self.disable_manifest:
 			kwds['from_scratch'] = True
+		kwds['find_invalid_path_char'] = self.find_invalid_path_char
 		return manifest.Manifest(*args, **kwds)
 
 	def update(self, new_repo):