[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     633cabfb9215633c554fd967b9875310be3718bd
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Tue May  3 07:18:05 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=633cabfb

repoman: Use lxml for parsing of metadata

Note that we no longer throw a QA error for a missing XML prolog, as long as
the encoding matches the default ('UTF-8'; lowercase is also allowed).
Update import error message to lxml pkg.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 46 +++++++-----------------
 1 file changed, 13 insertions(+), 33 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index bcddb3e..317ab56 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -7,14 +7,14 @@ import sys
 from itertools import chain
 
 try:
-	import xml.etree.ElementTree
-	from xml.parsers.expat import ExpatError
+	from lxml import etree
+	from lxml.etree import ParserError
 except (SystemExit, KeyboardInterrupt):
 	raise
 except (ImportError, SystemError, RuntimeError, Exception):
 	# broken or missing xml support
 	# http://bugs.python.org/issue14988
-	msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
+	msg = ["Please emerge dev-python/lxml in order to use repoman."]
 	from portage.output import EOutput
 	out = EOutput()
 	for line in msg:
@@ -26,12 +26,11 @@ from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
+from repoman._xml import XmlLint
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import _encodings, _unicode_encode
 from portage import exception
 from portage.dep import Atom
 
@@ -145,50 +144,31 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		# metadata.xml parse check
 		metadata_bad = False
-		xml_info = {}
-		xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
 
 		# read metadata.xml into memory
 		try:
-			_metadata_xml = xml.etree.ElementTree.parse(
-				_unicode_encode(
-					os.path.join(checkdir, "metadata.xml"),
-					encoding=_encodings['fs'], errors='strict'),
-				parser=xml_parser)
-		except (ExpatError, SyntaxError, EnvironmentError) as e:
+			_metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml'))
+		except (ParserError, SyntaxError, EnvironmentError) as e:
 			metadata_bad = True
 			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
 			del e
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if "XML_DECLARATION" not in xml_info:
+		xml_encoding = _metadata_xml.docinfo.encoding
+		if xml_encoding.upper() != metadata_xml_encoding:
 			self.qatracker.add_error(
 				"metadata.bad", "%s/metadata.xml: "
-				"xml declaration is missing on first line, "
-				"should be '%s'" % (xpkg, metadata_xml_declaration))
-		else:
-			xml_version, xml_encoding, xml_standalone = \
-				xml_info["XML_DECLARATION"]
-			if xml_encoding is None or \
-				xml_encoding.upper() != metadata_xml_encoding:
-				if xml_encoding is None:
-					encoding_problem = "but it is undefined"
-				else:
-					encoding_problem = "not '%s'" % xml_encoding
-				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration encoding should be '%s', %s" %
-					(xpkg, metadata_xml_encoding, encoding_problem))
+				"xml declaration encoding should be '%s', not '%s'" %
+				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if "DOCTYPE" not in xml_info:
+		if not _metadata_xml.docinfo:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",
 				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 		else:
-			doctype_name, doctype_system, doctype_pubid = \
-				xml_info["DOCTYPE"]
+			doctype_system = _metadata_xml.docinfo.system_url
 			if doctype_system != metadata_dtd_uri:
 				if doctype_system is None:
 					system_problem = "but it is undefined"
@@ -198,7 +178,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					"metadata.bad", "%s/metadata.xml: "
 					"DOCTYPE: SYSTEM should refer to '%s', %s" %
 					(xpkg, metadata_dtd_uri, system_problem))
-
+			doctype_name = _metadata_xml.docinfo.doctype.split(' ')[1]
 			if doctype_name != metadata_doctype_name:
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     633cabfb9215633c554fd967b9875310be3718bd
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Tue May  3 07:18:05 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=633cabfb

repoman: Use lxml for parsing of metadata

Note that we no longer throw a QA error for a missing XML prolog, as long as
the encoding matches the default ('UTF-8'; lowercase is also allowed).
Update import error message to lxml pkg.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 46 +++++++-----------------
 1 file changed, 13 insertions(+), 33 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index bcddb3e..317ab56 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -7,14 +7,14 @@ import sys
 from itertools import chain
 
 try:
-	import xml.etree.ElementTree
-	from xml.parsers.expat import ExpatError
+	from lxml import etree
+	from lxml.etree import ParserError
 except (SystemExit, KeyboardInterrupt):
 	raise
 except (ImportError, SystemError, RuntimeError, Exception):
 	# broken or missing xml support
 	# http://bugs.python.org/issue14988
-	msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
+	msg = ["Please emerge dev-python/lxml in order to use repoman."]
 	from portage.output import EOutput
 	out = EOutput()
 	for line in msg:
@@ -26,12 +26,11 @@ from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
+from repoman._xml import XmlLint
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import _encodings, _unicode_encode
 from portage import exception
 from portage.dep import Atom
 
@@ -145,50 +144,31 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		# metadata.xml parse check
 		metadata_bad = False
-		xml_info = {}
-		xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
 
 		# read metadata.xml into memory
 		try:
-			_metadata_xml = xml.etree.ElementTree.parse(
-				_unicode_encode(
-					os.path.join(checkdir, "metadata.xml"),
-					encoding=_encodings['fs'], errors='strict'),
-				parser=xml_parser)
-		except (ExpatError, SyntaxError, EnvironmentError) as e:
+			_metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml'))
+		except (ParserError, SyntaxError, EnvironmentError) as e:
 			metadata_bad = True
 			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
 			del e
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if "XML_DECLARATION" not in xml_info:
+		xml_encoding = _metadata_xml.docinfo.encoding
+		if xml_encoding.upper() != metadata_xml_encoding:
 			self.qatracker.add_error(
 				"metadata.bad", "%s/metadata.xml: "
-				"xml declaration is missing on first line, "
-				"should be '%s'" % (xpkg, metadata_xml_declaration))
-		else:
-			xml_version, xml_encoding, xml_standalone = \
-				xml_info["XML_DECLARATION"]
-			if xml_encoding is None or \
-				xml_encoding.upper() != metadata_xml_encoding:
-				if xml_encoding is None:
-					encoding_problem = "but it is undefined"
-				else:
-					encoding_problem = "not '%s'" % xml_encoding
-				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration encoding should be '%s', %s" %
-					(xpkg, metadata_xml_encoding, encoding_problem))
+				"xml declaration encoding should be '%s', not '%s'" %
+				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if "DOCTYPE" not in xml_info:
+		if not _metadata_xml.docinfo:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",
 				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 		else:
-			doctype_name, doctype_system, doctype_pubid = \
-				xml_info["DOCTYPE"]
+			doctype_system = _metadata_xml.docinfo.system_url
 			if doctype_system != metadata_dtd_uri:
 				if doctype_system is None:
 					system_problem = "but it is undefined"
@@ -198,7 +178,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					"metadata.bad", "%s/metadata.xml: "
 					"DOCTYPE: SYSTEM should refer to '%s', %s" %
 					(xpkg, metadata_dtd_uri, system_problem))
-
+			doctype_name = _metadata_xml.docinfo.doctype.split(' ')[1]
 			if doctype_name != metadata_doctype_name:
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     9c3e213142e0e39f69dc002bd15402a7a2ff22d1
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 19:45:31 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9c3e2131

repoman: mandate use of python 2.7 or 3.2+ for simplicity

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++++++++++-------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index af53f4b..bcddb3e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -162,54 +162,48 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if not hasattr(xml_parser, 'parser') or \
-			sys.hexversion < 0x2070000 or \
-			(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-			# doctype is not parsed with python 2.6 or 3.1
-			pass
+		if "XML_DECLARATION" not in xml_info:
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: "
+				"xml declaration is missing on first line, "
+				"should be '%s'" % (xpkg, metadata_xml_declaration))
 		else:
-			if "XML_DECLARATION" not in xml_info:
+			xml_version, xml_encoding, xml_standalone = \
+				xml_info["XML_DECLARATION"]
+			if xml_encoding is None or \
+				xml_encoding.upper() != metadata_xml_encoding:
+				if xml_encoding is None:
+					encoding_problem = "but it is undefined"
+				else:
+					encoding_problem = "not '%s'" % xml_encoding
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration is missing on first line, "
-					"should be '%s'" % (xpkg, metadata_xml_declaration))
-			else:
-				xml_version, xml_encoding, xml_standalone = \
-					xml_info["XML_DECLARATION"]
-				if xml_encoding is None or \
-					xml_encoding.upper() != metadata_xml_encoding:
-					if xml_encoding is None:
-						encoding_problem = "but it is undefined"
-					else:
-						encoding_problem = "not '%s'" % xml_encoding
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"xml declaration encoding should be '%s', %s" %
-						(xpkg, metadata_xml_encoding, encoding_problem))
+					"xml declaration encoding should be '%s', %s" %
+					(xpkg, metadata_xml_encoding, encoding_problem))
 
-			if "DOCTYPE" not in xml_info:
-				metadata_bad = True
+		if "DOCTYPE" not in xml_info:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
+		else:
+			doctype_name, doctype_system, doctype_pubid = \
+				xml_info["DOCTYPE"]
+			if doctype_system != metadata_dtd_uri:
+				if doctype_system is None:
+					system_problem = "but it is undefined"
+				else:
+					system_problem = "not '%s'" % doctype_system
 				self.qatracker.add_error(
-					"metadata.bad",
-					"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-			else:
-				doctype_name, doctype_system, doctype_pubid = \
-					xml_info["DOCTYPE"]
-				if doctype_system != metadata_dtd_uri:
-					if doctype_system is None:
-						system_problem = "but it is undefined"
-					else:
-						system_problem = "not '%s'" % doctype_system
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"DOCTYPE: SYSTEM should refer to '%s', %s" %
-						(xpkg, metadata_dtd_uri, system_problem))
+					"metadata.bad", "%s/metadata.xml: "
+					"DOCTYPE: SYSTEM should refer to '%s', %s" %
+					(xpkg, metadata_dtd_uri, system_problem))
 
-				if doctype_name != metadata_doctype_name:
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"DOCTYPE: name should be '%s', not '%s'" %
-						(xpkg, metadata_doctype_name, doctype_name))
+			if doctype_name != metadata_doctype_name:
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: "
+					"DOCTYPE: name should be '%s', not '%s'" %
+					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
 		try:

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     57d1ef7f8f98275b066735c66d0da05ea103a9da
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 18:46:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=57d1ef7f

repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class

This allows for a complete Q/A processing of all the use flags defined in the xml.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++++++++++------------
 1 file changed, 68 insertions(+), 70 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 1594b27..7117e7d 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import exception
 from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
@@ -43,55 +42,6 @@ metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 metadata_doctype_name = 'pkgmetadata'
 
 
-def parse_metadata_use(xml_tree):
-	"""
-	Records are wrapped in XML as per GLEP 56
-	returns a dict with keys constisting of USE flag names and values
-	containing their respective descriptions
-	"""
-	uselist = {}
-
-	usetags = xml_tree.findall("use")
-	if not usetags:
-		return uselist
-
-	# It's possible to have multiple 'use' elements.
-	for usetag in usetags:
-		flags = usetag.findall("flag")
-		if not flags:
-			# DTD allows use elements containing no flag elements.
-			continue
-
-		for flag in flags:
-			pkg_flag = flag.get("name")
-			if pkg_flag is None:
-				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
-			flag_restrict = flag.get("restrict")
-
-			# emulate the Element.itertext() method from python-2.7
-			inner_text = []
-			stack = []
-			stack.append(flag)
-			while stack:
-				obj = stack.pop()
-				if isinstance(obj, basestring):
-					inner_text.append(obj)
-					continue
-				if isinstance(obj.text, basestring):
-					inner_text.append(obj.text)
-				if isinstance(obj.tail, basestring):
-					stack.append(obj.tail)
-				stack.extend(reversed(obj))
-
-			if pkg_flag not in uselist:
-				uselist[pkg_flag] = {}
-
-			# (flag_restrict can be None)
-			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-	return uselist
-
-
 class PkgMetadata(ScanBase, USEFlagChecks):
 	'''Package metadata.xml checks'''
 
@@ -180,28 +130,23 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		try:
-			self.musedict = parse_metadata_use(_metadata_xml)
-		except portage.exception.ParseError as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-		else:
-			for atom in chain(*self.musedict.values()):
-				if atom is None:
-					continue
-				try:
-					atom = Atom(atom)
-				except InvalidAtom as e:
+		self.musedict, metadata_bad = self._parse_metadata_use(
+			_metadata_xml, xpkg, metadata_bad)
+		for atom in chain(*self.musedict.values()):
+			if atom is None:
+				continue
+			try:
+				atom = Atom(atom)
+			except InvalidAtom as e:
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+			else:
+				if atom.cp != xpkg:
 					self.qatracker.add_error(
 						"metadata.bad",
-						"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-				else:
-					if atom.cp != xpkg:
-						self.qatracker.add_error(
-							"metadata.bad",
-							"%s/metadata.xml: Atom contains "
-							"unexpected cat/pn: %s" % (xpkg, atom))
+						"%s/metadata.xml: Atom contains "
+						"unexpected cat/pn: %s" % (xpkg, atom))
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
@@ -237,6 +182,59 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
+	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+		"""
+		Records are wrapped in XML as per GLEP 56
+		returns a dict with keys constisting of USE flag names and values
+		containing their respective descriptions
+		"""
+		uselist = {}
+
+		usetags = xml_tree.findall("use")
+		if not usetags:
+			return uselist, metadata_bad
+
+		# It's possible to have multiple 'use' elements.
+		for usetag in usetags:
+			flags = usetag.findall("flag")
+			if not flags:
+				# DTD allows use elements containing no flag elements.
+				continue
+
+			for flag in flags:
+				pkg_flag = flag.get("name")
+				if pkg_flag is None:
+					metadata_bad = True
+					self.qatracker.add_error(
+						"metadata.bad",
+						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
+						% (xpkg, flag.sourceline, flag.text))
+					continue
+				flag_restrict = flag.get("restrict")
+
+				# emulate the Element.itertext() method from python-2.7
+				inner_text = []
+				stack = []
+				stack.append(flag)
+				while stack:
+					obj = stack.pop()
+					if isinstance(obj, basestring):
+						inner_text.append(obj)
+						continue
+					if isinstance(obj.text, basestring):
+						inner_text.append(obj.text)
+					if isinstance(obj.tail, basestring):
+						stack.append(obj.tail)
+					stack.extend(reversed(obj))
+
+				if pkg_flag not in uselist:
+					uselist[pkg_flag] = {}
+
+				# (flag_restrict can be None)
+				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
+
+		return uselist, metadata_bad
+
 	@property
 	def runInPkgs(self):
 		'''Package level scans'''

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     c6d6b003bc0545912f15c82a99c2e948a5c63e60
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 20:54:33 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c6d6b003

repoman/modules/.../pkgmetadata.py:  Have xml validation log all qatracker errors

Remove the use flag qatracker additions.
Add all logged XMLSchema errors to the qatracker .
This makes it a one run check to add all possible errors via the XMLSchema.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++++++++++-------------
 1 file changed, 36 insertions(+), 44 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7117e7d..83ca93f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -130,8 +130,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		self.musedict, metadata_bad = self._parse_metadata_use(
-			_metadata_xml, xpkg, metadata_bad)
+		self.musedict = self._parse_metadata_use(_metadata_xml, xpkg)
 		for atom in chain(*self.musedict.values()):
 			if atom is None:
 				continue
@@ -151,15 +150,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			try:
-				validator.assertValid(_metadata_xml)
-			except etree.DocumentInvalid as error:
-				self.qatracker.add_error(
-					"metadata.bad",
-					xpkg + "/metadata.xml: %s"
-					% (str(error))
-					)
-		del metadata_bad
+			if not validator.validate(_metadata_xml):
+				self._add_validate_errors(xpkg, validator.error_log)
 		self.muselist = frozenset(self.musedict)
 		return False
 
@@ -182,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
-	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+	def _parse_metadata_use(self, xml_tree, xpkg):
 		"""
 		Records are wrapped in XML as per GLEP 56
 		returns a dict with keys constisting of USE flag names and values
@@ -192,7 +184,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		usetags = xml_tree.findall("use")
 		if not usetags:
-			return uselist, metadata_bad
+			return uselist
 
 		# It's possible to have multiple 'use' elements.
 		for usetag in usetags:
@@ -203,37 +195,37 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 			for flag in flags:
 				pkg_flag = flag.get("name")
-				if pkg_flag is None:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad",
-						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
-						% (xpkg, flag.sourceline, flag.text))
-					continue
-				flag_restrict = flag.get("restrict")
-
-				# emulate the Element.itertext() method from python-2.7
-				inner_text = []
-				stack = []
-				stack.append(flag)
-				while stack:
-					obj = stack.pop()
-					if isinstance(obj, basestring):
-						inner_text.append(obj)
-						continue
-					if isinstance(obj.text, basestring):
-						inner_text.append(obj.text)
-					if isinstance(obj.tail, basestring):
-						stack.append(obj.tail)
-					stack.extend(reversed(obj))
-
-				if pkg_flag not in uselist:
-					uselist[pkg_flag] = {}
-
-				# (flag_restrict can be None)
-				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-		return uselist, metadata_bad
+				if pkg_flag is not None:
+					flag_restrict = flag.get("restrict")
+
+					# emulate the Element.itertext() method from python-2.7
+					inner_text = []
+					stack = []
+					stack.append(flag)
+					while stack:
+						obj = stack.pop()
+						if isinstance(obj, basestring):
+							inner_text.append(obj)
+							continue
+						if isinstance(obj.text, basestring):
+							inner_text.append(obj.text)
+						if isinstance(obj.tail, basestring):
+							stack.append(obj.tail)
+						stack.extend(reversed(obj))
+
+					if flag.get("name") not in uselist:
+						uselist[flag.get("name")] = {}
+
+					# (flag_restrict can be None)
+					uselist[flag.get("name")][flag_restrict] = " ".join("".join(inner_text).split())
+		return uselist
+
+	def _add_validate_errors(self, xpkg, log):
+		for error in log:
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: line: %s, %s"
+				% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     64bb8e17a5c050671765626bbe45174e10b5a3a4
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 17:38:19 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=64bb8e17

repoman/modules/.../pkgmetadata.py: Improve whole document validation

Change to using assertValid()  and add the error causing validation to fail to the qa tracker
error.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 4921b6f..1594b27 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -206,8 +206,14 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			if not validator.validate(_metadata_xml):
-				self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
+			try:
+				validator.assertValid(_metadata_xml)
+			except etree.DocumentInvalid as error:
+				self.qatracker.add_error(
+					"metadata.bad",
+					xpkg + "/metadata.xml: %s"
+					% (str(error))
+					)
 		del metadata_bad
 		self.muselist = frozenset(self.musedict)
 		return False

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     2eec213a7226380983f270276fef67f9c0337cab
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 07:39:09 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2eec213a

repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks

This change based on original work done by Dirkjan Ochtman <dirkjan <AT> ochtman.nl>

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3ca7897..4921b6f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception):
 # import our initialized portage instance
 from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
-from repoman.checks.herds.herdbase import get_herd_base
-from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
@@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 							"%s/metadata.xml: Atom contains "
 							"unexpected cat/pn: %s" % (xpkg, atom))
 
-		# Run other metadata.xml checkers
-		try:
-			check_metadata(_metadata_xml, get_herd_base(
-				self.repoman_settings))
-		except (UnknownHerdsError, ) as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-			del e
-
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     cfc31c04bf6e1d59b2756e3ac315a2040cdd162d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed May  4 06:08:50 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfc31c04

repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection

The docinfo is always created, so is not valid to use like a boolean to detect a missing
doctype.
doctype is a null string or equivalent and as such the correct choice for the if statement.

Failure discoverd by: Göktürk Yüksek <gokturk <AT> binghamton.edu> 
Thank you

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 83ca93f..d8344c2 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -106,7 +106,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 				"xml declaration encoding should be '%s', not '%s'" %
 				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if not _metadata_xml.docinfo:
+		if not _metadata_xml.docinfo.doctype:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     1f4d8908748dc59c362d655db3f6fe3b4c913476
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu May  5 16:04:54 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1f4d8908

repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors

Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..433551a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		return uselist
 
 	def _add_validate_errors(self, xpkg, log):
+		listed = set()
 		for error in log:
-			self.qatracker.add_error(
-				"metadata.bad",
-				"%s/metadata.xml: line: %s, %s"
-				% (xpkg, error.line, error.message))
+			msg_prefix = error.message.split(":",1)[0]
+			info = "%s %s" % (error.line, msg_prefix)
+			if info not in listed:
+				listed.add(info)
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: line: %s, %s"
+					% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):