[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     1fd217a14abd34e98c9bbee4ea5843b7460a6d4b
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:31:26 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Jan  6 04:08:21 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1fd217a1

scanner.py: Migrate another metadata check to ebuild_metadata

 pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 143a40e..2dc1db2 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -28,6 +28,9 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		if ebuild.metadata.get("PROVIDE"):
+			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+
 		return {'continue': False}
 
 	@property

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     4c9cb5cb9e90e2a576f4d4a37bafbea9372c3ebe
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Mar 15 18:45:45 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Mar 15 18:45:45 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=4c9cb5cb

repoman/modules.scan/metadata/unused.py: Add docstrings

Remove un-needed override functions

 pym/repoman/modules/scan/metadata/unused.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
index b3dc863..6def48f 100644
--- a/pym/repoman/modules/scan/metadata/unused.py
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -1,11 +1,23 @@
 
 
 class UnusedCheck(object):
+	'''Checks and reports any un-used metadata.xml use flag descriptions'''
 
 	def __init__(self, **kwargs):
+		'''UnusedCheck init function
+
+		@param qatracker: QATracker instance
+		'''
 		self.qatracker = kwargs.get('qatracker')
 
 	def check(self, **kwargs):
+		'''Reports on any unused metadata.xml use descriptions
+
+		@param xpkg: the pacakge being checked
+		@param muselist: use flag list
+		@param used_useflags: use flag list
+		@param validity_fuse: Fuse instance
+		'''
 		xpkg = kwargs.get('xpkg')
 		muselist = kwargs.get('muselist')
 		used_useflags = kwargs.get('used_useflags')
@@ -20,13 +32,6 @@ class UnusedCheck(object):
 		return {'continue': False}
 
 	@property
-	def runInPkgs(self):
-		return (False, [])
-
-	@property
-	def runInEbuilds(self):
-		return (False, [])
-
-	@property
 	def runInFinal(self):
+		'''Final scans at the package level'''
 		return (True, [self.check])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     01e09705312a9c94058a4ae12a6d01345dbc81d3
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Mar 15 18:45:45 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Apr 21 16:49:29 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=01e09705

repoman/modules.scan/metadata/unused.py: Add docstrings

Remove un-needed override functions

 pym/repoman/modules/scan/metadata/unused.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
index 9ff7e56..a58a614 100644
--- a/pym/repoman/modules/scan/metadata/unused.py
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -3,11 +3,23 @@ from repoman.modules.scan.scanbase import ScanBase
 
 
 class UnusedCheck(ScanBase):
+	'''Checks and reports any un-used metadata.xml use flag descriptions'''
 
 	def __init__(self, **kwargs):
+		'''UnusedCheck init function
+
+		@param qatracker: QATracker instance
+		'''
 		self.qatracker = kwargs.get('qatracker')
 
 	def check(self, **kwargs):
+		'''Reports on any unused metadata.xml use descriptions
+
+		@param xpkg: the pacakge being checked
+		@param muselist: use flag list
+		@param used_useflags: use flag list
+		@param validity_fuse: Fuse instance
+		'''
 		xpkg = kwargs.get('xpkg')
 		muselist = kwargs.get('muselist')
 		used_useflags = kwargs.get('used_useflags')
@@ -22,13 +34,6 @@ class UnusedCheck(ScanBase):
 		return {'continue': False}
 
 	@property
-	def runInPkgs(self):
-		return (False, [])
-
-	@property
-	def runInEbuilds(self):
-		return (False, [])
-
-	@property
 	def runInFinal(self):
+		'''Final scans at the package level'''
 		return (True, [self.check])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     926e2dc2975d1bea54454014cda714b426b3d5b9
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Mar 15 18:45:45 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Apr 25 15:28:53 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=926e2dc2

repoman/modules.scan/metadata/unused.py: Add docstrings

Remove un-needed override functions

 pym/repoman/modules/scan/metadata/unused.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
index 9ff7e56..a58a614 100644
--- a/pym/repoman/modules/scan/metadata/unused.py
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -3,11 +3,23 @@ from repoman.modules.scan.scanbase import ScanBase
 
 
 class UnusedCheck(ScanBase):
+	'''Checks and reports any un-used metadata.xml use flag descriptions'''
 
 	def __init__(self, **kwargs):
+		'''UnusedCheck init function
+
+		@param qatracker: QATracker instance
+		'''
 		self.qatracker = kwargs.get('qatracker')
 
 	def check(self, **kwargs):
+		'''Reports on any unused metadata.xml use descriptions
+
+		@param xpkg: the pacakge being checked
+		@param muselist: use flag list
+		@param used_useflags: use flag list
+		@param validity_fuse: Fuse instance
+		'''
 		xpkg = kwargs.get('xpkg')
 		muselist = kwargs.get('muselist')
 		used_useflags = kwargs.get('used_useflags')
@@ -22,13 +34,6 @@ class UnusedCheck(ScanBase):
 		return {'continue': False}
 
 	@property
-	def runInPkgs(self):
-		return (False, [])
-
-	@property
-	def runInEbuilds(self):
-		return (False, [])
-
-	@property
 	def runInFinal(self):
+		'''Final scans at the package level'''
 		return (True, [self.check])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     4ef6bc69a3fe62d980c285ac0b9bb5bbadeefe9b
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 06:33:47 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 06:33:47 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=4ef6bc69

repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 ++++++++++++-----------
 1 file changed, 75 insertions(+), 73 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3c1c2d0..3b48b8e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 			metadata_bad = True
 			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
 			del e
+			self.muselist = frozenset(self.musedict)
+			return False
+
+		if not hasattr(xml_parser, 'parser') or \
+			sys.hexversion < 0x2070000 or \
+			(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
+			# doctype is not parsed with python 2.6 or 3.1
+			pass
 		else:
-			if not hasattr(xml_parser, 'parser') or \
-				sys.hexversion < 0x2070000 or \
-				(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-				# doctype is not parsed with python 2.6 or 3.1
-				pass
+			if "XML_DECLARATION" not in xml_info:
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: "
+					"xml declaration is missing on first line, "
+					"should be '%s'" % (xpkg, metadata_xml_declaration))
 			else:
-				if "XML_DECLARATION" not in xml_info:
+				xml_version, xml_encoding, xml_standalone = \
+					xml_info["XML_DECLARATION"]
+				if xml_encoding is None or \
+					xml_encoding.upper() != metadata_xml_encoding:
+					if xml_encoding is None:
+						encoding_problem = "but it is undefined"
+					else:
+						encoding_problem = "not '%s'" % xml_encoding
 					self.qatracker.add_error(
 						"metadata.bad", "%s/metadata.xml: "
-						"xml declaration is missing on first line, "
-						"should be '%s'" % (xpkg, metadata_xml_declaration))
-				else:
-					xml_version, xml_encoding, xml_standalone = \
-						xml_info["XML_DECLARATION"]
-					if xml_encoding is None or \
-						xml_encoding.upper() != metadata_xml_encoding:
-						if xml_encoding is None:
-							encoding_problem = "but it is undefined"
-						else:
-							encoding_problem = "not '%s'" % xml_encoding
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"xml declaration encoding should be '%s', %s" %
-							(xpkg, metadata_xml_encoding, encoding_problem))
+						"xml declaration encoding should be '%s', %s" %
+						(xpkg, metadata_xml_encoding, encoding_problem))
 
-				if "DOCTYPE" not in xml_info:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad",
-						"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-				else:
-					doctype_name, doctype_system, doctype_pubid = \
-						xml_info["DOCTYPE"]
-					if doctype_system != metadata_dtd_uri:
-						if doctype_system is None:
-							system_problem = "but it is undefined"
-						else:
-							system_problem = "not '%s'" % doctype_system
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"DOCTYPE: SYSTEM should refer to '%s', %s" %
-							(xpkg, metadata_dtd_uri, system_problem))
-
-					if doctype_name != metadata_doctype_name:
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"DOCTYPE: name should be '%s', not '%s'" %
-							(xpkg, metadata_doctype_name, doctype_name))
-
-			# load USE flags from metadata.xml
-			try:
-				self.musedict = parse_metadata_use(_metadata_xml)
-			except portage.exception.ParseError as e:
+			if "DOCTYPE" not in xml_info:
 				metadata_bad = True
 				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+					"metadata.bad",
+					"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 			else:
-				for atom in chain(*self.musedict.values()):
-					if atom is None:
-						continue
-					try:
-						atom = Atom(atom)
-					except InvalidAtom as e:
+				doctype_name, doctype_system, doctype_pubid = \
+					xml_info["DOCTYPE"]
+				if doctype_system != metadata_dtd_uri:
+					if doctype_system is None:
+						system_problem = "but it is undefined"
+					else:
+						system_problem = "not '%s'" % doctype_system
+					self.qatracker.add_error(
+						"metadata.bad", "%s/metadata.xml: "
+						"DOCTYPE: SYSTEM should refer to '%s', %s" %
+						(xpkg, metadata_dtd_uri, system_problem))
+
+				if doctype_name != metadata_doctype_name:
+					self.qatracker.add_error(
+						"metadata.bad", "%s/metadata.xml: "
+						"DOCTYPE: name should be '%s', not '%s'" %
+						(xpkg, metadata_doctype_name, doctype_name))
+
+		# load USE flags from metadata.xml
+		try:
+			self.musedict = parse_metadata_use(_metadata_xml)
+		except portage.exception.ParseError as e:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+		else:
+			for atom in chain(*self.musedict.values()):
+				if atom is None:
+					continue
+				try:
+					atom = Atom(atom)
+				except InvalidAtom as e:
+					self.qatracker.add_error(
+						"metadata.bad",
+						"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+				else:
+					if atom.cp != xpkg:
 						self.qatracker.add_error(
 							"metadata.bad",
-							"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-					else:
-						if atom.cp != xpkg:
-							self.qatracker.add_error(
-								"metadata.bad",
-								"%s/metadata.xml: Atom contains "
-								"unexpected cat/pn: %s" % (xpkg, atom))
-
-			# Run other metadata.xml checkers
-			try:
-				check_metadata(_metadata_xml, get_herd_base(
-					self.repoman_settings))
-			except (UnknownHerdsError, ) as e:
-				metadata_bad = True
-				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-				del e
+							"%s/metadata.xml: Atom contains "
+							"unexpected cat/pn: %s" % (xpkg, atom))
+
+		# Run other metadata.xml checkers
+		try:
+			check_metadata(_metadata_xml, get_herd_base(
+				self.repoman_settings))
+		except (UnknownHerdsError, ) as e:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+			del e
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     e66ee7b35405a650c517939e503443948b660ca9
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 18:33:32 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 06:25:53 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e66ee7b3

repoman/modules/.../pkgmetadata.py: Early return in metadata check code for readability

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 190 ++++++++++++-----------
 1 file changed, 96 insertions(+), 94 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 5c6452a..3c1c2d0 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -83,112 +83,114 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# metadata.xml file check
 		if "metadata.xml" not in checkdirlist:
 			self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")
+			self.muselist = frozenset(self.musedict)
+			return False
+
 		# metadata.xml parse check
+		metadata_bad = False
+		xml_info = {}
+		xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
+
+		# read metadata.xml into memory
+		try:
+			_metadata_xml = xml.etree.ElementTree.parse(
+				_unicode_encode(
+					os.path.join(checkdir, "metadata.xml"),
+					encoding=_encodings['fs'], errors='strict'),
+				parser=xml_parser)
+		except (ExpatError, SyntaxError, EnvironmentError) as e:
+			metadata_bad = True
+			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+			del e
 		else:
-			metadata_bad = False
-			xml_info = {}
-			xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
-
-			# read metadata.xml into memory
-			try:
-				_metadata_xml = xml.etree.ElementTree.parse(
-					_unicode_encode(
-						os.path.join(checkdir, "metadata.xml"),
-						encoding=_encodings['fs'], errors='strict'),
-					parser=xml_parser)
-			except (ExpatError, SyntaxError, EnvironmentError) as e:
-				metadata_bad = True
-				self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-				del e
+			if not hasattr(xml_parser, 'parser') or \
+				sys.hexversion < 0x2070000 or \
+				(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
+				# doctype is not parsed with python 2.6 or 3.1
+				pass
 			else:
-				if not hasattr(xml_parser, 'parser') or \
-					sys.hexversion < 0x2070000 or \
-					(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-					# doctype is not parsed with python 2.6 or 3.1
-					pass
+				if "XML_DECLARATION" not in xml_info:
+					self.qatracker.add_error(
+						"metadata.bad", "%s/metadata.xml: "
+						"xml declaration is missing on first line, "
+						"should be '%s'" % (xpkg, metadata_xml_declaration))
 				else:
-					if "XML_DECLARATION" not in xml_info:
+					xml_version, xml_encoding, xml_standalone = \
+						xml_info["XML_DECLARATION"]
+					if xml_encoding is None or \
+						xml_encoding.upper() != metadata_xml_encoding:
+						if xml_encoding is None:
+							encoding_problem = "but it is undefined"
+						else:
+							encoding_problem = "not '%s'" % xml_encoding
 						self.qatracker.add_error(
 							"metadata.bad", "%s/metadata.xml: "
-							"xml declaration is missing on first line, "
-							"should be '%s'" % (xpkg, metadata_xml_declaration))
-					else:
-						xml_version, xml_encoding, xml_standalone = \
-							xml_info["XML_DECLARATION"]
-						if xml_encoding is None or \
-							xml_encoding.upper() != metadata_xml_encoding:
-							if xml_encoding is None:
-								encoding_problem = "but it is undefined"
-							else:
-								encoding_problem = "not '%s'" % xml_encoding
-							self.qatracker.add_error(
-								"metadata.bad", "%s/metadata.xml: "
-								"xml declaration encoding should be '%s', %s" %
-								(xpkg, metadata_xml_encoding, encoding_problem))
-
-					if "DOCTYPE" not in xml_info:
-						metadata_bad = True
-						self.qatracker.add_error(
-							"metadata.bad",
-							"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-					else:
-						doctype_name, doctype_system, doctype_pubid = \
-							xml_info["DOCTYPE"]
-						if doctype_system != metadata_dtd_uri:
-							if doctype_system is None:
-								system_problem = "but it is undefined"
-							else:
-								system_problem = "not '%s'" % doctype_system
-							self.qatracker.add_error(
-								"metadata.bad", "%s/metadata.xml: "
-								"DOCTYPE: SYSTEM should refer to '%s', %s" %
-								(xpkg, metadata_dtd_uri, system_problem))
+							"xml declaration encoding should be '%s', %s" %
+							(xpkg, metadata_xml_encoding, encoding_problem))
 
-						if doctype_name != metadata_doctype_name:
-							self.qatracker.add_error(
-								"metadata.bad", "%s/metadata.xml: "
-								"DOCTYPE: name should be '%s', not '%s'" %
-								(xpkg, metadata_doctype_name, doctype_name))
-
-				# load USE flags from metadata.xml
-				try:
-					self.musedict = parse_metadata_use(_metadata_xml)
-				except portage.exception.ParseError as e:
+				if "DOCTYPE" not in xml_info:
 					metadata_bad = True
 					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+						"metadata.bad",
+						"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 				else:
-					for atom in chain(*self.musedict.values()):
-						if atom is None:
-							continue
-						try:
-							atom = Atom(atom)
-						except InvalidAtom as e:
+					doctype_name, doctype_system, doctype_pubid = \
+						xml_info["DOCTYPE"]
+					if doctype_system != metadata_dtd_uri:
+						if doctype_system is None:
+							system_problem = "but it is undefined"
+						else:
+							system_problem = "not '%s'" % doctype_system
+						self.qatracker.add_error(
+							"metadata.bad", "%s/metadata.xml: "
+							"DOCTYPE: SYSTEM should refer to '%s', %s" %
+							(xpkg, metadata_dtd_uri, system_problem))
+
+					if doctype_name != metadata_doctype_name:
+						self.qatracker.add_error(
+							"metadata.bad", "%s/metadata.xml: "
+							"DOCTYPE: name should be '%s', not '%s'" %
+							(xpkg, metadata_doctype_name, doctype_name))
+
+			# load USE flags from metadata.xml
+			try:
+				self.musedict = parse_metadata_use(_metadata_xml)
+			except portage.exception.ParseError as e:
+				metadata_bad = True
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+			else:
+				for atom in chain(*self.musedict.values()):
+					if atom is None:
+						continue
+					try:
+						atom = Atom(atom)
+					except InvalidAtom as e:
+						self.qatracker.add_error(
+							"metadata.bad",
+							"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+					else:
+						if atom.cp != xpkg:
 							self.qatracker.add_error(
 								"metadata.bad",
-								"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-						else:
-							if atom.cp != xpkg:
-								self.qatracker.add_error(
-									"metadata.bad",
-									"%s/metadata.xml: Atom contains "
-									"unexpected cat/pn: %s" % (xpkg, atom))
-
-				# Run other metadata.xml checkers
-				try:
-					check_metadata(_metadata_xml, get_herd_base(
-						self.repoman_settings))
-				except (UnknownHerdsError, ) as e:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-					del e
-
-			# Only carry out if in package directory or check forced
-			if not metadata_bad:
-				if not self.xmllint.check(checkdir, repolevel):
-					self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
-			del metadata_bad
+								"%s/metadata.xml: Atom contains "
+								"unexpected cat/pn: %s" % (xpkg, atom))
+
+			# Run other metadata.xml checkers
+			try:
+				check_metadata(_metadata_xml, get_herd_base(
+					self.repoman_settings))
+			except (UnknownHerdsError, ) as e:
+				metadata_bad = True
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+				del e
+
+		# Only carry out if in package directory or check forced
+		if not metadata_bad:
+			if not self.xmllint.check(checkdir, repolevel):
+				self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
+		del metadata_bad
 		self.muselist = frozenset(self.musedict)
 		return False
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     6ca71f4c66e45d86503f1cc199e1547e19b35a2e
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 07:39:09 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 09:12:55 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ca71f4c

repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks

This change based on original work done by Dirkjan Ochtman <dirkjan <AT> ochtman.nl>

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 22afddf..c744c13 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception):
 # import our initialized portage instance
 from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
-from repoman.checks.herds.herdbase import get_herd_base
-from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
 from repoman._xml import XmlLint
 from repoman.modules.scan.scanbase import ScanBase
 
@@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 							"%s/metadata.xml: Atom contains "
 							"unexpected cat/pn: %s" % (xpkg, atom))
 
-		# Run other metadata.xml checkers
-		try:
-			check_metadata(_metadata_xml, get_herd_base(
-				self.repoman_settings))
-		except (UnknownHerdsError, ) as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-			del e
-
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			if not self.xmllint.check(checkdir, repolevel):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     57ad076b08d417bf5d2642131b76392a990f3c35
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 19:45:31 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 06:55:53 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=57ad076b

repoman: mandate use of python 2.7 or 3.2+ for simplicity

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++++++++++-------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index cde2ba0..e8db92f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -158,54 +158,48 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if not hasattr(xml_parser, 'parser') or \
-			sys.hexversion < 0x2070000 or \
-			(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-			# doctype is not parsed with python 2.6 or 3.1
-			pass
+		if "XML_DECLARATION" not in xml_info:
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: "
+				"xml declaration is missing on first line, "
+				"should be '%s'" % (xpkg, metadata_xml_declaration))
 		else:
-			if "XML_DECLARATION" not in xml_info:
+			xml_version, xml_encoding, xml_standalone = \
+				xml_info["XML_DECLARATION"]
+			if xml_encoding is None or \
+				xml_encoding.upper() != metadata_xml_encoding:
+				if xml_encoding is None:
+					encoding_problem = "but it is undefined"
+				else:
+					encoding_problem = "not '%s'" % xml_encoding
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration is missing on first line, "
-					"should be '%s'" % (xpkg, metadata_xml_declaration))
-			else:
-				xml_version, xml_encoding, xml_standalone = \
-					xml_info["XML_DECLARATION"]
-				if xml_encoding is None or \
-					xml_encoding.upper() != metadata_xml_encoding:
-					if xml_encoding is None:
-						encoding_problem = "but it is undefined"
-					else:
-						encoding_problem = "not '%s'" % xml_encoding
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"xml declaration encoding should be '%s', %s" %
-						(xpkg, metadata_xml_encoding, encoding_problem))
+					"xml declaration encoding should be '%s', %s" %
+					(xpkg, metadata_xml_encoding, encoding_problem))
 
-			if "DOCTYPE" not in xml_info:
-				metadata_bad = True
+		if "DOCTYPE" not in xml_info:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
+		else:
+			doctype_name, doctype_system, doctype_pubid = \
+				xml_info["DOCTYPE"]
+			if doctype_system != metadata_dtd_uri:
+				if doctype_system is None:
+					system_problem = "but it is undefined"
+				else:
+					system_problem = "not '%s'" % doctype_system
 				self.qatracker.add_error(
-					"metadata.bad",
-					"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-			else:
-				doctype_name, doctype_system, doctype_pubid = \
-					xml_info["DOCTYPE"]
-				if doctype_system != metadata_dtd_uri:
-					if doctype_system is None:
-						system_problem = "but it is undefined"
-					else:
-						system_problem = "not '%s'" % doctype_system
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"DOCTYPE: SYSTEM should refer to '%s', %s" %
-						(xpkg, metadata_dtd_uri, system_problem))
+					"metadata.bad", "%s/metadata.xml: "
+					"DOCTYPE: SYSTEM should refer to '%s', %s" %
+					(xpkg, metadata_dtd_uri, system_problem))
 
-				if doctype_name != metadata_doctype_name:
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"DOCTYPE: name should be '%s', not '%s'" %
-						(xpkg, metadata_doctype_name, doctype_name))
+			if doctype_name != metadata_doctype_name:
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: "
+					"DOCTYPE: name should be '%s', not '%s'" %
+					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
 		try:

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     75b897d91874c505e14dc2c0808c4cad4268ad76
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 07:18:05 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 09:12:30 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=75b897d9

repoman: Use lxml for parsing of metadata

Note that we no longer throw a QA error for a missing XML prolog, as long as
the encoding matches the default ('UTF-8'; lowercase is also allowed).

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 44 +++++++-----------------
 1 file changed, 12 insertions(+), 32 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index e8db92f..22afddf 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -7,8 +7,8 @@ import sys
 from itertools import chain
 
 try:
-	import xml.etree.ElementTree
-	from xml.parsers.expat import ExpatError
+	from lxml import etree
+	from lxml.etree import ParserError
 except (SystemExit, KeyboardInterrupt):
 	raise
 except (ImportError, SystemError, RuntimeError, Exception):
@@ -26,12 +26,11 @@ from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
+from repoman._xml import XmlLint
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import _encodings, _unicode_encode
 from portage import exception
 from portage.dep import Atom
 
@@ -141,50 +140,31 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		# metadata.xml parse check
 		metadata_bad = False
-		xml_info = {}
-		xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
 
 		# read metadata.xml into memory
 		try:
-			_metadata_xml = xml.etree.ElementTree.parse(
-				_unicode_encode(
-					os.path.join(checkdir, "metadata.xml"),
-					encoding=_encodings['fs'], errors='strict'),
-				parser=xml_parser)
-		except (ExpatError, SyntaxError, EnvironmentError) as e:
+			_metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml'))
+		except (ParserError, SyntaxError, EnvironmentError) as e:
 			metadata_bad = True
 			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
 			del e
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if "XML_DECLARATION" not in xml_info:
+		xml_encoding = _metadata_xml.docinfo.encoding
+		if xml_encoding.upper() != metadata_xml_encoding:
 			self.qatracker.add_error(
 				"metadata.bad", "%s/metadata.xml: "
-				"xml declaration is missing on first line, "
-				"should be '%s'" % (xpkg, metadata_xml_declaration))
-		else:
-			xml_version, xml_encoding, xml_standalone = \
-				xml_info["XML_DECLARATION"]
-			if xml_encoding is None or \
-				xml_encoding.upper() != metadata_xml_encoding:
-				if xml_encoding is None:
-					encoding_problem = "but it is undefined"
-				else:
-					encoding_problem = "not '%s'" % xml_encoding
-				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration encoding should be '%s', %s" %
-					(xpkg, metadata_xml_encoding, encoding_problem))
+				"xml declaration encoding should be '%s', not '%s'" %
+				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if "DOCTYPE" not in xml_info:
+		if not _metadata_xml.docinfo:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",
 				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 		else:
-			doctype_name, doctype_system, doctype_pubid = \
-				xml_info["DOCTYPE"]
+			doctype_system = _metadata_xml.docinfo.system_url
 			if doctype_system != metadata_dtd_uri:
 				if doctype_system is None:
 					system_problem = "but it is undefined"
@@ -194,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					"metadata.bad", "%s/metadata.xml: "
 					"DOCTYPE: SYSTEM should refer to '%s', %s" %
 					(xpkg, metadata_dtd_uri, system_problem))
-
+			doctype_name = _metadata_xml.docinfo.doctype.split(' ')[1]
 			if doctype_name != metadata_doctype_name:
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     621a6852099848474ed2c0880eae52ebd85f1703
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 17:38:19 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 17:38:19 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=621a6852

repoman/modules/.../pkgmetadata.py: Improve whole document validation

Change to using assertValid()  and add the error causing validation to fail to the qa tracker
error.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index b231370..7390b5b 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -202,8 +202,14 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			if not validator.validate(_metadata_xml):
-				self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
+			try:
+				validator.assertValid(_metadata_xml)
+			except etree.DocumentInvalid as error:
+				self.qatracker.add_error(
+					"metadata.bad",
+					xpkg + "/metadata.xml: %s"
+					% (str(error))
+					)
 		del metadata_bad
 		self.muselist = frozenset(self.musedict)
 		return False

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     235235c69343e95793525ee2104a82977a5a6729
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 17:42:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 17:42:22 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=235235c6

repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class

This allows for a complete Q/A processing of all the use flags defined in the xml.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 139 +++++++++++------------
 1 file changed, 68 insertions(+), 71 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7390b5b..ef57a5d 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import exception
 from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
@@ -39,55 +38,6 @@ metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 metadata_doctype_name = 'pkgmetadata'
 
 
-def parse_metadata_use(xml_tree):
-	"""
-	Records are wrapped in XML as per GLEP 56
-	returns a dict with keys constisting of USE flag names and values
-	containing their respective descriptions
-	"""
-	uselist = {}
-
-	usetags = xml_tree.findall("use")
-	if not usetags:
-		return uselist
-
-	# It's possible to have multiple 'use' elements.
-	for usetag in usetags:
-		flags = usetag.findall("flag")
-		if not flags:
-			# DTD allows use elements containing no flag elements.
-			continue
-
-		for flag in flags:
-			pkg_flag = flag.get("name")
-			if pkg_flag is None:
-				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
-			flag_restrict = flag.get("restrict")
-
-			# emulate the Element.itertext() method from python-2.7
-			inner_text = []
-			stack = []
-			stack.append(flag)
-			while stack:
-				obj = stack.pop()
-				if isinstance(obj, basestring):
-					inner_text.append(obj)
-					continue
-				if isinstance(obj.text, basestring):
-					inner_text.append(obj.text)
-				if isinstance(obj.tail, basestring):
-					stack.append(obj.tail)
-				stack.extend(reversed(obj))
-
-			if pkg_flag not in uselist:
-				uselist[pkg_flag] = {}
-
-			# (flag_restrict can be None)
-			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-	return uselist
-
-
 class PkgMetadata(ScanBase, USEFlagChecks):
 	'''Package metadata.xml checks'''
 
@@ -176,32 +126,26 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		try:
-			self.musedict = parse_metadata_use(_metadata_xml)
-		except portage.exception.ParseError as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-		else:
-			for atom in chain(*self.musedict.values()):
-				if atom is None:
-					continue
-				try:
-					atom = Atom(atom)
-				except InvalidAtom as e:
+		self.musedict, metadata_bad = self._parse_metadata_use(
+			_metadata_xml, xpkg, metadata_bad)
+		for atom in chain(*self.musedict.values()):
+			if atom is None:
+				continue
+			try:
+				atom = Atom(atom)
+			except InvalidAtom as e:
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+			else:
+				if atom.cp != xpkg:
 					self.qatracker.add_error(
 						"metadata.bad",
-						"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-				else:
-					if atom.cp != xpkg:
-						self.qatracker.add_error(
-							"metadata.bad",
-							"%s/metadata.xml: Atom contains "
-							"unexpected cat/pn: %s" % (xpkg, atom))
+						"%s/metadata.xml: Atom contains "
+						"unexpected cat/pn: %s" % (xpkg, atom))
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
-			validator = etree.XMLSchema(file=self.metadata_xsd)
 			try:
 				validator.assertValid(_metadata_xml)
 			except etree.DocumentInvalid as error:
@@ -233,6 +177,59 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
+	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+		"""
+		Records are wrapped in XML as per GLEP 56
+		returns a dict with keys constisting of USE flag names and values
+		containing their respective descriptions
+		"""
+		uselist = {}
+
+		usetags = xml_tree.findall("use")
+		if not usetags:
+			return uselist, metadata_bad
+
+		# It's possible to have multiple 'use' elements.
+		for usetag in usetags:
+			flags = usetag.findall("flag")
+			if not flags:
+				# DTD allows use elements containing no flag elements.
+				continue
+
+			for flag in flags:
+				pkg_flag = flag.get("name")
+				if pkg_flag is None:
+					metadata_bad = True
+					self.qatracker.add_error(
+						"metadata.bad",
+						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
+						% (xpkg, flag.sourceline, flag.text))
+					continue
+				flag_restrict = flag.get("restrict")
+
+				# emulate the Element.itertext() method from python-2.7
+				inner_text = []
+				stack = []
+				stack.append(flag)
+				while stack:
+					obj = stack.pop()
+					if isinstance(obj, basestring):
+						inner_text.append(obj)
+						continue
+					if isinstance(obj.text, basestring):
+						inner_text.append(obj.text)
+					if isinstance(obj.tail, basestring):
+						stack.append(obj.tail)
+					stack.extend(reversed(obj))
+
+				if pkg_flag not in uselist:
+					uselist[pkg_flag] = {}
+
+				# (flag_restrict can be None)
+				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
+
+		return uselist, metadata_bad
+
 	@property
 	def runInPkgs(self):
 		'''Package level scans'''

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     bf7dc3159db2c15ff7aa61c2c72b143bbd420be0
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 20:54:33 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 20:54:33 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf7dc315

repoman/modules/.../pkgmetadata.py:  Have xml validation log all qatracker errors

Remove the use flag qatracker additions.
Add all logged XMLSchema errors to the qatracker .
This makes it a one run check to add all possible errors via the XMLSchema.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++++++++++-------------
 1 file changed, 36 insertions(+), 44 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d6d8557..44b5edd 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -126,8 +126,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		self.musedict, metadata_bad = self._parse_metadata_use(
-			_metadata_xml, xpkg, metadata_bad)
+		self.musedict = self._parse_metadata_use(_metadata_xml, xpkg)
 		for atom in chain(*self.musedict.values()):
 			if atom is None:
 				continue
@@ -147,15 +146,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			try:
-				validator.assertValid(_metadata_xml)
-			except etree.DocumentInvalid as error:
-				self.qatracker.add_error(
-					"metadata.bad",
-					xpkg + "/metadata.xml: %s"
-					% (str(error))
-					)
-		del metadata_bad
+			if not validator.validate(_metadata_xml):
+				self._add_validate_errors(xpkg, validator.error_log)
 		self.muselist = frozenset(self.musedict)
 		return False
 
@@ -178,7 +170,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
-	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+	def _parse_metadata_use(self, xml_tree, xpkg):
 		"""
 		Records are wrapped in XML as per GLEP 56
 		returns a dict with keys constisting of USE flag names and values
@@ -188,7 +180,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		usetags = xml_tree.findall("use")
 		if not usetags:
-			return uselist, metadata_bad
+			return uselist
 
 		# It's possible to have multiple 'use' elements.
 		for usetag in usetags:
@@ -199,37 +191,37 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 			for flag in flags:
 				pkg_flag = flag.get("name")
-				if pkg_flag is None:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad",
-						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
-						% (xpkg, flag.sourceline, flag.text))
-					continue
-				flag_restrict = flag.get("restrict")
-
-				# emulate the Element.itertext() method from python-2.7
-				inner_text = []
-				stack = []
-				stack.append(flag)
-				while stack:
-					obj = stack.pop()
-					if isinstance(obj, basestring):
-						inner_text.append(obj)
-						continue
-					if isinstance(obj.text, basestring):
-						inner_text.append(obj.text)
-					if isinstance(obj.tail, basestring):
-						stack.append(obj.tail)
-					stack.extend(reversed(obj))
-
-				if pkg_flag not in uselist:
-					uselist[pkg_flag] = {}
-
-				# (flag_restrict can be None)
-				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-		return uselist, metadata_bad
+				if pkg_flag is not None:
+					flag_restrict = flag.get("restrict")
+
+					# emulate the Element.itertext() method from python-2.7
+					inner_text = []
+					stack = []
+					stack.append(flag)
+					while stack:
+						obj = stack.pop()
+						if isinstance(obj, basestring):
+							inner_text.append(obj)
+							continue
+						if isinstance(obj.text, basestring):
+							inner_text.append(obj.text)
+						if isinstance(obj.tail, basestring):
+							stack.append(obj.tail)
+						stack.extend(reversed(obj))
+
+					if flag.get("name") not in uselist:
+						uselist[flag.get("name")] = {}
+
+					# (flag_restrict can be None)
+					uselist[flag.get("name")][flag_restrict] = " ".join("".join(inner_text).split())
+		return uselist
+
+	def _add_validate_errors(self, xpkg, log):
+		for error in log:
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: line: %s, %s"
+				% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     50de92c080ac8e5bc3f0d6cd2bc69fb919afc186
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 18:46:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 18:46:22 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=50de92c0

repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class

This allows for a complete Q/A processing of all the use flags defined in the xml.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++++++++++------------
 1 file changed, 68 insertions(+), 70 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7390b5b..d6d8557 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import exception
 from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
@@ -39,55 +38,6 @@ metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 metadata_doctype_name = 'pkgmetadata'
 
 
-def parse_metadata_use(xml_tree):
-	"""
-	Records are wrapped in XML as per GLEP 56
-	returns a dict with keys constisting of USE flag names and values
-	containing their respective descriptions
-	"""
-	uselist = {}
-
-	usetags = xml_tree.findall("use")
-	if not usetags:
-		return uselist
-
-	# It's possible to have multiple 'use' elements.
-	for usetag in usetags:
-		flags = usetag.findall("flag")
-		if not flags:
-			# DTD allows use elements containing no flag elements.
-			continue
-
-		for flag in flags:
-			pkg_flag = flag.get("name")
-			if pkg_flag is None:
-				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
-			flag_restrict = flag.get("restrict")
-
-			# emulate the Element.itertext() method from python-2.7
-			inner_text = []
-			stack = []
-			stack.append(flag)
-			while stack:
-				obj = stack.pop()
-				if isinstance(obj, basestring):
-					inner_text.append(obj)
-					continue
-				if isinstance(obj.text, basestring):
-					inner_text.append(obj.text)
-				if isinstance(obj.tail, basestring):
-					stack.append(obj.tail)
-				stack.extend(reversed(obj))
-
-			if pkg_flag not in uselist:
-				uselist[pkg_flag] = {}
-
-			# (flag_restrict can be None)
-			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-	return uselist
-
-
 class PkgMetadata(ScanBase, USEFlagChecks):
 	'''Package metadata.xml checks'''
 
@@ -176,28 +126,23 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		try:
-			self.musedict = parse_metadata_use(_metadata_xml)
-		except portage.exception.ParseError as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-		else:
-			for atom in chain(*self.musedict.values()):
-				if atom is None:
-					continue
-				try:
-					atom = Atom(atom)
-				except InvalidAtom as e:
+		self.musedict, metadata_bad = self._parse_metadata_use(
+			_metadata_xml, xpkg, metadata_bad)
+		for atom in chain(*self.musedict.values()):
+			if atom is None:
+				continue
+			try:
+				atom = Atom(atom)
+			except InvalidAtom as e:
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+			else:
+				if atom.cp != xpkg:
 					self.qatracker.add_error(
 						"metadata.bad",
-						"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-				else:
-					if atom.cp != xpkg:
-						self.qatracker.add_error(
-							"metadata.bad",
-							"%s/metadata.xml: Atom contains "
-							"unexpected cat/pn: %s" % (xpkg, atom))
+						"%s/metadata.xml: Atom contains "
+						"unexpected cat/pn: %s" % (xpkg, atom))
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
@@ -233,6 +178,59 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
+	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+		"""
+		Records are wrapped in XML as per GLEP 56
+		returns a dict with keys constisting of USE flag names and values
+		containing their respective descriptions
+		"""
+		uselist = {}
+
+		usetags = xml_tree.findall("use")
+		if not usetags:
+			return uselist, metadata_bad
+
+		# It's possible to have multiple 'use' elements.
+		for usetag in usetags:
+			flags = usetag.findall("flag")
+			if not flags:
+				# DTD allows use elements containing no flag elements.
+				continue
+
+			for flag in flags:
+				pkg_flag = flag.get("name")
+				if pkg_flag is None:
+					metadata_bad = True
+					self.qatracker.add_error(
+						"metadata.bad",
+						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
+						% (xpkg, flag.sourceline, flag.text))
+					continue
+				flag_restrict = flag.get("restrict")
+
+				# emulate the Element.itertext() method from python-2.7
+				inner_text = []
+				stack = []
+				stack.append(flag)
+				while stack:
+					obj = stack.pop()
+					if isinstance(obj, basestring):
+						inner_text.append(obj)
+						continue
+					if isinstance(obj.text, basestring):
+						inner_text.append(obj.text)
+					if isinstance(obj.tail, basestring):
+						stack.append(obj.tail)
+					stack.extend(reversed(obj))
+
+				if pkg_flag not in uselist:
+					uselist[pkg_flag] = {}
+
+				# (flag_restrict can be None)
+				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
+
+		return uselist, metadata_bad
+
 	@property
 	def runInPkgs(self):
 		'''Package level scans'''

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     378e5458a9a68231c8b7abc5df375eff8062a402
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed May  4 03:44:17 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed May  4 03:44:17 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=378e5458

repoman/modules/.../pkgmetadata.py: Update import error message for lxml pkg

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 44b5edd..c76922b 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -14,7 +14,7 @@ except (SystemExit, KeyboardInterrupt):
 except (ImportError, SystemError, RuntimeError, Exception):
 	# broken or missing xml support
 	# http://bugs.python.org/issue14988
-	msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
+	msg = ["Please emerge dev-python/lxml in order to use repoman."]
 	from portage.output import EOutput
 	out = EOutput()
 	for line in msg:

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     55943b69a42c292bc0922a22d71f9d992ea035e7
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed May  4 06:08:50 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed May  4 06:08:50 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=55943b69

repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection

The docinfo is always created, so is not valid to use like a boolean to detect a missing
doctype.
doctype is a null string or equivalent and as such the correct choice for the if statement.

Failure discoverd by: Göktürk Yüksek <gokturk <AT> binghamton.edu> 
Thank you

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index c76922b..8b31e3f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -102,7 +102,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 				"xml declaration encoding should be '%s', not '%s'" %
 				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if not _metadata_xml.docinfo:
+		if not _metadata_xml.docinfo.doctype:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     9357614944df633df54871bc732d37abfe3a035f
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed May  4 06:23:05 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed May  4 06:23:05 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=93576149

repoman/modules/.../pkgmetadata.py: Add missing basestring compat code

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 8b31e3f..d8344c2 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -32,6 +32,10 @@ from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
 
+if sys.hexversion >= 0x3000000:
+	# pylint: disable=W0622
+	basestring = str
+
 metadata_xml_encoding = 'UTF-8'
 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 	% (metadata_xml_encoding,)

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     afefcbc188ea25540d72224034668207d9f4e024
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu May  5 16:01:02 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu May  5 16:01:02 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=afefcbc1

repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors

Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..81e435e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		return uselist
 
 	def _add_validate_errors(self, xpkg, log):
+		listed = set()
 		for error in log:
-			self.qatracker.add_error(
-				"metadata.bad",
-				"%s/metadata.xml: line: %s, %s"
-				% (xpkg, error.line, error.message))
+			msg_prefix = error.message.split(":",1)[0]
+			info = "%s %s" % (error.line, msg_prefix)
+			if info not in listed:
+				listed.add(info)
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: line: %s, %s, %s"
+					% (xpkg, error.line, error.type, error.message))
 
 	@property
 	def runInPkgs(self):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     6a61a12b827a0fbcdf6c111f936d8f17f6b10a33
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu May  5 16:04:54 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu May  5 16:04:54 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6a61a12b

repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors

Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..433551a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		return uselist
 
 	def _add_validate_errors(self, xpkg, log):
+		listed = set()
 		for error in log:
-			self.qatracker.add_error(
-				"metadata.bad",
-				"%s/metadata.xml: line: %s, %s"
-				% (xpkg, error.line, error.message))
+			msg_prefix = error.message.split(":",1)[0]
+			info = "%s %s" % (error.line, msg_prefix)
+			if info not in listed:
+				listed.add(info)
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: line: %s, %s"
+					% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     5f15e81967f8dca211cbd87d8696059aeac9ada2
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Tue May  3 06:33:47 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:24 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5f15e819

repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 ++++++++++++-----------
 1 file changed, 75 insertions(+), 73 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3c1c2d0..3b48b8e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 			metadata_bad = True
 			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
 			del e
+			self.muselist = frozenset(self.musedict)
+			return False
+
+		if not hasattr(xml_parser, 'parser') or \
+			sys.hexversion < 0x2070000 or \
+			(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
+			# doctype is not parsed with python 2.6 or 3.1
+			pass
 		else:
-			if not hasattr(xml_parser, 'parser') or \
-				sys.hexversion < 0x2070000 or \
-				(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-				# doctype is not parsed with python 2.6 or 3.1
-				pass
+			if "XML_DECLARATION" not in xml_info:
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: "
+					"xml declaration is missing on first line, "
+					"should be '%s'" % (xpkg, metadata_xml_declaration))
 			else:
-				if "XML_DECLARATION" not in xml_info:
+				xml_version, xml_encoding, xml_standalone = \
+					xml_info["XML_DECLARATION"]
+				if xml_encoding is None or \
+					xml_encoding.upper() != metadata_xml_encoding:
+					if xml_encoding is None:
+						encoding_problem = "but it is undefined"
+					else:
+						encoding_problem = "not '%s'" % xml_encoding
 					self.qatracker.add_error(
 						"metadata.bad", "%s/metadata.xml: "
-						"xml declaration is missing on first line, "
-						"should be '%s'" % (xpkg, metadata_xml_declaration))
-				else:
-					xml_version, xml_encoding, xml_standalone = \
-						xml_info["XML_DECLARATION"]
-					if xml_encoding is None or \
-						xml_encoding.upper() != metadata_xml_encoding:
-						if xml_encoding is None:
-							encoding_problem = "but it is undefined"
-						else:
-							encoding_problem = "not '%s'" % xml_encoding
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"xml declaration encoding should be '%s', %s" %
-							(xpkg, metadata_xml_encoding, encoding_problem))
+						"xml declaration encoding should be '%s', %s" %
+						(xpkg, metadata_xml_encoding, encoding_problem))
 
-				if "DOCTYPE" not in xml_info:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad",
-						"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-				else:
-					doctype_name, doctype_system, doctype_pubid = \
-						xml_info["DOCTYPE"]
-					if doctype_system != metadata_dtd_uri:
-						if doctype_system is None:
-							system_problem = "but it is undefined"
-						else:
-							system_problem = "not '%s'" % doctype_system
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"DOCTYPE: SYSTEM should refer to '%s', %s" %
-							(xpkg, metadata_dtd_uri, system_problem))
-
-					if doctype_name != metadata_doctype_name:
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"DOCTYPE: name should be '%s', not '%s'" %
-							(xpkg, metadata_doctype_name, doctype_name))
-
-			# load USE flags from metadata.xml
-			try:
-				self.musedict = parse_metadata_use(_metadata_xml)
-			except portage.exception.ParseError as e:
+			if "DOCTYPE" not in xml_info:
 				metadata_bad = True
 				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+					"metadata.bad",
+					"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 			else:
-				for atom in chain(*self.musedict.values()):
-					if atom is None:
-						continue
-					try:
-						atom = Atom(atom)
-					except InvalidAtom as e:
+				doctype_name, doctype_system, doctype_pubid = \
+					xml_info["DOCTYPE"]
+				if doctype_system != metadata_dtd_uri:
+					if doctype_system is None:
+						system_problem = "but it is undefined"
+					else:
+						system_problem = "not '%s'" % doctype_system
+					self.qatracker.add_error(
+						"metadata.bad", "%s/metadata.xml: "
+						"DOCTYPE: SYSTEM should refer to '%s', %s" %
+						(xpkg, metadata_dtd_uri, system_problem))
+
+				if doctype_name != metadata_doctype_name:
+					self.qatracker.add_error(
+						"metadata.bad", "%s/metadata.xml: "
+						"DOCTYPE: name should be '%s', not '%s'" %
+						(xpkg, metadata_doctype_name, doctype_name))
+
+		# load USE flags from metadata.xml
+		try:
+			self.musedict = parse_metadata_use(_metadata_xml)
+		except portage.exception.ParseError as e:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+		else:
+			for atom in chain(*self.musedict.values()):
+				if atom is None:
+					continue
+				try:
+					atom = Atom(atom)
+				except InvalidAtom as e:
+					self.qatracker.add_error(
+						"metadata.bad",
+						"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+				else:
+					if atom.cp != xpkg:
 						self.qatracker.add_error(
 							"metadata.bad",
-							"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-					else:
-						if atom.cp != xpkg:
-							self.qatracker.add_error(
-								"metadata.bad",
-								"%s/metadata.xml: Atom contains "
-								"unexpected cat/pn: %s" % (xpkg, atom))
-
-			# Run other metadata.xml checkers
-			try:
-				check_metadata(_metadata_xml, get_herd_base(
-					self.repoman_settings))
-			except (UnknownHerdsError, ) as e:
-				metadata_bad = True
-				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-				del e
+							"%s/metadata.xml: Atom contains "
+							"unexpected cat/pn: %s" % (xpkg, atom))
+
+		# Run other metadata.xml checkers
+		try:
+			check_metadata(_metadata_xml, get_herd_base(
+				self.repoman_settings))
+		except (UnknownHerdsError, ) as e:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+			del e
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     8075f0e91fc06125d6e2bcaaf655f66de694a568
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Tue May  3 07:18:05 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:41 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8075f0e9

repoman: Use lxml for parsing of metadata

Note that we no longer throw a QA error for a missing XML prolog, as long as
the encoding matches the default ('UTF-8'; lowercase is also allowed).
Update import error message to lxml pkg.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 46 +++++++-----------------
 1 file changed, 13 insertions(+), 33 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index bcddb3e..317ab56 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -7,14 +7,14 @@ import sys
 from itertools import chain
 
 try:
-	import xml.etree.ElementTree
-	from xml.parsers.expat import ExpatError
+	from lxml import etree
+	from lxml.etree import ParserError
 except (SystemExit, KeyboardInterrupt):
 	raise
 except (ImportError, SystemError, RuntimeError, Exception):
 	# broken or missing xml support
 	# http://bugs.python.org/issue14988
-	msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
+	msg = ["Please emerge dev-python/lxml in order to use repoman."]
 	from portage.output import EOutput
 	out = EOutput()
 	for line in msg:
@@ -26,12 +26,11 @@ from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
+from repoman._xml import XmlLint
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import _encodings, _unicode_encode
 from portage import exception
 from portage.dep import Atom
 
@@ -145,50 +144,31 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		# metadata.xml parse check
 		metadata_bad = False
-		xml_info = {}
-		xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
 
 		# read metadata.xml into memory
 		try:
-			_metadata_xml = xml.etree.ElementTree.parse(
-				_unicode_encode(
-					os.path.join(checkdir, "metadata.xml"),
-					encoding=_encodings['fs'], errors='strict'),
-				parser=xml_parser)
-		except (ExpatError, SyntaxError, EnvironmentError) as e:
+			_metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml'))
+		except (ParserError, SyntaxError, EnvironmentError) as e:
 			metadata_bad = True
 			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
 			del e
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if "XML_DECLARATION" not in xml_info:
+		xml_encoding = _metadata_xml.docinfo.encoding
+		if xml_encoding.upper() != metadata_xml_encoding:
 			self.qatracker.add_error(
 				"metadata.bad", "%s/metadata.xml: "
-				"xml declaration is missing on first line, "
-				"should be '%s'" % (xpkg, metadata_xml_declaration))
-		else:
-			xml_version, xml_encoding, xml_standalone = \
-				xml_info["XML_DECLARATION"]
-			if xml_encoding is None or \
-				xml_encoding.upper() != metadata_xml_encoding:
-				if xml_encoding is None:
-					encoding_problem = "but it is undefined"
-				else:
-					encoding_problem = "not '%s'" % xml_encoding
-				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration encoding should be '%s', %s" %
-					(xpkg, metadata_xml_encoding, encoding_problem))
+				"xml declaration encoding should be '%s', not '%s'" %
+				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if "DOCTYPE" not in xml_info:
+		if not _metadata_xml.docinfo:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",
 				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 		else:
-			doctype_name, doctype_system, doctype_pubid = \
-				xml_info["DOCTYPE"]
+			doctype_system = _metadata_xml.docinfo.system_url
 			if doctype_system != metadata_dtd_uri:
 				if doctype_system is None:
 					system_problem = "but it is undefined"
@@ -198,7 +178,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					"metadata.bad", "%s/metadata.xml: "
 					"DOCTYPE: SYSTEM should refer to '%s', %s" %
 					(xpkg, metadata_dtd_uri, system_problem))
-
+			doctype_name = _metadata_xml.docinfo.doctype.split(' ')[1]
 			if doctype_name != metadata_doctype_name:
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     119a9e27ef5e3442205f2e9be8f379a10177d673
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 19:45:31 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:41 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=119a9e27

repoman: mandate use of python 2.7 or 3.2+ for simplicity

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++++++++++-------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index af53f4b..bcddb3e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -162,54 +162,48 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if not hasattr(xml_parser, 'parser') or \
-			sys.hexversion < 0x2070000 or \
-			(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-			# doctype is not parsed with python 2.6 or 3.1
-			pass
+		if "XML_DECLARATION" not in xml_info:
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: "
+				"xml declaration is missing on first line, "
+				"should be '%s'" % (xpkg, metadata_xml_declaration))
 		else:
-			if "XML_DECLARATION" not in xml_info:
+			xml_version, xml_encoding, xml_standalone = \
+				xml_info["XML_DECLARATION"]
+			if xml_encoding is None or \
+				xml_encoding.upper() != metadata_xml_encoding:
+				if xml_encoding is None:
+					encoding_problem = "but it is undefined"
+				else:
+					encoding_problem = "not '%s'" % xml_encoding
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration is missing on first line, "
-					"should be '%s'" % (xpkg, metadata_xml_declaration))
-			else:
-				xml_version, xml_encoding, xml_standalone = \
-					xml_info["XML_DECLARATION"]
-				if xml_encoding is None or \
-					xml_encoding.upper() != metadata_xml_encoding:
-					if xml_encoding is None:
-						encoding_problem = "but it is undefined"
-					else:
-						encoding_problem = "not '%s'" % xml_encoding
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"xml declaration encoding should be '%s', %s" %
-						(xpkg, metadata_xml_encoding, encoding_problem))
+					"xml declaration encoding should be '%s', %s" %
+					(xpkg, metadata_xml_encoding, encoding_problem))
 
-			if "DOCTYPE" not in xml_info:
-				metadata_bad = True
+		if "DOCTYPE" not in xml_info:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
+		else:
+			doctype_name, doctype_system, doctype_pubid = \
+				xml_info["DOCTYPE"]
+			if doctype_system != metadata_dtd_uri:
+				if doctype_system is None:
+					system_problem = "but it is undefined"
+				else:
+					system_problem = "not '%s'" % doctype_system
 				self.qatracker.add_error(
-					"metadata.bad",
-					"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-			else:
-				doctype_name, doctype_system, doctype_pubid = \
-					xml_info["DOCTYPE"]
-				if doctype_system != metadata_dtd_uri:
-					if doctype_system is None:
-						system_problem = "but it is undefined"
-					else:
-						system_problem = "not '%s'" % doctype_system
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"DOCTYPE: SYSTEM should refer to '%s', %s" %
-						(xpkg, metadata_dtd_uri, system_problem))
+					"metadata.bad", "%s/metadata.xml: "
+					"DOCTYPE: SYSTEM should refer to '%s', %s" %
+					(xpkg, metadata_dtd_uri, system_problem))
 
-				if doctype_name != metadata_doctype_name:
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"DOCTYPE: name should be '%s', not '%s'" %
-						(xpkg, metadata_doctype_name, doctype_name))
+			if doctype_name != metadata_doctype_name:
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: "
+					"DOCTYPE: name should be '%s', not '%s'" %
+					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
 		try:

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     82846cb35a214c51731fa701fab26119c1b3832d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 07:39:09 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:41 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=82846cb3

repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks

This change based on original work done by Dirkjan Ochtman <dirkjan <AT> ochtman.nl>

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3ca7897..4921b6f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception):
 # import our initialized portage instance
 from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
-from repoman.checks.herds.herdbase import get_herd_base
-from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
@@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 							"%s/metadata.xml: Atom contains "
 							"unexpected cat/pn: %s" % (xpkg, atom))
 
-		# Run other metadata.xml checkers
-		try:
-			check_metadata(_metadata_xml, get_herd_base(
-				self.repoman_settings))
-		except (UnknownHerdsError, ) as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-			del e
-
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     8c8ddcfa78eee27a823bc83dc67b34fda77387d6
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu May  5 16:04:54 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:42 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8c8ddcfa

repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors

Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..433551a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		return uselist
 
 	def _add_validate_errors(self, xpkg, log):
+		listed = set()
 		for error in log:
-			self.qatracker.add_error(
-				"metadata.bad",
-				"%s/metadata.xml: line: %s, %s"
-				% (xpkg, error.line, error.message))
+			msg_prefix = error.message.split(":",1)[0]
+			info = "%s %s" % (error.line, msg_prefix)
+			if info not in listed:
+				listed.add(info)
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: line: %s, %s"
+					% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     278f13dfd0308f2d7d80fb332034611e0fa9f6ae
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 18:46:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:42 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=278f13df

repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class

This allows for a complete Q/A processing of all the use flags defined in the xml.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++++++++++------------
 1 file changed, 68 insertions(+), 70 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 1594b27..7117e7d 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import exception
 from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
@@ -43,55 +42,6 @@ metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 metadata_doctype_name = 'pkgmetadata'
 
 
-def parse_metadata_use(xml_tree):
-	"""
-	Records are wrapped in XML as per GLEP 56
-	returns a dict with keys constisting of USE flag names and values
-	containing their respective descriptions
-	"""
-	uselist = {}
-
-	usetags = xml_tree.findall("use")
-	if not usetags:
-		return uselist
-
-	# It's possible to have multiple 'use' elements.
-	for usetag in usetags:
-		flags = usetag.findall("flag")
-		if not flags:
-			# DTD allows use elements containing no flag elements.
-			continue
-
-		for flag in flags:
-			pkg_flag = flag.get("name")
-			if pkg_flag is None:
-				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
-			flag_restrict = flag.get("restrict")
-
-			# emulate the Element.itertext() method from python-2.7
-			inner_text = []
-			stack = []
-			stack.append(flag)
-			while stack:
-				obj = stack.pop()
-				if isinstance(obj, basestring):
-					inner_text.append(obj)
-					continue
-				if isinstance(obj.text, basestring):
-					inner_text.append(obj.text)
-				if isinstance(obj.tail, basestring):
-					stack.append(obj.tail)
-				stack.extend(reversed(obj))
-
-			if pkg_flag not in uselist:
-				uselist[pkg_flag] = {}
-
-			# (flag_restrict can be None)
-			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-	return uselist
-
-
 class PkgMetadata(ScanBase, USEFlagChecks):
 	'''Package metadata.xml checks'''
 
@@ -180,28 +130,23 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		try:
-			self.musedict = parse_metadata_use(_metadata_xml)
-		except portage.exception.ParseError as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-		else:
-			for atom in chain(*self.musedict.values()):
-				if atom is None:
-					continue
-				try:
-					atom = Atom(atom)
-				except InvalidAtom as e:
+		self.musedict, metadata_bad = self._parse_metadata_use(
+			_metadata_xml, xpkg, metadata_bad)
+		for atom in chain(*self.musedict.values()):
+			if atom is None:
+				continue
+			try:
+				atom = Atom(atom)
+			except InvalidAtom as e:
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+			else:
+				if atom.cp != xpkg:
 					self.qatracker.add_error(
 						"metadata.bad",
-						"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-				else:
-					if atom.cp != xpkg:
-						self.qatracker.add_error(
-							"metadata.bad",
-							"%s/metadata.xml: Atom contains "
-							"unexpected cat/pn: %s" % (xpkg, atom))
+						"%s/metadata.xml: Atom contains "
+						"unexpected cat/pn: %s" % (xpkg, atom))
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
@@ -237,6 +182,59 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
+	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+		"""
+		Records are wrapped in XML as per GLEP 56
+		returns a dict with keys constisting of USE flag names and values
+		containing their respective descriptions
+		"""
+		uselist = {}
+
+		usetags = xml_tree.findall("use")
+		if not usetags:
+			return uselist, metadata_bad
+
+		# It's possible to have multiple 'use' elements.
+		for usetag in usetags:
+			flags = usetag.findall("flag")
+			if not flags:
+				# DTD allows use elements containing no flag elements.
+				continue
+
+			for flag in flags:
+				pkg_flag = flag.get("name")
+				if pkg_flag is None:
+					metadata_bad = True
+					self.qatracker.add_error(
+						"metadata.bad",
+						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
+						% (xpkg, flag.sourceline, flag.text))
+					continue
+				flag_restrict = flag.get("restrict")
+
+				# emulate the Element.itertext() method from python-2.7
+				inner_text = []
+				stack = []
+				stack.append(flag)
+				while stack:
+					obj = stack.pop()
+					if isinstance(obj, basestring):
+						inner_text.append(obj)
+						continue
+					if isinstance(obj.text, basestring):
+						inner_text.append(obj.text)
+					if isinstance(obj.tail, basestring):
+						stack.append(obj.tail)
+					stack.extend(reversed(obj))
+
+				if pkg_flag not in uselist:
+					uselist[pkg_flag] = {}
+
+				# (flag_restrict can be None)
+				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
+
+		return uselist, metadata_bad
+
 	@property
 	def runInPkgs(self):
 		'''Package level scans'''

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     0cd414979f4b5531fdd158b3c4766c436bd057eb
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 20:54:33 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:42 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0cd41497

repoman/modules/.../pkgmetadata.py:  Have xml validation log all qatracker errors

Remove the use flag qatracker additions.
Add all logged XMLSchema errors to the qatracker .
This makes it a one run check to add all possible errors via the XMLSchema.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++++++++++-------------
 1 file changed, 36 insertions(+), 44 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7117e7d..83ca93f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -130,8 +130,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		self.musedict, metadata_bad = self._parse_metadata_use(
-			_metadata_xml, xpkg, metadata_bad)
+		self.musedict = self._parse_metadata_use(_metadata_xml, xpkg)
 		for atom in chain(*self.musedict.values()):
 			if atom is None:
 				continue
@@ -151,15 +150,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			try:
-				validator.assertValid(_metadata_xml)
-			except etree.DocumentInvalid as error:
-				self.qatracker.add_error(
-					"metadata.bad",
-					xpkg + "/metadata.xml: %s"
-					% (str(error))
-					)
-		del metadata_bad
+			if not validator.validate(_metadata_xml):
+				self._add_validate_errors(xpkg, validator.error_log)
 		self.muselist = frozenset(self.musedict)
 		return False
 
@@ -182,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
-	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+	def _parse_metadata_use(self, xml_tree, xpkg):
 		"""
 		Records are wrapped in XML as per GLEP 56
 		returns a dict with keys constisting of USE flag names and values
@@ -192,7 +184,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		usetags = xml_tree.findall("use")
 		if not usetags:
-			return uselist, metadata_bad
+			return uselist
 
 		# It's possible to have multiple 'use' elements.
 		for usetag in usetags:
@@ -203,37 +195,37 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 			for flag in flags:
 				pkg_flag = flag.get("name")
-				if pkg_flag is None:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad",
-						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
-						% (xpkg, flag.sourceline, flag.text))
-					continue
-				flag_restrict = flag.get("restrict")
-
-				# emulate the Element.itertext() method from python-2.7
-				inner_text = []
-				stack = []
-				stack.append(flag)
-				while stack:
-					obj = stack.pop()
-					if isinstance(obj, basestring):
-						inner_text.append(obj)
-						continue
-					if isinstance(obj.text, basestring):
-						inner_text.append(obj.text)
-					if isinstance(obj.tail, basestring):
-						stack.append(obj.tail)
-					stack.extend(reversed(obj))
-
-				if pkg_flag not in uselist:
-					uselist[pkg_flag] = {}
-
-				# (flag_restrict can be None)
-				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-		return uselist, metadata_bad
+				if pkg_flag is not None:
+					flag_restrict = flag.get("restrict")
+
+					# emulate the Element.itertext() method from python-2.7
+					inner_text = []
+					stack = []
+					stack.append(flag)
+					while stack:
+						obj = stack.pop()
+						if isinstance(obj, basestring):
+							inner_text.append(obj)
+							continue
+						if isinstance(obj.text, basestring):
+							inner_text.append(obj.text)
+						if isinstance(obj.tail, basestring):
+							stack.append(obj.tail)
+						stack.extend(reversed(obj))
+
+					if flag.get("name") not in uselist:
+						uselist[flag.get("name")] = {}
+
+					# (flag_restrict can be None)
+					uselist[flag.get("name")][flag_restrict] = " ".join("".join(inner_text).split())
+		return uselist
+
+	def _add_validate_errors(self, xpkg, log):
+		for error in log:
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: line: %s, %s"
+				% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     b1a831bffd28fc1bb889cc90bb7690c7e5e2b8d9
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed May  4 06:08:50 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:42 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b1a831bf

repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection

The docinfo is always created, so is not valid to use like a boolean to detect a missing
doctype.
doctype is a null string or equivalent and as such the correct choice for the if statement.

Failure discoverd by: Göktürk Yüksek <gokturk <AT> binghamton.edu> 
Thank you

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 83ca93f..d8344c2 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -106,7 +106,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 				"xml declaration encoding should be '%s', not '%s'" %
 				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if not _metadata_xml.docinfo:
+		if not _metadata_xml.docinfo.doctype:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     4d2cb398e48cdd90499dee48d52072e147c64e0f
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 17:38:19 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:42 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=4d2cb398

repoman/modules/.../pkgmetadata.py: Improve whole document validation

Change to using assertValid()  and add the error causing validation to fail to the qa tracker
error.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 4921b6f..1594b27 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -206,8 +206,14 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			if not validator.validate(_metadata_xml):
-				self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
+			try:
+				validator.assertValid(_metadata_xml)
+			except etree.DocumentInvalid as error:
+				self.qatracker.add_error(
+					"metadata.bad",
+					xpkg + "/metadata.xml: %s"
+					% (str(error))
+					)
 		del metadata_bad
 		self.muselist = frozenset(self.musedict)
 		return False

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     2eec213a7226380983f270276fef67f9c0337cab
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 07:39:09 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2eec213a

repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks

This change based on original work done by Dirkjan Ochtman <dirkjan <AT> ochtman.nl>

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3ca7897..4921b6f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception):
 # import our initialized portage instance
 from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
-from repoman.checks.herds.herdbase import get_herd_base
-from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
@@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 							"%s/metadata.xml: Atom contains "
 							"unexpected cat/pn: %s" % (xpkg, atom))
 
-		# Run other metadata.xml checkers
-		try:
-			check_metadata(_metadata_xml, get_herd_base(
-				self.repoman_settings))
-		except (UnknownHerdsError, ) as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-			del e
-
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     b78d61dd3e4281a43523aea1a24e395a2674c3ab
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Tue May  3 06:33:47 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b78d61dd

repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 ++++++++++++-----------
 1 file changed, 75 insertions(+), 73 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3c1c2d0..3b48b8e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 			metadata_bad = True
 			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
 			del e
+			self.muselist = frozenset(self.musedict)
+			return False
+
+		if not hasattr(xml_parser, 'parser') or \
+			sys.hexversion < 0x2070000 or \
+			(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
+			# doctype is not parsed with python 2.6 or 3.1
+			pass
 		else:
-			if not hasattr(xml_parser, 'parser') or \
-				sys.hexversion < 0x2070000 or \
-				(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-				# doctype is not parsed with python 2.6 or 3.1
-				pass
+			if "XML_DECLARATION" not in xml_info:
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: "
+					"xml declaration is missing on first line, "
+					"should be '%s'" % (xpkg, metadata_xml_declaration))
 			else:
-				if "XML_DECLARATION" not in xml_info:
+				xml_version, xml_encoding, xml_standalone = \
+					xml_info["XML_DECLARATION"]
+				if xml_encoding is None or \
+					xml_encoding.upper() != metadata_xml_encoding:
+					if xml_encoding is None:
+						encoding_problem = "but it is undefined"
+					else:
+						encoding_problem = "not '%s'" % xml_encoding
 					self.qatracker.add_error(
 						"metadata.bad", "%s/metadata.xml: "
-						"xml declaration is missing on first line, "
-						"should be '%s'" % (xpkg, metadata_xml_declaration))
-				else:
-					xml_version, xml_encoding, xml_standalone = \
-						xml_info["XML_DECLARATION"]
-					if xml_encoding is None or \
-						xml_encoding.upper() != metadata_xml_encoding:
-						if xml_encoding is None:
-							encoding_problem = "but it is undefined"
-						else:
-							encoding_problem = "not '%s'" % xml_encoding
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"xml declaration encoding should be '%s', %s" %
-							(xpkg, metadata_xml_encoding, encoding_problem))
+						"xml declaration encoding should be '%s', %s" %
+						(xpkg, metadata_xml_encoding, encoding_problem))
 
-				if "DOCTYPE" not in xml_info:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad",
-						"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-				else:
-					doctype_name, doctype_system, doctype_pubid = \
-						xml_info["DOCTYPE"]
-					if doctype_system != metadata_dtd_uri:
-						if doctype_system is None:
-							system_problem = "but it is undefined"
-						else:
-							system_problem = "not '%s'" % doctype_system
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"DOCTYPE: SYSTEM should refer to '%s', %s" %
-							(xpkg, metadata_dtd_uri, system_problem))
-
-					if doctype_name != metadata_doctype_name:
-						self.qatracker.add_error(
-							"metadata.bad", "%s/metadata.xml: "
-							"DOCTYPE: name should be '%s', not '%s'" %
-							(xpkg, metadata_doctype_name, doctype_name))
-
-			# load USE flags from metadata.xml
-			try:
-				self.musedict = parse_metadata_use(_metadata_xml)
-			except portage.exception.ParseError as e:
+			if "DOCTYPE" not in xml_info:
 				metadata_bad = True
 				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+					"metadata.bad",
+					"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 			else:
-				for atom in chain(*self.musedict.values()):
-					if atom is None:
-						continue
-					try:
-						atom = Atom(atom)
-					except InvalidAtom as e:
+				doctype_name, doctype_system, doctype_pubid = \
+					xml_info["DOCTYPE"]
+				if doctype_system != metadata_dtd_uri:
+					if doctype_system is None:
+						system_problem = "but it is undefined"
+					else:
+						system_problem = "not '%s'" % doctype_system
+					self.qatracker.add_error(
+						"metadata.bad", "%s/metadata.xml: "
+						"DOCTYPE: SYSTEM should refer to '%s', %s" %
+						(xpkg, metadata_dtd_uri, system_problem))
+
+				if doctype_name != metadata_doctype_name:
+					self.qatracker.add_error(
+						"metadata.bad", "%s/metadata.xml: "
+						"DOCTYPE: name should be '%s', not '%s'" %
+						(xpkg, metadata_doctype_name, doctype_name))
+
+		# load USE flags from metadata.xml
+		try:
+			self.musedict = parse_metadata_use(_metadata_xml)
+		except portage.exception.ParseError as e:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+		else:
+			for atom in chain(*self.musedict.values()):
+				if atom is None:
+					continue
+				try:
+					atom = Atom(atom)
+				except InvalidAtom as e:
+					self.qatracker.add_error(
+						"metadata.bad",
+						"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+				else:
+					if atom.cp != xpkg:
 						self.qatracker.add_error(
 							"metadata.bad",
-							"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-					else:
-						if atom.cp != xpkg:
-							self.qatracker.add_error(
-								"metadata.bad",
-								"%s/metadata.xml: Atom contains "
-								"unexpected cat/pn: %s" % (xpkg, atom))
-
-			# Run other metadata.xml checkers
-			try:
-				check_metadata(_metadata_xml, get_herd_base(
-					self.repoman_settings))
-			except (UnknownHerdsError, ) as e:
-				metadata_bad = True
-				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-				del e
+							"%s/metadata.xml: Atom contains "
+							"unexpected cat/pn: %s" % (xpkg, atom))
+
+		# Run other metadata.xml checkers
+		try:
+			check_metadata(_metadata_xml, get_herd_base(
+				self.repoman_settings))
+		except (UnknownHerdsError, ) as e:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+			del e
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     9c3e213142e0e39f69dc002bd15402a7a2ff22d1
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 19:45:31 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9c3e2131

repoman: mandate use of python 2.7 or 3.2+ for simplicity

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++++++++++-------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index af53f4b..bcddb3e 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -162,54 +162,48 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if not hasattr(xml_parser, 'parser') or \
-			sys.hexversion < 0x2070000 or \
-			(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-			# doctype is not parsed with python 2.6 or 3.1
-			pass
+		if "XML_DECLARATION" not in xml_info:
+			self.qatracker.add_error(
+				"metadata.bad", "%s/metadata.xml: "
+				"xml declaration is missing on first line, "
+				"should be '%s'" % (xpkg, metadata_xml_declaration))
 		else:
-			if "XML_DECLARATION" not in xml_info:
+			xml_version, xml_encoding, xml_standalone = \
+				xml_info["XML_DECLARATION"]
+			if xml_encoding is None or \
+				xml_encoding.upper() != metadata_xml_encoding:
+				if xml_encoding is None:
+					encoding_problem = "but it is undefined"
+				else:
+					encoding_problem = "not '%s'" % xml_encoding
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration is missing on first line, "
-					"should be '%s'" % (xpkg, metadata_xml_declaration))
-			else:
-				xml_version, xml_encoding, xml_standalone = \
-					xml_info["XML_DECLARATION"]
-				if xml_encoding is None or \
-					xml_encoding.upper() != metadata_xml_encoding:
-					if xml_encoding is None:
-						encoding_problem = "but it is undefined"
-					else:
-						encoding_problem = "not '%s'" % xml_encoding
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"xml declaration encoding should be '%s', %s" %
-						(xpkg, metadata_xml_encoding, encoding_problem))
+					"xml declaration encoding should be '%s', %s" %
+					(xpkg, metadata_xml_encoding, encoding_problem))
 
-			if "DOCTYPE" not in xml_info:
-				metadata_bad = True
+		if "DOCTYPE" not in xml_info:
+			metadata_bad = True
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
+		else:
+			doctype_name, doctype_system, doctype_pubid = \
+				xml_info["DOCTYPE"]
+			if doctype_system != metadata_dtd_uri:
+				if doctype_system is None:
+					system_problem = "but it is undefined"
+				else:
+					system_problem = "not '%s'" % doctype_system
 				self.qatracker.add_error(
-					"metadata.bad",
-					"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-			else:
-				doctype_name, doctype_system, doctype_pubid = \
-					xml_info["DOCTYPE"]
-				if doctype_system != metadata_dtd_uri:
-					if doctype_system is None:
-						system_problem = "but it is undefined"
-					else:
-						system_problem = "not '%s'" % doctype_system
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"DOCTYPE: SYSTEM should refer to '%s', %s" %
-						(xpkg, metadata_dtd_uri, system_problem))
+					"metadata.bad", "%s/metadata.xml: "
+					"DOCTYPE: SYSTEM should refer to '%s', %s" %
+					(xpkg, metadata_dtd_uri, system_problem))
 
-				if doctype_name != metadata_doctype_name:
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: "
-						"DOCTYPE: name should be '%s', not '%s'" %
-						(xpkg, metadata_doctype_name, doctype_name))
+			if doctype_name != metadata_doctype_name:
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: "
+					"DOCTYPE: name should be '%s', not '%s'" %
+					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
 		try:

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     633cabfb9215633c554fd967b9875310be3718bd
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Tue May  3 07:18:05 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=633cabfb

repoman: Use lxml for parsing of metadata

Note that we no longer throw a QA error for a missing XML prolog, as long as
the encoding matches the default ('UTF-8'; lowercase is also allowed).
Update import error message to lxml pkg.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 46 +++++++-----------------
 1 file changed, 13 insertions(+), 33 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index bcddb3e..317ab56 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -7,14 +7,14 @@ import sys
 from itertools import chain
 
 try:
-	import xml.etree.ElementTree
-	from xml.parsers.expat import ExpatError
+	from lxml import etree
+	from lxml.etree import ParserError
 except (SystemExit, KeyboardInterrupt):
 	raise
 except (ImportError, SystemError, RuntimeError, Exception):
 	# broken or missing xml support
 	# http://bugs.python.org/issue14988
-	msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
+	msg = ["Please emerge dev-python/lxml in order to use repoman."]
 	from portage.output import EOutput
 	out = EOutput()
 	for line in msg:
@@ -26,12 +26,11 @@ from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
+from repoman._xml import XmlLint
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import _encodings, _unicode_encode
 from portage import exception
 from portage.dep import Atom
 
@@ -145,50 +144,31 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		# metadata.xml parse check
 		metadata_bad = False
-		xml_info = {}
-		xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
 
 		# read metadata.xml into memory
 		try:
-			_metadata_xml = xml.etree.ElementTree.parse(
-				_unicode_encode(
-					os.path.join(checkdir, "metadata.xml"),
-					encoding=_encodings['fs'], errors='strict'),
-				parser=xml_parser)
-		except (ExpatError, SyntaxError, EnvironmentError) as e:
+			_metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml'))
+		except (ParserError, SyntaxError, EnvironmentError) as e:
 			metadata_bad = True
 			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
 			del e
 			self.muselist = frozenset(self.musedict)
 			return False
 
-		if "XML_DECLARATION" not in xml_info:
+		xml_encoding = _metadata_xml.docinfo.encoding
+		if xml_encoding.upper() != metadata_xml_encoding:
 			self.qatracker.add_error(
 				"metadata.bad", "%s/metadata.xml: "
-				"xml declaration is missing on first line, "
-				"should be '%s'" % (xpkg, metadata_xml_declaration))
-		else:
-			xml_version, xml_encoding, xml_standalone = \
-				xml_info["XML_DECLARATION"]
-			if xml_encoding is None or \
-				xml_encoding.upper() != metadata_xml_encoding:
-				if xml_encoding is None:
-					encoding_problem = "but it is undefined"
-				else:
-					encoding_problem = "not '%s'" % xml_encoding
-				self.qatracker.add_error(
-					"metadata.bad", "%s/metadata.xml: "
-					"xml declaration encoding should be '%s', %s" %
-					(xpkg, metadata_xml_encoding, encoding_problem))
+				"xml declaration encoding should be '%s', not '%s'" %
+				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if "DOCTYPE" not in xml_info:
+		if not _metadata_xml.docinfo:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",
 				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 		else:
-			doctype_name, doctype_system, doctype_pubid = \
-				xml_info["DOCTYPE"]
+			doctype_system = _metadata_xml.docinfo.system_url
 			if doctype_system != metadata_dtd_uri:
 				if doctype_system is None:
 					system_problem = "but it is undefined"
@@ -198,7 +178,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					"metadata.bad", "%s/metadata.xml: "
 					"DOCTYPE: SYSTEM should refer to '%s', %s" %
 					(xpkg, metadata_dtd_uri, system_problem))
-
+			doctype_name = _metadata_xml.docinfo.doctype.split(' ')[1]
 			if doctype_name != metadata_doctype_name:
 				self.qatracker.add_error(
 					"metadata.bad", "%s/metadata.xml: "

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     42b531b2810ec3e8e81348ebc4519a56e01a5a47
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 18:33:32 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=42b531b2

repoman/modules/.../pkgmetadata.py: Early return in metadata check code for readability

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 190 ++++++++++++-----------
 1 file changed, 96 insertions(+), 94 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 5c6452a..3c1c2d0 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -83,112 +83,114 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# metadata.xml file check
 		if "metadata.xml" not in checkdirlist:
 			self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")
+			self.muselist = frozenset(self.musedict)
+			return False
+
 		# metadata.xml parse check
+		metadata_bad = False
+		xml_info = {}
+		xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
+
+		# read metadata.xml into memory
+		try:
+			_metadata_xml = xml.etree.ElementTree.parse(
+				_unicode_encode(
+					os.path.join(checkdir, "metadata.xml"),
+					encoding=_encodings['fs'], errors='strict'),
+				parser=xml_parser)
+		except (ExpatError, SyntaxError, EnvironmentError) as e:
+			metadata_bad = True
+			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+			del e
 		else:
-			metadata_bad = False
-			xml_info = {}
-			xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
-
-			# read metadata.xml into memory
-			try:
-				_metadata_xml = xml.etree.ElementTree.parse(
-					_unicode_encode(
-						os.path.join(checkdir, "metadata.xml"),
-						encoding=_encodings['fs'], errors='strict'),
-					parser=xml_parser)
-			except (ExpatError, SyntaxError, EnvironmentError) as e:
-				metadata_bad = True
-				self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-				del e
+			if not hasattr(xml_parser, 'parser') or \
+				sys.hexversion < 0x2070000 or \
+				(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
+				# doctype is not parsed with python 2.6 or 3.1
+				pass
 			else:
-				if not hasattr(xml_parser, 'parser') or \
-					sys.hexversion < 0x2070000 or \
-					(sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
-					# doctype is not parsed with python 2.6 or 3.1
-					pass
+				if "XML_DECLARATION" not in xml_info:
+					self.qatracker.add_error(
+						"metadata.bad", "%s/metadata.xml: "
+						"xml declaration is missing on first line, "
+						"should be '%s'" % (xpkg, metadata_xml_declaration))
 				else:
-					if "XML_DECLARATION" not in xml_info:
+					xml_version, xml_encoding, xml_standalone = \
+						xml_info["XML_DECLARATION"]
+					if xml_encoding is None or \
+						xml_encoding.upper() != metadata_xml_encoding:
+						if xml_encoding is None:
+							encoding_problem = "but it is undefined"
+						else:
+							encoding_problem = "not '%s'" % xml_encoding
 						self.qatracker.add_error(
 							"metadata.bad", "%s/metadata.xml: "
-							"xml declaration is missing on first line, "
-							"should be '%s'" % (xpkg, metadata_xml_declaration))
-					else:
-						xml_version, xml_encoding, xml_standalone = \
-							xml_info["XML_DECLARATION"]
-						if xml_encoding is None or \
-							xml_encoding.upper() != metadata_xml_encoding:
-							if xml_encoding is None:
-								encoding_problem = "but it is undefined"
-							else:
-								encoding_problem = "not '%s'" % xml_encoding
-							self.qatracker.add_error(
-								"metadata.bad", "%s/metadata.xml: "
-								"xml declaration encoding should be '%s', %s" %
-								(xpkg, metadata_xml_encoding, encoding_problem))
-
-					if "DOCTYPE" not in xml_info:
-						metadata_bad = True
-						self.qatracker.add_error(
-							"metadata.bad",
-							"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
-					else:
-						doctype_name, doctype_system, doctype_pubid = \
-							xml_info["DOCTYPE"]
-						if doctype_system != metadata_dtd_uri:
-							if doctype_system is None:
-								system_problem = "but it is undefined"
-							else:
-								system_problem = "not '%s'" % doctype_system
-							self.qatracker.add_error(
-								"metadata.bad", "%s/metadata.xml: "
-								"DOCTYPE: SYSTEM should refer to '%s', %s" %
-								(xpkg, metadata_dtd_uri, system_problem))
+							"xml declaration encoding should be '%s', %s" %
+							(xpkg, metadata_xml_encoding, encoding_problem))
 
-						if doctype_name != metadata_doctype_name:
-							self.qatracker.add_error(
-								"metadata.bad", "%s/metadata.xml: "
-								"DOCTYPE: name should be '%s', not '%s'" %
-								(xpkg, metadata_doctype_name, doctype_name))
-
-				# load USE flags from metadata.xml
-				try:
-					self.musedict = parse_metadata_use(_metadata_xml)
-				except portage.exception.ParseError as e:
+				if "DOCTYPE" not in xml_info:
 					metadata_bad = True
 					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+						"metadata.bad",
+						"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
 				else:
-					for atom in chain(*self.musedict.values()):
-						if atom is None:
-							continue
-						try:
-							atom = Atom(atom)
-						except InvalidAtom as e:
+					doctype_name, doctype_system, doctype_pubid = \
+						xml_info["DOCTYPE"]
+					if doctype_system != metadata_dtd_uri:
+						if doctype_system is None:
+							system_problem = "but it is undefined"
+						else:
+							system_problem = "not '%s'" % doctype_system
+						self.qatracker.add_error(
+							"metadata.bad", "%s/metadata.xml: "
+							"DOCTYPE: SYSTEM should refer to '%s', %s" %
+							(xpkg, metadata_dtd_uri, system_problem))
+
+					if doctype_name != metadata_doctype_name:
+						self.qatracker.add_error(
+							"metadata.bad", "%s/metadata.xml: "
+							"DOCTYPE: name should be '%s', not '%s'" %
+							(xpkg, metadata_doctype_name, doctype_name))
+
+			# load USE flags from metadata.xml
+			try:
+				self.musedict = parse_metadata_use(_metadata_xml)
+			except portage.exception.ParseError as e:
+				metadata_bad = True
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+			else:
+				for atom in chain(*self.musedict.values()):
+					if atom is None:
+						continue
+					try:
+						atom = Atom(atom)
+					except InvalidAtom as e:
+						self.qatracker.add_error(
+							"metadata.bad",
+							"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+					else:
+						if atom.cp != xpkg:
 							self.qatracker.add_error(
 								"metadata.bad",
-								"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-						else:
-							if atom.cp != xpkg:
-								self.qatracker.add_error(
-									"metadata.bad",
-									"%s/metadata.xml: Atom contains "
-									"unexpected cat/pn: %s" % (xpkg, atom))
-
-				# Run other metadata.xml checkers
-				try:
-					check_metadata(_metadata_xml, get_herd_base(
-						self.repoman_settings))
-				except (UnknownHerdsError, ) as e:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-					del e
-
-			# Only carry out if in package directory or check forced
-			if not metadata_bad:
-				if not self.xmllint.check(checkdir, repolevel):
-					self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
-			del metadata_bad
+								"%s/metadata.xml: Atom contains "
+								"unexpected cat/pn: %s" % (xpkg, atom))
+
+			# Run other metadata.xml checkers
+			try:
+				check_metadata(_metadata_xml, get_herd_base(
+					self.repoman_settings))
+			except (UnknownHerdsError, ) as e:
+				metadata_bad = True
+				self.qatracker.add_error(
+					"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
+				del e
+
+		# Only carry out if in package directory or check forced
+		if not metadata_bad:
+			if not self.xmllint.check(checkdir, repolevel):
+				self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
+		del metadata_bad
 		self.muselist = frozenset(self.musedict)
 		return False
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     c6d6b003bc0545912f15c82a99c2e948a5c63e60
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 20:54:33 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c6d6b003

repoman/modules/.../pkgmetadata.py:  Have xml validation log all qatracker errors

Remove the use flag qatracker additions.
Add all logged XMLSchema errors to the qatracker .
This makes it a one run check to add all possible errors via the XMLSchema.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++++++++++-------------
 1 file changed, 36 insertions(+), 44 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7117e7d..83ca93f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -130,8 +130,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		self.musedict, metadata_bad = self._parse_metadata_use(
-			_metadata_xml, xpkg, metadata_bad)
+		self.musedict = self._parse_metadata_use(_metadata_xml, xpkg)
 		for atom in chain(*self.musedict.values()):
 			if atom is None:
 				continue
@@ -151,15 +150,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			try:
-				validator.assertValid(_metadata_xml)
-			except etree.DocumentInvalid as error:
-				self.qatracker.add_error(
-					"metadata.bad",
-					xpkg + "/metadata.xml: %s"
-					% (str(error))
-					)
-		del metadata_bad
+			if not validator.validate(_metadata_xml):
+				self._add_validate_errors(xpkg, validator.error_log)
 		self.muselist = frozenset(self.musedict)
 		return False
 
@@ -182,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
-	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+	def _parse_metadata_use(self, xml_tree, xpkg):
 		"""
 		Records are wrapped in XML as per GLEP 56
 		returns a dict with keys constisting of USE flag names and values
@@ -192,7 +184,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		usetags = xml_tree.findall("use")
 		if not usetags:
-			return uselist, metadata_bad
+			return uselist
 
 		# It's possible to have multiple 'use' elements.
 		for usetag in usetags:
@@ -203,37 +195,37 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 			for flag in flags:
 				pkg_flag = flag.get("name")
-				if pkg_flag is None:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad",
-						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
-						% (xpkg, flag.sourceline, flag.text))
-					continue
-				flag_restrict = flag.get("restrict")
-
-				# emulate the Element.itertext() method from python-2.7
-				inner_text = []
-				stack = []
-				stack.append(flag)
-				while stack:
-					obj = stack.pop()
-					if isinstance(obj, basestring):
-						inner_text.append(obj)
-						continue
-					if isinstance(obj.text, basestring):
-						inner_text.append(obj.text)
-					if isinstance(obj.tail, basestring):
-						stack.append(obj.tail)
-					stack.extend(reversed(obj))
-
-				if pkg_flag not in uselist:
-					uselist[pkg_flag] = {}
-
-				# (flag_restrict can be None)
-				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-		return uselist, metadata_bad
+				if pkg_flag is not None:
+					flag_restrict = flag.get("restrict")
+
+					# emulate the Element.itertext() method from python-2.7
+					inner_text = []
+					stack = []
+					stack.append(flag)
+					while stack:
+						obj = stack.pop()
+						if isinstance(obj, basestring):
+							inner_text.append(obj)
+							continue
+						if isinstance(obj.text, basestring):
+							inner_text.append(obj.text)
+						if isinstance(obj.tail, basestring):
+							stack.append(obj.tail)
+						stack.extend(reversed(obj))
+
+					if flag.get("name") not in uselist:
+						uselist[flag.get("name")] = {}
+
+					# (flag_restrict can be None)
+					uselist[flag.get("name")][flag_restrict] = " ".join("".join(inner_text).split())
+		return uselist
+
+	def _add_validate_errors(self, xpkg, log):
+		for error in log:
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: line: %s, %s"
+				% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     64bb8e17a5c050671765626bbe45174e10b5a3a4
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 17:38:19 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=64bb8e17

repoman/modules/.../pkgmetadata.py: Improve whole document validation

Change to using assertValid()  and add the error causing validation to fail to the qa tracker
error.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 4921b6f..1594b27 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -206,8 +206,14 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			if not validator.validate(_metadata_xml):
-				self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
+			try:
+				validator.assertValid(_metadata_xml)
+			except etree.DocumentInvalid as error:
+				self.qatracker.add_error(
+					"metadata.bad",
+					xpkg + "/metadata.xml: %s"
+					% (str(error))
+					)
 		del metadata_bad
 		self.muselist = frozenset(self.musedict)
 		return False

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     57d1ef7f8f98275b066735c66d0da05ea103a9da
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 18:46:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=57d1ef7f

repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class

This allows for a complete Q/A processing of all the use flags defined in the xml.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++++++++++------------
 1 file changed, 68 insertions(+), 70 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 1594b27..7117e7d 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
 from portage import os
-from portage import exception
 from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
@@ -43,55 +42,6 @@ metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 metadata_doctype_name = 'pkgmetadata'
 
 
-def parse_metadata_use(xml_tree):
-	"""
-	Records are wrapped in XML as per GLEP 56
-	returns a dict with keys constisting of USE flag names and values
-	containing their respective descriptions
-	"""
-	uselist = {}
-
-	usetags = xml_tree.findall("use")
-	if not usetags:
-		return uselist
-
-	# It's possible to have multiple 'use' elements.
-	for usetag in usetags:
-		flags = usetag.findall("flag")
-		if not flags:
-			# DTD allows use elements containing no flag elements.
-			continue
-
-		for flag in flags:
-			pkg_flag = flag.get("name")
-			if pkg_flag is None:
-				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
-			flag_restrict = flag.get("restrict")
-
-			# emulate the Element.itertext() method from python-2.7
-			inner_text = []
-			stack = []
-			stack.append(flag)
-			while stack:
-				obj = stack.pop()
-				if isinstance(obj, basestring):
-					inner_text.append(obj)
-					continue
-				if isinstance(obj.text, basestring):
-					inner_text.append(obj.text)
-				if isinstance(obj.tail, basestring):
-					stack.append(obj.tail)
-				stack.extend(reversed(obj))
-
-			if pkg_flag not in uselist:
-				uselist[pkg_flag] = {}
-
-			# (flag_restrict can be None)
-			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-	return uselist
-
-
 class PkgMetadata(ScanBase, USEFlagChecks):
 	'''Package metadata.xml checks'''
 
@@ -180,28 +130,23 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		try:
-			self.musedict = parse_metadata_use(_metadata_xml)
-		except portage.exception.ParseError as e:
-			metadata_bad = True
-			self.qatracker.add_error(
-				"metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
-		else:
-			for atom in chain(*self.musedict.values()):
-				if atom is None:
-					continue
-				try:
-					atom = Atom(atom)
-				except InvalidAtom as e:
+		self.musedict, metadata_bad = self._parse_metadata_use(
+			_metadata_xml, xpkg, metadata_bad)
+		for atom in chain(*self.musedict.values()):
+			if atom is None:
+				continue
+			try:
+				atom = Atom(atom)
+			except InvalidAtom as e:
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
+			else:
+				if atom.cp != xpkg:
 					self.qatracker.add_error(
 						"metadata.bad",
-						"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
-				else:
-					if atom.cp != xpkg:
-						self.qatracker.add_error(
-							"metadata.bad",
-							"%s/metadata.xml: Atom contains "
-							"unexpected cat/pn: %s" % (xpkg, atom))
+						"%s/metadata.xml: Atom contains "
+						"unexpected cat/pn: %s" % (xpkg, atom))
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
@@ -237,6 +182,59 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
+	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+		"""
+		Records are wrapped in XML as per GLEP 56
+		returns a dict with keys constisting of USE flag names and values
+		containing their respective descriptions
+		"""
+		uselist = {}
+
+		usetags = xml_tree.findall("use")
+		if not usetags:
+			return uselist, metadata_bad
+
+		# It's possible to have multiple 'use' elements.
+		for usetag in usetags:
+			flags = usetag.findall("flag")
+			if not flags:
+				# DTD allows use elements containing no flag elements.
+				continue
+
+			for flag in flags:
+				pkg_flag = flag.get("name")
+				if pkg_flag is None:
+					metadata_bad = True
+					self.qatracker.add_error(
+						"metadata.bad",
+						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
+						% (xpkg, flag.sourceline, flag.text))
+					continue
+				flag_restrict = flag.get("restrict")
+
+				# emulate the Element.itertext() method from python-2.7
+				inner_text = []
+				stack = []
+				stack.append(flag)
+				while stack:
+					obj = stack.pop()
+					if isinstance(obj, basestring):
+						inner_text.append(obj)
+						continue
+					if isinstance(obj.text, basestring):
+						inner_text.append(obj.text)
+					if isinstance(obj.tail, basestring):
+						stack.append(obj.tail)
+					stack.extend(reversed(obj))
+
+				if pkg_flag not in uselist:
+					uselist[pkg_flag] = {}
+
+				# (flag_restrict can be None)
+				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
+
+		return uselist, metadata_bad
+
 	@property
 	def runInPkgs(self):
 		'''Package level scans'''

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     cfc31c04bf6e1d59b2756e3ac315a2040cdd162d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed May  4 06:08:50 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfc31c04

repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection

The docinfo is always created, so is not valid to use like a boolean to detect a missing
doctype.
doctype is a null string or equivalent and as such the correct choice for the if statement.

Failure discoverd by: Göktürk Yüksek <gokturk <AT> binghamton.edu> 
Thank you

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 83ca93f..d8344c2 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -106,7 +106,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 				"xml declaration encoding should be '%s', not '%s'" %
 				(xpkg, metadata_xml_encoding, xml_encoding))
 
-		if not _metadata_xml.docinfo:
+		if not _metadata_xml.docinfo.doctype:
 			metadata_bad = True
 			self.qatracker.add_error(
 				"metadata.bad",

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     1f4d8908748dc59c362d655db3f6fe3b4c913476
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu May  5 16:04:54 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1f4d8908

repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors

Some types of errors produce two error messages cluttering up the output.
The first error message is clearer, listing the possible option values allowed.
This filters out the second error message for that same line and attribute.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index d8344c2..433551a 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		return uselist
 
 	def _add_validate_errors(self, xpkg, log):
+		listed = set()
 		for error in log:
-			self.qatracker.add_error(
-				"metadata.bad",
-				"%s/metadata.xml: line: %s, %s"
-				% (xpkg, error.line, error.message))
+			msg_prefix = error.message.split(":",1)[0]
+			info = "%s %s" % (error.line, msg_prefix)
+			if info not in listed:
+				listed.add(info)
+				self.qatracker.add_error(
+					"metadata.bad",
+					"%s/metadata.xml: line: %s, %s"
+					% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):

[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/

[Brian Dolbec]

commit:     c6d6b003bc0545912f15c82a99c2e948a5c63e60
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May  3 20:54:33 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c6d6b003

repoman/modules/.../pkgmetadata.py:  Have xml validation log all qatracker errors

Remove the use flag qatracker additions.
Add all logged XMLSchema errors to the qatracker .
This makes it a one run check to add all possible errors via the XMLSchema.

 pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++++++++++-------------
 1 file changed, 36 insertions(+), 44 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 7117e7d..83ca93f 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -130,8 +130,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					(xpkg, metadata_doctype_name, doctype_name))
 
 		# load USE flags from metadata.xml
-		self.musedict, metadata_bad = self._parse_metadata_use(
-			_metadata_xml, xpkg, metadata_bad)
+		self.musedict = self._parse_metadata_use(_metadata_xml, xpkg)
 		for atom in chain(*self.musedict.values()):
 			if atom is None:
 				continue
@@ -151,15 +150,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
 			validator = etree.XMLSchema(file=self.metadata_xsd)
-			try:
-				validator.assertValid(_metadata_xml)
-			except etree.DocumentInvalid as error:
-				self.qatracker.add_error(
-					"metadata.bad",
-					xpkg + "/metadata.xml: %s"
-					% (str(error))
-					)
-		del metadata_bad
+			if not validator.validate(_metadata_xml):
+				self._add_validate_errors(xpkg, validator.error_log)
 		self.muselist = frozenset(self.musedict)
 		return False
 
@@ -182,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 					% (xpkg, myflag))
 		return False
 
-	def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad):
+	def _parse_metadata_use(self, xml_tree, xpkg):
 		"""
 		Records are wrapped in XML as per GLEP 56
 		returns a dict with keys constisting of USE flag names and values
@@ -192,7 +184,7 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		usetags = xml_tree.findall("use")
 		if not usetags:
-			return uselist, metadata_bad
+			return uselist
 
 		# It's possible to have multiple 'use' elements.
 		for usetag in usetags:
@@ -203,37 +195,37 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 			for flag in flags:
 				pkg_flag = flag.get("name")
-				if pkg_flag is None:
-					metadata_bad = True
-					self.qatracker.add_error(
-						"metadata.bad",
-						"%s/metadata.xml: line: %s, '%s', missing attribute: name"
-						% (xpkg, flag.sourceline, flag.text))
-					continue
-				flag_restrict = flag.get("restrict")
-
-				# emulate the Element.itertext() method from python-2.7
-				inner_text = []
-				stack = []
-				stack.append(flag)
-				while stack:
-					obj = stack.pop()
-					if isinstance(obj, basestring):
-						inner_text.append(obj)
-						continue
-					if isinstance(obj.text, basestring):
-						inner_text.append(obj.text)
-					if isinstance(obj.tail, basestring):
-						stack.append(obj.tail)
-					stack.extend(reversed(obj))
-
-				if pkg_flag not in uselist:
-					uselist[pkg_flag] = {}
-
-				# (flag_restrict can be None)
-				uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-		return uselist, metadata_bad
+				if pkg_flag is not None:
+					flag_restrict = flag.get("restrict")
+
+					# emulate the Element.itertext() method from python-2.7
+					inner_text = []
+					stack = []
+					stack.append(flag)
+					while stack:
+						obj = stack.pop()
+						if isinstance(obj, basestring):
+							inner_text.append(obj)
+							continue
+						if isinstance(obj.text, basestring):
+							inner_text.append(obj.text)
+						if isinstance(obj.tail, basestring):
+							stack.append(obj.tail)
+						stack.extend(reversed(obj))
+
+					if flag.get("name") not in uselist:
+						uselist[flag.get("name")] = {}
+
+					# (flag_restrict can be None)
+					uselist[flag.get("name")][flag_restrict] = " ".join("".join(inner_text).split())
+		return uselist
+
+	def _add_validate_errors(self, xpkg, log):
+		for error in log:
+			self.qatracker.add_error(
+				"metadata.bad",
+				"%s/metadata.xml: line: %s, %s"
+				% (xpkg, error.line, error.message))
 
 	@property
 	def runInPkgs(self):