[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     96c4b3db3f34c1e5f0996fd7975a73c846cd7ff5
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 30 07:50:20 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=96c4b3db

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml.

 pym/repoman/modules/scan/metadata/__init__.py |  9 +++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 4f376e1..f619764 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -55,6 +55,15 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'sourcefile': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 1e61eb6..4dd5408 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -270,7 +270,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -318,11 +317,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     2d2b99917ed054bea083a0e47f30aac34f0fefdd
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Tue May  3 09:01:29 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2d2b9991

repoman: Migrate from XmlLint to etree.XMLSchema for validation

Remove No longer used repoman._xml module
This change based on work by Dirkjan Ochtman <djc <AT> gentoo.org>
Updated the change from XML.DTD to XMLSchema.
Additionally:
    Move the metadata.xsd path determination code to metadata.py.
    Add the DISTDIR backup location and fetching of the file if missing or stale.

 pym/repoman/_xml.py                              | 105 -----------------------
 pym/repoman/metadata.py                          |  21 +++++
 pym/repoman/modules/scan/metadata/pkgmetadata.py |   9 +-
 pym/repoman/scanner.py                           |  10 +--
 4 files changed, 26 insertions(+), 119 deletions(-)

diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
deleted file mode 100644
index 33a536a..0000000
--- a/pym/repoman/_xml.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
-import sys
-import xml
-
-# import our initialized portage instance
-from repoman._portage import portage
-
-from portage import os
-from portage.output import red
-from portage.process import find_binary
-
-from repoman.metadata import fetch_metadata_xsd
-from repoman._subprocess import repoman_getstatusoutput
-
-
-class _XMLParser(xml.etree.ElementTree.XMLParser):
-
-	def __init__(self, data, **kwargs):
-		xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
-		self._portage_data = data
-		if hasattr(self, 'parser'):
-			self._base_XmlDeclHandler = self.parser.XmlDeclHandler
-			self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
-			self._base_StartDoctypeDeclHandler = \
-				self.parser.StartDoctypeDeclHandler
-			self.parser.StartDoctypeDeclHandler = \
-				self._portage_StartDoctypeDeclHandler
-
-	def _portage_XmlDeclHandler(self, version, encoding, standalone):
-		if self._base_XmlDeclHandler is not None:
-			self._base_XmlDeclHandler(version, encoding, standalone)
-		self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
-
-	def _portage_StartDoctypeDeclHandler(
-		self, doctypeName, systemId, publicId, has_internal_subset):
-		if self._base_StartDoctypeDeclHandler is not None:
-			self._base_StartDoctypeDeclHandler(
-				doctypeName, systemId, publicId, has_internal_subset)
-		self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
-
-
-class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
-	"""
-	Implements doctype() as required to avoid deprecation warnings with
-	>=python-2.7.
-	"""
-	def doctype(self, name, pubid, system):
-		pass
-
-
-class XmlLint(object):
-
-	def __init__(self, options, repoman_settings, metadata_xsd=None):
-		self.metadata_xsd = (metadata_xsd or
-			os.path.join(repoman_settings["DISTDIR"], 'metadata.xsd'))
-		self.options = options
-		self.repoman_settings = repoman_settings
-		self._is_capable = metadata_xsd is not None
-		self.binary = None
-		self._check_capable()
-
-	def _check_capable(self):
-		if self.options.mode == "manifest":
-			return
-		self.binary = find_binary('xmllint')
-		if not self.binary:
-			print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
-		elif not self._is_capable:
-			if not fetch_metadata_xsd(self.metadata_xsd, self.repoman_settings):
-				sys.exit(1)
-			# this can be problematic if xmllint changes their output
-			self._is_capable = True
-
-	@property
-	def capable(self):
-		return self._is_capable
-
-	def check(self, checkdir, repolevel):
-		'''Runs checks on the package metadata.xml file
-
-		@param checkdir: string, path
-		@param repolevel: integer
-		@return boolean, False == bad metadata
-		'''
-		if not self.capable:
-			if self.options.xml_parse or repolevel == 3:
-				print("%s sorry, xmllint is needed.  failing\n" % red("!!!"))
-				sys.exit(1)
-			return True
-		# xmlint can produce garbage output even on success, so only dump
-		# the ouput when it fails.
-		st, out = repoman_getstatusoutput(
-			self.binary + " --nonet --noout --schema %s %s" % (
-				portage._shell_quote(self.metadata_xsd),
-				portage._shell_quote(
-					os.path.join(checkdir, "metadata.xml"))))
-		if st != os.EX_OK:
-			print(red("!!!") + " metadata.xml is invalid:")
-			for z in out.splitlines():
-				print(red("!!! ") + z)
-			return False
-		return True

diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index 7c64c8e..a9ad3e8 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -99,3 +99,24 @@ def fetch_metadata_xsd(metadata_xsd, repoman_settings):
 				pass
 
 	return True
+
+
+def get_metadata_xsd(repo_settings):
+	'''Locate and or fetch the metadata.xsd file
+
+	@param repo_settings: RepoSettings instance
+	@returns: path to the metadata.xsd file
+	'''
+	metadata_xsd = None
+	for path in reversed(repo_settings.repo_config.eclass_db.porttrees):
+		path = os.path.join(path, 'metadata/xml-schema/metadata.xsd')
+		if os.path.exists(path):
+			metadata_xsd = path
+			break
+	if metadata_xsd is None:
+		metadata_xsd = os.path.join(
+			repo_settings.repoman_settings["DISTDIR"], 'metadata.xsd'
+			)
+
+		fetch_metadata_xsd(metadata_xsd, repo_settings.repoman_settings)
+	return metadata_xsd

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 317ab56..3ca7897 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -26,7 +26,6 @@ from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import XmlLint
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
@@ -110,13 +109,11 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		repo_settings = kwargs.get('repo_settings')
 		self.qatracker = kwargs.get('qatracker')
 		self.options = kwargs.get('options')
-		metadata_xsd = kwargs.get('metadata_xsd')
+		self.metadata_xsd = kwargs.get('metadata_xsd')
 		self.globalUseFlags = kwargs.get('uselist')
 		self.repoman_settings = repo_settings.repoman_settings
 		self.musedict = {}
 		self.muselist = set()
-		self.xmllint = XmlLint(self.options, self.repoman_settings,
-			metadata_xsd=metadata_xsd)
 
 	def check(self, **kwargs):
 		'''Performs the checks on the metadata.xml for the package
@@ -129,7 +126,6 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		xpkg = kwargs.get('xpkg')
 		checkdir = kwargs.get('checkdir')
 		checkdirlist = kwargs.get('checkdirlist').get()
-		repolevel = kwargs.get('repolevel')
 
 		self.musedict = {}
 		if self.options.mode in ['manifest']:
@@ -221,7 +217,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
-			if not self.xmllint.check(checkdir, repolevel):
+			validator = etree.XMLSchema(file=self.metadata_xsd)
+			if not validator.validate(_metadata_xml):
 				self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
 		del metadata_bad
 		self.muselist = frozenset(self.musedict)

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index fd07209..48d9001 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -10,6 +10,7 @@ from portage import normalize_path
 from portage import os
 from portage.output import green
 from portage.util.futures.extendedfutures import ExtendedFuture
+from repoman.metadata import get_metadata_xsd
 from repoman.modules.commit import repochecks
 from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
 from repoman.repos import repo_metadata
@@ -56,13 +57,6 @@ class Scanner(object):
 			portage.util.stack_lists([self.categories], incremental=1))
 		self.categories = self.repo_settings.repoman_settings.categories
 
-		metadata_xsd = None
-		for path in reversed(self.repo_settings.repo_config.eclass_db.porttrees):
-			path = os.path.join(path, 'metadata/xml-schema/metadata.xsd')
-			if os.path.exists(path):
-				metadata_xsd = path
-				break
-
 		self.portdb = repo_settings.portdb
 		self.portdb.settings = self.repo_settings.repoman_settings
 		# We really only need to cache the metadata that's necessary for visibility
@@ -187,7 +181,7 @@ class Scanner(object):
 			"qatracker": self.qatracker,
 			"vcs_settings": self.vcs_settings,
 			"options": self.options,
-			"metadata_xsd": metadata_xsd,
+			"metadata_xsd": get_metadata_xsd(self.repo_settings),
 			"uselist": uselist,
 			"checks": checks,
 			"repo_metadata": self.repo_metadata,

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     22a9d9df146499ebf740eb61964633ba7d6a10ef
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 18:42:41 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat May 14 18:29:40 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=22a9d9df

repoman: Move some metdata_* variables to module it's used

 pym/repoman/metadata.py                          | 4 ----
 pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 ++++++---
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index 7a514dc..a980184 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -28,10 +28,6 @@ if sys.hexversion >= 0x3000000:
 if sys.hexversion >= 0x3000000:
 	basestring = str
 
-metadata_xml_encoding = 'UTF-8'
-metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
-	% (metadata_xml_encoding,)
-metadata_doctype_name = 'pkgmetadata'
 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd'
 # force refetch if the local copy creation time is older than this

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 3b48b8e..a7150a9 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -23,9 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception):
 
 # import our initialized portage instance
 from repoman._portage import portage
-from repoman.metadata import (
-	metadata_xml_encoding, metadata_doctype_name,
-	metadata_dtd_uri, metadata_xml_declaration, parse_metadata_use)
+from repoman.metadata import metadata_dtd_uri, parse_metadata_use
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
 from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
@@ -38,6 +36,11 @@ from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
 
+metadata_xml_encoding = 'UTF-8'
+metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
+	% (metadata_xml_encoding,)
+metadata_doctype_name = 'pkgmetadata'
+
 
 class PkgMetadata(ScanBase, USEFlagChecks):
 	'''Package metadata.xml checks'''

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     20fd37b3ddb901f15135d05518a7d8a7bfb7c320
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 19:40:29 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:41 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=20fd37b3

repoman: Move parse_metadata_use function to the pkgmetadata.py

Add missing basestring compat code

 pym/repoman/metadata.py                          | 50 ---------------------
 pym/repoman/modules/scan/metadata/pkgmetadata.py | 56 +++++++++++++++++++++++-
 2 files changed, 55 insertions(+), 51 deletions(-)

diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index a980184..7c64c8e 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -17,7 +17,6 @@ except ImportError:
 # import our initialized portage instance
 from repoman._portage import portage
 
-from portage import exception
 from portage import os
 from portage import shutil
 from portage.output import green
@@ -34,55 +33,6 @@ metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd'
 metadata_xsd_ctime_interval = 60 * 60 * 24 * 7  # 7 days
 
 
-def parse_metadata_use(xml_tree):
-	"""
-	Records are wrapped in XML as per GLEP 56
-	returns a dict with keys constisting of USE flag names and values
-	containing their respective descriptions
-	"""
-	uselist = {}
-
-	usetags = xml_tree.findall("use")
-	if not usetags:
-		return uselist
-
-	# It's possible to have multiple 'use' elements.
-	for usetag in usetags:
-		flags = usetag.findall("flag")
-		if not flags:
-			# DTD allows use elements containing no flag elements.
-			continue
-
-		for flag in flags:
-			pkg_flag = flag.get("name")
-			if pkg_flag is None:
-				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
-			flag_restrict = flag.get("restrict")
-
-			# emulate the Element.itertext() method from python-2.7
-			inner_text = []
-			stack = []
-			stack.append(flag)
-			while stack:
-				obj = stack.pop()
-				if isinstance(obj, basestring):
-					inner_text.append(obj)
-					continue
-				if isinstance(obj.text, basestring):
-					inner_text.append(obj.text)
-				if isinstance(obj.tail, basestring):
-					stack.append(obj.tail)
-				stack.extend(reversed(obj))
-
-			if pkg_flag not in uselist:
-				uselist[pkg_flag] = {}
-
-			# (flag_restrict can be None)
-			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-	return uselist
-
-
 def fetch_metadata_xsd(metadata_xsd, repoman_settings):
 	"""
 	Fetch metadata.xsd if it doesn't exist or the ctime is older than

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index a7150a9..af53f4b 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -23,7 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception):
 
 # import our initialized portage instance
 from repoman._portage import portage
-from repoman.metadata import metadata_dtd_uri, parse_metadata_use
+from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
 from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
@@ -32,16 +32,70 @@ from repoman.modules.scan.scanbase import ScanBase
 from portage.exception import InvalidAtom
 from portage import os
 from portage import _encodings, _unicode_encode
+from portage import exception
 from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
 
+if sys.hexversion >= 0x3000000:
+	# pylint: disable=W0622
+	basestring = str
+
 metadata_xml_encoding = 'UTF-8'
 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 	% (metadata_xml_encoding,)
 metadata_doctype_name = 'pkgmetadata'
 
 
+def parse_metadata_use(xml_tree):
+	"""
+	Records are wrapped in XML as per GLEP 56
+	returns a dict with keys constisting of USE flag names and values
+	containing their respective descriptions
+	"""
+	uselist = {}
+
+	usetags = xml_tree.findall("use")
+	if not usetags:
+		return uselist
+
+	# It's possible to have multiple 'use' elements.
+	for usetag in usetags:
+		flags = usetag.findall("flag")
+		if not flags:
+			# DTD allows use elements containing no flag elements.
+			continue
+
+		for flag in flags:
+			pkg_flag = flag.get("name")
+			if pkg_flag is None:
+				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
+			flag_restrict = flag.get("restrict")
+
+			# emulate the Element.itertext() method from python-2.7
+			inner_text = []
+			stack = []
+			stack.append(flag)
+			while stack:
+				obj = stack.pop()
+				if isinstance(obj, basestring):
+					inner_text.append(obj)
+					continue
+				if isinstance(obj.text, basestring):
+					inner_text.append(obj.text)
+				if isinstance(obj.tail, basestring):
+					stack.append(obj.tail)
+				stack.extend(reversed(obj))
+
+			if pkg_flag not in uselist:
+				uselist[pkg_flag] = {}
+
+			# (flag_restrict can be None)
+			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
+
+	return uselist
+
+
 class PkgMetadata(ScanBase, USEFlagChecks):
 	'''Package metadata.xml checks'''
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     b7eec0cd3bb59c2b21e91a71619212dbff8a5b0b
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Tue May  3 09:01:29 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun May  8 21:18:41 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7eec0cd

repoman: Migrate from XmlLint to etree.XMLSchema for validation

Remove No longer used repoman._xml module
This change based on work by Dirkjan Ochtman <djc <AT> gentoo.org>
Updated the change from XML.DTD to XMLSchema.
Additionally:
    Move the metadata.xsd path determination code to metadata.py.
    Add the DISTDIR backup location and fetching of the file if missing or stale.

 pym/repoman/_xml.py                              | 105 -----------------------
 pym/repoman/metadata.py                          |  21 +++++
 pym/repoman/modules/scan/metadata/pkgmetadata.py |   9 +-
 pym/repoman/scanner.py                           |  10 +--
 4 files changed, 26 insertions(+), 119 deletions(-)

diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
deleted file mode 100644
index 33a536a..0000000
--- a/pym/repoman/_xml.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
-import sys
-import xml
-
-# import our initialized portage instance
-from repoman._portage import portage
-
-from portage import os
-from portage.output import red
-from portage.process import find_binary
-
-from repoman.metadata import fetch_metadata_xsd
-from repoman._subprocess import repoman_getstatusoutput
-
-
-class _XMLParser(xml.etree.ElementTree.XMLParser):
-
-	def __init__(self, data, **kwargs):
-		xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
-		self._portage_data = data
-		if hasattr(self, 'parser'):
-			self._base_XmlDeclHandler = self.parser.XmlDeclHandler
-			self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
-			self._base_StartDoctypeDeclHandler = \
-				self.parser.StartDoctypeDeclHandler
-			self.parser.StartDoctypeDeclHandler = \
-				self._portage_StartDoctypeDeclHandler
-
-	def _portage_XmlDeclHandler(self, version, encoding, standalone):
-		if self._base_XmlDeclHandler is not None:
-			self._base_XmlDeclHandler(version, encoding, standalone)
-		self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
-
-	def _portage_StartDoctypeDeclHandler(
-		self, doctypeName, systemId, publicId, has_internal_subset):
-		if self._base_StartDoctypeDeclHandler is not None:
-			self._base_StartDoctypeDeclHandler(
-				doctypeName, systemId, publicId, has_internal_subset)
-		self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
-
-
-class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
-	"""
-	Implements doctype() as required to avoid deprecation warnings with
-	>=python-2.7.
-	"""
-	def doctype(self, name, pubid, system):
-		pass
-
-
-class XmlLint(object):
-
-	def __init__(self, options, repoman_settings, metadata_xsd=None):
-		self.metadata_xsd = (metadata_xsd or
-			os.path.join(repoman_settings["DISTDIR"], 'metadata.xsd'))
-		self.options = options
-		self.repoman_settings = repoman_settings
-		self._is_capable = metadata_xsd is not None
-		self.binary = None
-		self._check_capable()
-
-	def _check_capable(self):
-		if self.options.mode == "manifest":
-			return
-		self.binary = find_binary('xmllint')
-		if not self.binary:
-			print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
-		elif not self._is_capable:
-			if not fetch_metadata_xsd(self.metadata_xsd, self.repoman_settings):
-				sys.exit(1)
-			# this can be problematic if xmllint changes their output
-			self._is_capable = True
-
-	@property
-	def capable(self):
-		return self._is_capable
-
-	def check(self, checkdir, repolevel):
-		'''Runs checks on the package metadata.xml file
-
-		@param checkdir: string, path
-		@param repolevel: integer
-		@return boolean, False == bad metadata
-		'''
-		if not self.capable:
-			if self.options.xml_parse or repolevel == 3:
-				print("%s sorry, xmllint is needed.  failing\n" % red("!!!"))
-				sys.exit(1)
-			return True
-		# xmlint can produce garbage output even on success, so only dump
-		# the ouput when it fails.
-		st, out = repoman_getstatusoutput(
-			self.binary + " --nonet --noout --schema %s %s" % (
-				portage._shell_quote(self.metadata_xsd),
-				portage._shell_quote(
-					os.path.join(checkdir, "metadata.xml"))))
-		if st != os.EX_OK:
-			print(red("!!!") + " metadata.xml is invalid:")
-			for z in out.splitlines():
-				print(red("!!! ") + z)
-			return False
-		return True

diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index 7c64c8e..a9ad3e8 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -99,3 +99,24 @@ def fetch_metadata_xsd(metadata_xsd, repoman_settings):
 				pass
 
 	return True
+
+
+def get_metadata_xsd(repo_settings):
+	'''Locate and or fetch the metadata.xsd file
+
+	@param repo_settings: RepoSettings instance
+	@returns: path to the metadata.xsd file
+	'''
+	metadata_xsd = None
+	for path in reversed(repo_settings.repo_config.eclass_db.porttrees):
+		path = os.path.join(path, 'metadata/xml-schema/metadata.xsd')
+		if os.path.exists(path):
+			metadata_xsd = path
+			break
+	if metadata_xsd is None:
+		metadata_xsd = os.path.join(
+			repo_settings.repoman_settings["DISTDIR"], 'metadata.xsd'
+			)
+
+		fetch_metadata_xsd(metadata_xsd, repo_settings.repoman_settings)
+	return metadata_xsd

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 317ab56..3ca7897 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -26,7 +26,6 @@ from repoman._portage import portage
 from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
-from repoman._xml import XmlLint
 from repoman.modules.scan.scanbase import ScanBase
 
 from portage.exception import InvalidAtom
@@ -110,13 +109,11 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		repo_settings = kwargs.get('repo_settings')
 		self.qatracker = kwargs.get('qatracker')
 		self.options = kwargs.get('options')
-		metadata_xsd = kwargs.get('metadata_xsd')
+		self.metadata_xsd = kwargs.get('metadata_xsd')
 		self.globalUseFlags = kwargs.get('uselist')
 		self.repoman_settings = repo_settings.repoman_settings
 		self.musedict = {}
 		self.muselist = set()
-		self.xmllint = XmlLint(self.options, self.repoman_settings,
-			metadata_xsd=metadata_xsd)
 
 	def check(self, **kwargs):
 		'''Performs the checks on the metadata.xml for the package
@@ -129,7 +126,6 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 		xpkg = kwargs.get('xpkg')
 		checkdir = kwargs.get('checkdir')
 		checkdirlist = kwargs.get('checkdirlist').get()
-		repolevel = kwargs.get('repolevel')
 
 		self.musedict = {}
 		if self.options.mode in ['manifest']:
@@ -221,7 +217,8 @@ class PkgMetadata(ScanBase, USEFlagChecks):
 
 		# Only carry out if in package directory or check forced
 		if not metadata_bad:
-			if not self.xmllint.check(checkdir, repolevel):
+			validator = etree.XMLSchema(file=self.metadata_xsd)
+			if not validator.validate(_metadata_xml):
 				self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml")
 		del metadata_bad
 		self.muselist = frozenset(self.musedict)

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index fd07209..48d9001 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -10,6 +10,7 @@ from portage import normalize_path
 from portage import os
 from portage.output import green
 from portage.util.futures.extendedfutures import ExtendedFuture
+from repoman.metadata import get_metadata_xsd
 from repoman.modules.commit import repochecks
 from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
 from repoman.repos import repo_metadata
@@ -56,13 +57,6 @@ class Scanner(object):
 			portage.util.stack_lists([self.categories], incremental=1))
 		self.categories = self.repo_settings.repoman_settings.categories
 
-		metadata_xsd = None
-		for path in reversed(self.repo_settings.repo_config.eclass_db.porttrees):
-			path = os.path.join(path, 'metadata/xml-schema/metadata.xsd')
-			if os.path.exists(path):
-				metadata_xsd = path
-				break
-
 		self.portdb = repo_settings.portdb
 		self.portdb.settings = self.repo_settings.repoman_settings
 		# We really only need to cache the metadata that's necessary for visibility
@@ -187,7 +181,7 @@ class Scanner(object):
 			"qatracker": self.qatracker,
 			"vcs_settings": self.vcs_settings,
 			"options": self.options,
-			"metadata_xsd": metadata_xsd,
+			"metadata_xsd": get_metadata_xsd(self.repo_settings),
 			"uselist": uselist,
 			"checks": checks,
 			"repo_metadata": self.repo_metadata,

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     fdb0fd395a582996cf88df2819bf246a2ddcf914
Author:     Dirkjan Ochtman <dirkjan <AT> ochtman <DOT> nl>
AuthorDate: Mon Jan 25 19:40:29 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May  3 06:54:24 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=fdb0fd39

repoman: Move parse_metadata_use function to the pkgmetadata.py

 pym/repoman/metadata.py                          | 50 -----------------------
 pym/repoman/modules/scan/metadata/pkgmetadata.py | 52 +++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index a980184..7c64c8e 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -17,7 +17,6 @@ except ImportError:
 # import our initialized portage instance
 from repoman._portage import portage
 
-from portage import exception
 from portage import os
 from portage import shutil
 from portage.output import green
@@ -34,55 +33,6 @@ metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd'
 metadata_xsd_ctime_interval = 60 * 60 * 24 * 7  # 7 days
 
 
-def parse_metadata_use(xml_tree):
-	"""
-	Records are wrapped in XML as per GLEP 56
-	returns a dict with keys constisting of USE flag names and values
-	containing their respective descriptions
-	"""
-	uselist = {}
-
-	usetags = xml_tree.findall("use")
-	if not usetags:
-		return uselist
-
-	# It's possible to have multiple 'use' elements.
-	for usetag in usetags:
-		flags = usetag.findall("flag")
-		if not flags:
-			# DTD allows use elements containing no flag elements.
-			continue
-
-		for flag in flags:
-			pkg_flag = flag.get("name")
-			if pkg_flag is None:
-				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
-			flag_restrict = flag.get("restrict")
-
-			# emulate the Element.itertext() method from python-2.7
-			inner_text = []
-			stack = []
-			stack.append(flag)
-			while stack:
-				obj = stack.pop()
-				if isinstance(obj, basestring):
-					inner_text.append(obj)
-					continue
-				if isinstance(obj.text, basestring):
-					inner_text.append(obj.text)
-				if isinstance(obj.tail, basestring):
-					stack.append(obj.tail)
-				stack.extend(reversed(obj))
-
-			if pkg_flag not in uselist:
-				uselist[pkg_flag] = {}
-
-			# (flag_restrict can be None)
-			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
-
-	return uselist
-
-
 def fetch_metadata_xsd(metadata_xsd, repoman_settings):
 	"""
 	Fetch metadata.xsd if it doesn't exist or the ctime is older than

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index a7150a9..cde2ba0 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -23,7 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception):
 
 # import our initialized portage instance
 from repoman._portage import portage
-from repoman.metadata import metadata_dtd_uri, parse_metadata_use
+from repoman.metadata import metadata_dtd_uri
 from repoman.checks.herds.herdbase import get_herd_base
 from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError
 from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint
@@ -32,6 +32,7 @@ from repoman.modules.scan.scanbase import ScanBase
 from portage.exception import InvalidAtom
 from portage import os
 from portage import _encodings, _unicode_encode
+from portage import exception
 from portage.dep import Atom
 
 from .use_flags import USEFlagChecks
@@ -42,6 +43,55 @@ metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 metadata_doctype_name = 'pkgmetadata'
 
 
+def parse_metadata_use(xml_tree):
+	"""
+	Records are wrapped in XML as per GLEP 56
+	returns a dict with keys constisting of USE flag names and values
+	containing their respective descriptions
+	"""
+	uselist = {}
+
+	usetags = xml_tree.findall("use")
+	if not usetags:
+		return uselist
+
+	# It's possible to have multiple 'use' elements.
+	for usetag in usetags:
+		flags = usetag.findall("flag")
+		if not flags:
+			# DTD allows use elements containing no flag elements.
+			continue
+
+		for flag in flags:
+			pkg_flag = flag.get("name")
+			if pkg_flag is None:
+				raise exception.ParseError("missing 'name' attribute for 'flag' tag")
+			flag_restrict = flag.get("restrict")
+
+			# emulate the Element.itertext() method from python-2.7
+			inner_text = []
+			stack = []
+			stack.append(flag)
+			while stack:
+				obj = stack.pop()
+				if isinstance(obj, basestring):
+					inner_text.append(obj)
+					continue
+				if isinstance(obj.text, basestring):
+					inner_text.append(obj.text)
+				if isinstance(obj.tail, basestring):
+					stack.append(obj.tail)
+				stack.extend(reversed(obj))
+
+			if pkg_flag not in uselist:
+				uselist[pkg_flag] = {}
+
+			# (flag_restrict can be None)
+			uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split())
+
+	return uselist
+
+
 class PkgMetadata(ScanBase, USEFlagChecks):
 	'''Package metadata.xml checks'''
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     b275450533245b5097b5f6c08a84df999ed2c989
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 17 08:06:28 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Apr 25 15:03:37 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2754505

repoman: Use XML Schema for metadata.xml validation

 pym/repoman/_xml.py                              | 16 +++++-----
 pym/repoman/metadata.py                          | 39 ++++++++++++------------
 pym/repoman/modules/scan/metadata/__init__.py    |  2 +-
 pym/repoman/modules/scan/metadata/pkgmetadata.py |  6 ++--
 pym/repoman/scanner.py                           |  8 ++---
 5 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
index d55dda5..33a536a 100644
--- a/pym/repoman/_xml.py
+++ b/pym/repoman/_xml.py
@@ -12,7 +12,7 @@ from portage import os
 from portage.output import red
 from portage.process import find_binary
 
-from repoman.metadata import fetch_metadata_dtd
+from repoman.metadata import fetch_metadata_xsd
 from repoman._subprocess import repoman_getstatusoutput
 
 
@@ -53,12 +53,12 @@ class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
 
 class XmlLint(object):
 
-	def __init__(self, options, repoman_settings, metadata_dtd=None):
-		self.metadata_dtd = (metadata_dtd or
-			os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd'))
+	def __init__(self, options, repoman_settings, metadata_xsd=None):
+		self.metadata_xsd = (metadata_xsd or
+			os.path.join(repoman_settings["DISTDIR"], 'metadata.xsd'))
 		self.options = options
 		self.repoman_settings = repoman_settings
-		self._is_capable = metadata_dtd is not None
+		self._is_capable = metadata_xsd is not None
 		self.binary = None
 		self._check_capable()
 
@@ -69,7 +69,7 @@ class XmlLint(object):
 		if not self.binary:
 			print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
 		elif not self._is_capable:
-			if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings):
+			if not fetch_metadata_xsd(self.metadata_xsd, self.repoman_settings):
 				sys.exit(1)
 			# this can be problematic if xmllint changes their output
 			self._is_capable = True
@@ -93,8 +93,8 @@ class XmlLint(object):
 		# xmlint can produce garbage output even on success, so only dump
 		# the ouput when it fails.
 		st, out = repoman_getstatusoutput(
-			self.binary + " --nonet --noout --dtdvalid %s %s" % (
-				portage._shell_quote(self.metadata_dtd),
+			self.binary + " --nonet --noout --schema %s %s" % (
+				portage._shell_quote(self.metadata_xsd),
 				portage._shell_quote(
 					os.path.join(checkdir, "metadata.xml"))))
 		if st != os.EX_OK:

diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py
index e95ad41..7a514dc 100644
--- a/pym/repoman/metadata.py
+++ b/pym/repoman/metadata.py
@@ -33,8 +33,9 @@ metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
 	% (metadata_xml_encoding,)
 metadata_doctype_name = 'pkgmetadata'
 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
+metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd'
 # force refetch if the local copy creation time is older than this
-metadata_dtd_ctime_interval = 60 * 60 * 24 * 7  # 7 days
+metadata_xsd_ctime_interval = 60 * 60 * 24 * 7  # 7 days
 
 
 def parse_metadata_use(xml_tree):
@@ -86,36 +87,36 @@ def parse_metadata_use(xml_tree):
 	return uselist
 
 
-def fetch_metadata_dtd(metadata_dtd, repoman_settings):
+def fetch_metadata_xsd(metadata_xsd, repoman_settings):
 	"""
-	Fetch metadata.dtd if it doesn't exist or the ctime is older than
-	metadata_dtd_ctime_interval.
+	Fetch metadata.xsd if it doesn't exist or the ctime is older than
+	metadata_xsd_ctime_interval.
 	@rtype: bool
 	@return: True if successful, otherwise False
 	"""
 
 	must_fetch = True
-	metadata_dtd_st = None
+	metadata_xsd_st = None
 	current_time = int(time.time())
 	try:
-		metadata_dtd_st = os.stat(metadata_dtd)
+		metadata_xsd_st = os.stat(metadata_xsd)
 	except EnvironmentError as e:
 		if e.errno not in (errno.ENOENT, errno.ESTALE):
 			raise
 		del e
 	else:
-		# Trigger fetch if metadata.dtd mtime is old or clock is wrong.
-		if abs(current_time - metadata_dtd_st.st_ctime) \
-			< metadata_dtd_ctime_interval:
+		# Trigger fetch if metadata.xsd mtime is old or clock is wrong.
+		if abs(current_time - metadata_xsd_st.st_ctime) \
+			< metadata_xsd_ctime_interval:
 			must_fetch = False
 
 	if must_fetch:
 		print()
 		print(
-			"%s the local copy of metadata.dtd "
+			"%s the local copy of metadata.xsd "
 			"needs to be refetched, doing that now" % green("***"))
 		print()
-		parsed_url = urlparse(metadata_dtd_uri)
+		parsed_url = urlparse(metadata_xsd_uri)
 		setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
 		fcmd = repoman_settings.get(setting)
 		if not fcmd:
@@ -125,29 +126,29 @@ def fetch_metadata_dtd(metadata_dtd, repoman_settings):
 				return False
 
 		destdir = repoman_settings["DISTDIR"]
-		fd, metadata_dtd_tmp = tempfile.mkstemp(
-			prefix='metadata.dtd.', dir=destdir)
+		fd, metadata_xsd_tmp = tempfile.mkstemp(
+			prefix='metadata.xsd.', dir=destdir)
 		os.close(fd)
 
 		try:
 			if not portage.getbinpkg.file_get(
-				metadata_dtd_uri, destdir, fcmd=fcmd,
-				filename=os.path.basename(metadata_dtd_tmp)):
+				metadata_xsd_uri, destdir, fcmd=fcmd,
+				filename=os.path.basename(metadata_xsd_tmp)):
 				logging.error(
-					"failed to fetch metadata.dtd from '%s'" % metadata_dtd_uri)
+					"failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri)
 				return False
 
 			try:
 				portage.util.apply_secpass_permissions(
-					metadata_dtd_tmp,
+					metadata_xsd_tmp,
 					gid=portage.data.portage_gid, mode=0o664, mask=0o2)
 			except portage.exception.PortageException:
 				pass
 
-			shutil.move(metadata_dtd_tmp, metadata_dtd)
+			shutil.move(metadata_xsd_tmp, metadata_xsd)
 		finally:
 			try:
-				os.unlink(metadata_dtd_tmp)
+				os.unlink(metadata_xsd_tmp)
 			except OSError:
 				pass
 

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 98df9cf..1a6c0ed 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -18,7 +18,7 @@ module_spec = {
 			'functions': ['check'],
 			'func_desc': {
 			},
-			'mod_kwargs': ['repo_settings', 'qatracker', 'options', 'metadata_dtd',
+			'mod_kwargs': ['repo_settings', 'qatracker', 'options', 'metadata_xsd',
 			],
 			'func_kwargs': {
 				'checkdir': (None, None),

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 9216d77..25907b3 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -46,17 +46,17 @@ class PkgMetadata(ScanBase):
 		@param repo_settings: settings instance
 		@param qatracker: QATracker instance
 		@param options: argparse options instance
-		@param metadata_dtd: path of metadata.dtd
+		@param metadata_xsd: path of metadata.xsd
 		'''
 		super(PkgMetadata, self).__init__(**kwargs)
 		repo_settings = kwargs.get('repo_settings')
 		self.qatracker = kwargs.get('qatracker')
 		self.options = kwargs.get('options')
-		metadata_dtd = kwargs.get('metadata_dtd')
+		metadata_xsd = kwargs.get('metadata_xsd')
 		self.repoman_settings = repo_settings.repoman_settings
 		self.musedict = {}
 		self.xmllint = XmlLint(self.options, self.repoman_settings,
-			metadata_dtd=metadata_dtd)
+			metadata_xsd=metadata_xsd)
 
 	def check(self, **kwargs):
 		'''Performs the checks on the metadata.xml for the package

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 74bb7e3..508fedb 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -56,11 +56,11 @@ class Scanner(object):
 			portage.util.stack_lists([self.categories], incremental=1))
 		self.categories = self.repo_settings.repoman_settings.categories
 
-		metadata_dtd = None
+		metadata_xsd = None
 		for path in reversed(self.repo_settings.repo_config.eclass_db.porttrees):
-			path = os.path.join(path, 'metadata/dtd/metadata.dtd')
+			path = os.path.join(path, 'metadata/xml-schema/metadata.xsd')
 			if os.path.exists(path):
-				metadata_dtd = path
+				metadata_xsd = path
 				break
 
 		self.portdb = repo_settings.portdb
@@ -189,7 +189,7 @@ class Scanner(object):
 			"qatracker": self.qatracker,
 			"vcs_settings": self.vcs_settings,
 			"options": self.options,
-			"metadata_dtd": metadata_dtd,
+			"metadata_xsd": metadata_xsd,
 			"uselist": uselist,
 			"checks": self.checks,
 			"repo_metadata": self.repo_metadata,

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     f70c87e8f0704e25550a9401ab1785e692efe3c3
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Mar 10 23:47:35 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f70c87e8

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml.

 pym/repoman/modules/scan/metadata/__init__.py |  9 +++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 4f376e1..f619764 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -55,6 +55,15 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'sourcefile': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 4f4f78b..2f717e2 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -270,7 +270,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -318,11 +317,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     048c07d97b0a813ab0547ab23e27f6667751d166
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Mar  7 21:21:33 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=048c07d9

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml.

 pym/repoman/modules/scan/metadata/__init__.py |  9 +++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 4f376e1..f619764 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -55,6 +55,15 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'sourcefile': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index ca1de5f..da9dcac 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -270,7 +270,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -318,11 +317,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     4a4f67beab47c0adcc8048afb4159794a486faaf
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 30 20:25:24 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a4f67be

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml.

 pym/repoman/modules/scan/metadata/__init__.py |  9 +++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 4f376e1..f619764 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -55,6 +55,15 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'sourcefile': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index ca1de5f..da9dcac 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -270,7 +270,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -318,11 +317,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     07b87ced7b378bace32904c6322cc3d00d610a01
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:56:25 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 30 20:25:21 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=07b87ced

repoman: Migrate more metadata checks to ebuild_metadata.py

 .../modules/scan/metadata/ebuild_metadata.py       | 32 ++++++++++++++++++++--
 pym/repoman/scanner.py                             | 17 ------------
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 2dc1db2..77c947e 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+from repoman.qa_data import missingvars
+
 if sys.hexversion >= 0x3000000:
 	basestring = str
 
@@ -16,7 +18,7 @@ class EbuildMetadata(object):
 	def __init__(self, **kwargs):
 		self.qatracker = kwargs.get('qatracker')
 
-	def check(self, **kwargs):
+	def invalidchar(self, **kwargs):
 		ebuild = kwargs.get('ebuild')
 		for k, v in ebuild.metadata.items():
 			if not isinstance(v, basestring):
@@ -28,9 +30,35 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	def missing(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for pos, missing_var in enumerate(missingvars):
+			if not ebuild.metadata.get(missing_var):
+				if kwargs.get('catdir') == "virtual" and \
+					missing_var in ("HOMEPAGE", "LICENSE"):
+					continue
+				if kwargs.get('live_ebuild') and missing_var == "KEYWORDS":
+					continue
+				myqakey = missingvars[pos] + ".missing"
+				self.qatracker.add_error(myqakey, '%s/%s.ebuild'
+					% (kwargs.get('xpkg'), kwargs.get('y_ebuild')))
+		return {'continue': False}
+
+	def old_virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
 		if ebuild.metadata.get("PROVIDE"):
 			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+		return {'continue': False}
 
+	def virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		if kwargs.get('catdir') == "virtual":
+			for var in ("HOMEPAGE", "LICENSE"):
+				if ebuild.metadata.get(var):
+					myqakey = var + ".virtual"
+					self.qatracker.add_error(myqakey, ebuild.relative_path)
 		return {'continue': False}
 
 	@property
@@ -39,4 +67,4 @@ class EbuildMetadata(object):
 
 	@property
 	def runInEbuilds(self):
-		return (True, [self.check])
+		return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index ec126e6..1a2de05 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -327,23 +327,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for pos, missing_var in enumerate(missingvars):
-				if not dynamic_data['ebuild'].metadata.get(missing_var):
-					if dynamic_data['catdir'] == "virtual" and \
-						missing_var in ("HOMEPAGE", "LICENSE"):
-						continue
-					if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS":
-						continue
-					myqakey = missingvars[pos] + ".missing"
-					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
-
-			if dynamic_data['catdir'] == "virtual":
-				for var in ("HOMEPAGE", "LICENSE"):
-					if dynamic_data['ebuild'].metadata.get(var):
-						myqakey = var + ".virtual"
-						self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['live_ebuild'] and self.repo_settings.repo_config.name == "gentoo":
 				self.liveeclasscheck.check(
 					dynamic_data['pkg'], xpkg, dynamic_data['ebuild'], y_ebuild, dynamic_data['ebuild'].keywords, self.repo_metadata['pmaskdict'])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     de3ef8986c2607fb4741ab664b31c29e1a3c936e
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:31:26 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 30 20:25:21 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=de3ef898

scanner.py: Migrate another metadata check to ebuild_metadata

 pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++
 pym/repoman/scanner.py                               | 2 --
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 143a40e..2dc1db2 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -28,6 +28,9 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		if ebuild.metadata.get("PROVIDE"):
+			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+
 		return {'continue': False}
 
 	@property

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index c5601dd..ec126e6 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -327,8 +327,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
-				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 
 			for pos, missing_var in enumerate(missingvars):
 				if not dynamic_data['ebuild'].metadata.get(missing_var):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     75d44c473dfe67a828c33a6ce3f097ab3c483f13
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:31:26 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 30 07:50:18 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=75d44c47

scanner.py: Migrate another metadata check to ebuild_metadata

 pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++
 pym/repoman/scanner.py                               | 2 --
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 143a40e..2dc1db2 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -28,6 +28,9 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		if ebuild.metadata.get("PROVIDE"):
+			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+
 		return {'continue': False}
 
 	@property

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 006c3f7..803e5a3 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -327,8 +327,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
-				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 
 			for pos, missing_var in enumerate(missingvars):
 				if not dynamic_data['ebuild'].metadata.get(missing_var):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     ecb12dd210c1452c3690702640161d252e8baf00
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:56:25 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 30 07:50:18 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ecb12dd2

repoman: Migrate more metadata checks to ebuild_metadata.py

 .../modules/scan/metadata/ebuild_metadata.py       | 32 ++++++++++++++++++++--
 pym/repoman/scanner.py                             | 17 ------------
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 2dc1db2..77c947e 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+from repoman.qa_data import missingvars
+
 if sys.hexversion >= 0x3000000:
 	basestring = str
 
@@ -16,7 +18,7 @@ class EbuildMetadata(object):
 	def __init__(self, **kwargs):
 		self.qatracker = kwargs.get('qatracker')
 
-	def check(self, **kwargs):
+	def invalidchar(self, **kwargs):
 		ebuild = kwargs.get('ebuild')
 		for k, v in ebuild.metadata.items():
 			if not isinstance(v, basestring):
@@ -28,9 +30,35 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	def missing(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for pos, missing_var in enumerate(missingvars):
+			if not ebuild.metadata.get(missing_var):
+				if kwargs.get('catdir') == "virtual" and \
+					missing_var in ("HOMEPAGE", "LICENSE"):
+					continue
+				if kwargs.get('live_ebuild') and missing_var == "KEYWORDS":
+					continue
+				myqakey = missingvars[pos] + ".missing"
+				self.qatracker.add_error(myqakey, '%s/%s.ebuild'
+					% (kwargs.get('xpkg'), kwargs.get('y_ebuild')))
+		return {'continue': False}
+
+	def old_virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
 		if ebuild.metadata.get("PROVIDE"):
 			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+		return {'continue': False}
 
+	def virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		if kwargs.get('catdir') == "virtual":
+			for var in ("HOMEPAGE", "LICENSE"):
+				if ebuild.metadata.get(var):
+					myqakey = var + ".virtual"
+					self.qatracker.add_error(myqakey, ebuild.relative_path)
 		return {'continue': False}
 
 	@property
@@ -39,4 +67,4 @@ class EbuildMetadata(object):
 
 	@property
 	def runInEbuilds(self):
-		return (True, [self.check])
+		return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 803e5a3..5c35519 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -327,23 +327,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for pos, missing_var in enumerate(missingvars):
-				if not dynamic_data['ebuild'].metadata.get(missing_var):
-					if dynamic_data['catdir'] == "virtual" and \
-						missing_var in ("HOMEPAGE", "LICENSE"):
-						continue
-					if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS":
-						continue
-					myqakey = missingvars[pos] + ".missing"
-					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
-
-			if dynamic_data['catdir'] == "virtual":
-				for var in ("HOMEPAGE", "LICENSE"):
-					if dynamic_data['ebuild'].metadata.get(var):
-						myqakey = var + ".virtual"
-						self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['live_ebuild'] and self.repo_settings.repo_config.name == "gentoo":
 				self.liveeclasscheck.check(
 					dynamic_data['pkg'], xpkg, dynamic_data['ebuild'], y_ebuild, dynamic_data['ebuild'].keywords, self.repo_metadata['pmaskdict'])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     b1711bafa09ca63c8d256c4a57dd49d25d049cd7
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 30 06:51:58 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b1711baf

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml.

 pym/repoman/modules/scan/metadata/__init__.py |  9 +++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 4f376e1..f619764 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -55,6 +55,15 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'sourcefile': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 1e61eb6..4dd5408 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -270,7 +270,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -318,11 +317,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     0b5a15fbbd8d012f82f9014d6ef013fb00efcd0c
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:56:25 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 30 06:33:56 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b5a15fb

repoman: Migrate more metadata checks to ebuild_metadata.py

 .../modules/scan/metadata/ebuild_metadata.py       | 32 ++++++++++++++++++++--
 pym/repoman/scanner.py                             | 17 ------------
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 2dc1db2..77c947e 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+from repoman.qa_data import missingvars
+
 if sys.hexversion >= 0x3000000:
 	basestring = str
 
@@ -16,7 +18,7 @@ class EbuildMetadata(object):
 	def __init__(self, **kwargs):
 		self.qatracker = kwargs.get('qatracker')
 
-	def check(self, **kwargs):
+	def invalidchar(self, **kwargs):
 		ebuild = kwargs.get('ebuild')
 		for k, v in ebuild.metadata.items():
 			if not isinstance(v, basestring):
@@ -28,9 +30,35 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	def missing(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for pos, missing_var in enumerate(missingvars):
+			if not ebuild.metadata.get(missing_var):
+				if kwargs.get('catdir') == "virtual" and \
+					missing_var in ("HOMEPAGE", "LICENSE"):
+					continue
+				if kwargs.get('live_ebuild') and missing_var == "KEYWORDS":
+					continue
+				myqakey = missingvars[pos] + ".missing"
+				self.qatracker.add_error(myqakey, '%s/%s.ebuild'
+					% (kwargs.get('xpkg'), kwargs.get('y_ebuild')))
+		return {'continue': False}
+
+	def old_virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
 		if ebuild.metadata.get("PROVIDE"):
 			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+		return {'continue': False}
 
+	def virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		if kwargs.get('catdir') == "virtual":
+			for var in ("HOMEPAGE", "LICENSE"):
+				if ebuild.metadata.get(var):
+					myqakey = var + ".virtual"
+					self.qatracker.add_error(myqakey, ebuild.relative_path)
 		return {'continue': False}
 
 	@property
@@ -39,4 +67,4 @@ class EbuildMetadata(object):
 
 	@property
 	def runInEbuilds(self):
-		return (True, [self.check])
+		return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 803e5a3..5c35519 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -327,23 +327,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for pos, missing_var in enumerate(missingvars):
-				if not dynamic_data['ebuild'].metadata.get(missing_var):
-					if dynamic_data['catdir'] == "virtual" and \
-						missing_var in ("HOMEPAGE", "LICENSE"):
-						continue
-					if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS":
-						continue
-					myqakey = missingvars[pos] + ".missing"
-					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
-
-			if dynamic_data['catdir'] == "virtual":
-				for var in ("HOMEPAGE", "LICENSE"):
-					if dynamic_data['ebuild'].metadata.get(var):
-						myqakey = var + ".virtual"
-						self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['live_ebuild'] and self.repo_settings.repo_config.name == "gentoo":
 				self.liveeclasscheck.check(
 					dynamic_data['pkg'], xpkg, dynamic_data['ebuild'], y_ebuild, dynamic_data['ebuild'].keywords, self.repo_metadata['pmaskdict'])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     3c7b49d26de283239a1789832972353a623ec676
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 21 18:29:07 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Fri Jan 29 04:53:00 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3c7b49d2

repoman: Fix a traceback due to xmlint not being installed for a manifest generation

Mike helped find teh fact that self.binary was None, causing the misleading traceback.
Tighten up the logic in XmlLint.
Bypass the PkgMetadata check for manifest mode.

floppym@naomi btrfs-progs % repoman manifest
>>> Creating Manifest for /home/floppym/repos/gentoo/sys-fs/btrfs-progs
Traceback (most recent call last):
  File "/home/floppym/bin/repoman", line 37, in <module>
    repoman_main(sys.argv[1:])
  File "/home/floppym/src/portage/pym/repoman/main.py", line 111, in repoman_main
    can_force = scanner.scan_pkgs(can_force)
  File "/home/floppym/src/portage/pym/repoman/scanner.py", line 242, in scan_pkgs
    rdata = func(**dynamic_data)
  File "/home/floppym/src/portage/pym/repoman/modules/scan/metadata/pkgmetadata.py", line 180, in check
    if not self.xmllint.check(checkdir, repolevel):
  File "/home/floppym/src/portage/pym/repoman/_xml.py", line 98, in check
    os.path.join(checkdir, "metadata.xml"))))
TypeError: unsupported operand type(s) for +: 'NoneType' and 'str'

 pym/repoman/_xml.py                              | 2 +-
 pym/repoman/modules/scan/metadata/pkgmetadata.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
index 2661f14..f7ff9fb 100644
--- a/pym/repoman/_xml.py
+++ b/pym/repoman/_xml.py
@@ -60,6 +60,7 @@ class XmlLint(object):
 		self.repoman_settings = repoman_settings
 		self._is_capable = metadata_dtd is not None
 		self.binary = None
+		self._is_capable = False
 		self._check_capable()
 
 	def _check_capable(self):
@@ -68,7 +69,6 @@ class XmlLint(object):
 		self.binary = find_binary('xmllint')
 		if not self.binary:
 			print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
-			self._is_capable = False
 		elif not self._is_capable:
 			if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings):
 				sys.exit(1)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 220fd23..8e93457 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -71,6 +71,9 @@ class PkgMetadata(ScanBase):
 		repolevel = kwargs.get('repolevel')
 
 		self.musedict = {}
+		if self.options.mode in ['manifest']:
+			return {'continue': False, 'muselist': frozenset(self.musedict)}
+
 		# metadata.xml file check
 		if "metadata.xml" not in checkdirlist:
 			self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     20b86b331841e540b9fca8b22c0e615ed1fb7c77
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Jan 27 22:44:25 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=20b86b33

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml

 pym/repoman/modules/scan/metadata/__init__.py |  9 +++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 4f376e1..f619764 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -55,6 +55,15 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'sourcefile': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 4472f88..e486282 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -272,7 +272,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -320,11 +319,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     563f65e4b167f213981ec030c71ebd35a2ca9780
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 21 18:29:07 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Jan 27 22:44:26 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=563f65e4

repoman: Fix a traceback due to xmlint not being installed for a manifest generation

Mike helped find teh fact that self.binary was None, causing the misleading traceback.
Tighten up the logic in XmlLint.
Bypass the PkgMetadata check for manifest mode.

floppym@naomi btrfs-progs % repoman manifest
>>> Creating Manifest for /home/floppym/repos/gentoo/sys-fs/btrfs-progs
Traceback (most recent call last):
  File "/home/floppym/bin/repoman", line 37, in <module>
    repoman_main(sys.argv[1:])
  File "/home/floppym/src/portage/pym/repoman/main.py", line 111, in repoman_main
    can_force = scanner.scan_pkgs(can_force)
  File "/home/floppym/src/portage/pym/repoman/scanner.py", line 242, in scan_pkgs
    rdata = func(**dynamic_data)
  File "/home/floppym/src/portage/pym/repoman/modules/scan/metadata/pkgmetadata.py", line 180, in check
    if not self.xmllint.check(checkdir, repolevel):
  File "/home/floppym/src/portage/pym/repoman/_xml.py", line 98, in check
    os.path.join(checkdir, "metadata.xml"))))
TypeError: unsupported operand type(s) for +: 'NoneType' and 'str'

 pym/repoman/_xml.py                              | 2 +-
 pym/repoman/modules/scan/metadata/pkgmetadata.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
index 2661f14..f7ff9fb 100644
--- a/pym/repoman/_xml.py
+++ b/pym/repoman/_xml.py
@@ -60,6 +60,7 @@ class XmlLint(object):
 		self.repoman_settings = repoman_settings
 		self._is_capable = metadata_dtd is not None
 		self.binary = None
+		self._is_capable = False
 		self._check_capable()
 
 	def _check_capable(self):
@@ -68,7 +69,6 @@ class XmlLint(object):
 		self.binary = find_binary('xmllint')
 		if not self.binary:
 			print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
-			self._is_capable = False
 		elif not self._is_capable:
 			if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings):
 				sys.exit(1)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 220fd23..8e93457 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -71,6 +71,9 @@ class PkgMetadata(ScanBase):
 		repolevel = kwargs.get('repolevel')
 
 		self.musedict = {}
+		if self.options.mode in ['manifest']:
+			return {'continue': False, 'muselist': frozenset(self.musedict)}
+
 		# metadata.xml file check
 		if "metadata.xml" not in checkdirlist:
 			self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     0dcc20356b9c4e482c0e898d4a16fe63fc13e14d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:56:25 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Jan 27 22:44:23 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0dcc2035

repoman: Migrate more metadata checks to ebuild_metadata.py

 .../modules/scan/metadata/ebuild_metadata.py       | 32 ++++++++++++++++++++--
 pym/repoman/scanner.py                             | 17 ------------
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 2dc1db2..77c947e 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+from repoman.qa_data import missingvars
+
 if sys.hexversion >= 0x3000000:
 	basestring = str
 
@@ -16,7 +18,7 @@ class EbuildMetadata(object):
 	def __init__(self, **kwargs):
 		self.qatracker = kwargs.get('qatracker')
 
-	def check(self, **kwargs):
+	def invalidchar(self, **kwargs):
 		ebuild = kwargs.get('ebuild')
 		for k, v in ebuild.metadata.items():
 			if not isinstance(v, basestring):
@@ -28,9 +30,35 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	def missing(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for pos, missing_var in enumerate(missingvars):
+			if not ebuild.metadata.get(missing_var):
+				if kwargs.get('catdir') == "virtual" and \
+					missing_var in ("HOMEPAGE", "LICENSE"):
+					continue
+				if kwargs.get('live_ebuild') and missing_var == "KEYWORDS":
+					continue
+				myqakey = missingvars[pos] + ".missing"
+				self.qatracker.add_error(myqakey, '%s/%s.ebuild'
+					% (kwargs.get('xpkg'), kwargs.get('y_ebuild')))
+		return {'continue': False}
+
+	def old_virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
 		if ebuild.metadata.get("PROVIDE"):
 			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+		return {'continue': False}
 
+	def virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		if kwargs.get('catdir') == "virtual":
+			for var in ("HOMEPAGE", "LICENSE"):
+				if ebuild.metadata.get(var):
+					myqakey = var + ".virtual"
+					self.qatracker.add_error(myqakey, ebuild.relative_path)
 		return {'continue': False}
 
 	@property
@@ -39,4 +67,4 @@ class EbuildMetadata(object):
 
 	@property
 	def runInEbuilds(self):
-		return (True, [self.check])
+		return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 3ea27b6..335590b 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -327,23 +327,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for pos, missing_var in enumerate(missingvars):
-				if not dynamic_data['ebuild'].metadata.get(missing_var):
-					if dynamic_data['catdir'] == "virtual" and \
-						missing_var in ("HOMEPAGE", "LICENSE"):
-						continue
-					if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS":
-						continue
-					myqakey = missingvars[pos] + ".missing"
-					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
-
-			if dynamic_data['catdir'] == "virtual":
-				for var in ("HOMEPAGE", "LICENSE"):
-					if dynamic_data['ebuild'].metadata.get(var):
-						myqakey = var + ".virtual"
-						self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['live_ebuild'] and self.repo_settings.repo_config.name == "gentoo":
 				self.liveeclasscheck.check(
 					dynamic_data['pkg'], xpkg, dynamic_data['ebuild'], y_ebuild, dynamic_data['ebuild'].keywords, self.repo_metadata['pmaskdict'])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     13e0d777ca6dbb32f75e7600a3260ebb4d3803d5
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 21 18:29:07 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 23 01:38:03 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=13e0d777

repoman: Fix a traceback due to xmlint not being installed for a manifest generation

Mike helped find teh fact that self.binary was None, causing the misleading traceback.
Tighten up the logic in XmlLint.
Bypass the PkgMetadata check for manifest mode.

floppym@naomi btrfs-progs % repoman manifest
>>> Creating Manifest for /home/floppym/repos/gentoo/sys-fs/btrfs-progs
Traceback (most recent call last):
  File "/home/floppym/bin/repoman", line 37, in <module>
    repoman_main(sys.argv[1:])
  File "/home/floppym/src/portage/pym/repoman/main.py", line 111, in repoman_main
    can_force = scanner.scan_pkgs(can_force)
  File "/home/floppym/src/portage/pym/repoman/scanner.py", line 242, in scan_pkgs
    rdata = func(**dynamic_data)
  File "/home/floppym/src/portage/pym/repoman/modules/scan/metadata/pkgmetadata.py", line 180, in check
    if not self.xmllint.check(checkdir, repolevel):
  File "/home/floppym/src/portage/pym/repoman/_xml.py", line 98, in check
    os.path.join(checkdir, "metadata.xml"))))
TypeError: unsupported operand type(s) for +: 'NoneType' and 'str'

 pym/repoman/_xml.py                              | 2 +-
 pym/repoman/modules/scan/metadata/pkgmetadata.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
index 2661f14..f7ff9fb 100644
--- a/pym/repoman/_xml.py
+++ b/pym/repoman/_xml.py
@@ -60,6 +60,7 @@ class XmlLint(object):
 		self.repoman_settings = repoman_settings
 		self._is_capable = metadata_dtd is not None
 		self.binary = None
+		self._is_capable = False
 		self._check_capable()
 
 	def _check_capable(self):
@@ -68,7 +69,6 @@ class XmlLint(object):
 		self.binary = find_binary('xmllint')
 		if not self.binary:
 			print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
-			self._is_capable = False
 		elif not self._is_capable:
 			if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings):
 				sys.exit(1)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 220fd23..8e93457 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -71,6 +71,9 @@ class PkgMetadata(ScanBase):
 		repolevel = kwargs.get('repolevel')
 
 		self.musedict = {}
+		if self.options.mode in ['manifest']:
+			return {'continue': False, 'muselist': frozenset(self.musedict)}
+
 		# metadata.xml file check
 		if "metadata.xml" not in checkdirlist:
 			self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     24aa36db0fcd5cddd2ed58e0c5735413a535e1bc
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 23 01:37:59 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=24aa36db

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml

 pym/repoman/modules/scan/metadata/__init__.py |  9 +++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 4f376e1..f619764 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -55,6 +55,15 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'sourcefile': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 1cd37d0..4cc2e67 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -269,7 +269,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -317,11 +316,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     2f2c6d6d06c42dff98bdf9950b1d1c14efb819c5
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 10:35:49 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jan 23 01:23:14 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=2f2c6d6d

repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check

 pym/repoman/modules/scan/metadata/__init__.py      | 11 +++++-
 .../modules/scan/metadata/ebuild_metadata.py       | 39 ++++++++++++++++++++++
 pym/repoman/scanner.py                             | 27 +--------------
 3 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index ae9b18b..bacedf5 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -10,7 +10,7 @@ module_spec = {
 	'name': 'metadata',
 	'description': doc,
 	'provides':{
-		'metadata-module': {
+		'pkg-metadata': {
 			'name': "pkgmetadata",
 			'sourcefile': "pkgmetadata",
 			'class': "PkgMetadata",
@@ -19,6 +19,15 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'ebuild-metadata': {
+			'name': "ebuild_metadata",
+			'sourcefile': "ebuild_metadata",
+			'class': "EbuildMetadata",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
new file mode 100644
index 0000000..143a40e
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -0,0 +1,39 @@
+# -*- coding:utf-8 -*-
+
+'''Ebuild Metadata Checks'''
+
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+class EbuildMetadata(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for k, v in ebuild.metadata.items():
+			if not isinstance(v, basestring):
+				continue
+			m = NON_ASCII_RE.search(v)
+			if m is not None:
+				self.qatracker.add_error(
+					"variable.invalidchar",
+					"%s: %s variable contains non-ASCII "
+					"character at position %s" %
+					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index a8aa2f3..6f3fb53 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals
 import copy
 import io
 import logging
-import re
-import sys
 from itertools import chain
 from pprint import pformat
 
@@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
 MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
 
 
-
-if sys.hexversion >= 0x3000000:
-	basestring = str
-
-NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
-
-
 def sort_key(item):
 	return item[2].sub_path
 
 
-
 class Scanner(object):
 	'''Primary scan class.  Operates all the small Q/A tests and checks'''
 
@@ -311,7 +301,7 @@ class Scanner(object):
 			# initialize per ebuild plugin checks here
 			# need to set it up for ==> self.modules_list or some other ordered list
 			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
-				('eapi', 'EAPIChecks')]:
+				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]:
 				if mod[0]:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])
 					logging.debug("Initializing class name: %s", mod_class.__name__)
@@ -338,21 +328,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for k, v in dynamic_data['ebuild'].metadata.items():
-				if not isinstance(v, basestring):
-					continue
-				m = NON_ASCII_RE.search(v)
-				if m is not None:
-					self.qatracker.add_error(
-						"variable.invalidchar",
-						"%s: %s variable contains non-ASCII "
-						"character at position %s" %
-						(dynamic_data['ebuild'].relative_path, k, m.start() + 1))
-
-			if not dynamic_data['src_uri_error']:
-				self.thirdparty.check(dynamic_data['ebuild'].metadata, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
 				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     009529b32d753e53e43673c53db8004703a53c17
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 10:35:49 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Fri Jan 22 18:44:10 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=009529b3

repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check

 pym/repoman/modules/scan/metadata/__init__.py      | 10 +++++-
 .../modules/scan/metadata/ebuild_metadata.py       | 39 ++++++++++++++++++++++
 pym/repoman/scanner.py                             | 27 +--------------
 3 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 7327ec0..eba6565 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -10,7 +10,7 @@ module_spec = {
 	'name': 'metadata',
 	'description': doc,
 	'provides':{
-		'metadata-module': {
+		'pkg-metadata': {
 			'name': "pkgmetadata",
 			'class': "PkgMetadata",
 			'description': doc,
@@ -18,6 +18,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'ebuild-metadata': {
+			'name': "ebuild_metadata",
+			'class': "EbuildMetadata",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
new file mode 100644
index 0000000..143a40e
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -0,0 +1,39 @@
+# -*- coding:utf-8 -*-
+
+'''Ebuild Metadata Checks'''
+
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+class EbuildMetadata(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for k, v in ebuild.metadata.items():
+			if not isinstance(v, basestring):
+				continue
+			m = NON_ASCII_RE.search(v)
+			if m is not None:
+				self.qatracker.add_error(
+					"variable.invalidchar",
+					"%s: %s variable contains non-ASCII "
+					"character at position %s" %
+					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index a8aa2f3..6f3fb53 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals
 import copy
 import io
 import logging
-import re
-import sys
 from itertools import chain
 from pprint import pformat
 
@@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
 MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
 
 
-
-if sys.hexversion >= 0x3000000:
-	basestring = str
-
-NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
-
-
 def sort_key(item):
 	return item[2].sub_path
 
 
-
 class Scanner(object):
 	'''Primary scan class.  Operates all the small Q/A tests and checks'''
 
@@ -311,7 +301,7 @@ class Scanner(object):
 			# initialize per ebuild plugin checks here
 			# need to set it up for ==> self.modules_list or some other ordered list
 			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
-				('eapi', 'EAPIChecks')]:
+				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]:
 				if mod[0]:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])
 					logging.debug("Initializing class name: %s", mod_class.__name__)
@@ -338,21 +328,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for k, v in dynamic_data['ebuild'].metadata.items():
-				if not isinstance(v, basestring):
-					continue
-				m = NON_ASCII_RE.search(v)
-				if m is not None:
-					self.qatracker.add_error(
-						"variable.invalidchar",
-						"%s: %s variable contains non-ASCII "
-						"character at position %s" %
-						(dynamic_data['ebuild'].relative_path, k, m.start() + 1))
-
-			if not dynamic_data['src_uri_error']:
-				self.thirdparty.check(dynamic_data['ebuild'].metadata, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
 				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     fd22acf104e1a2a501ea8ae940de11f62dbd58a6
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 21 18:29:07 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 21 19:28:20 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=fd22acf1

repoman: Fix a traceback due to xmlint not being installed for a manifest generation

Mike helped find teh fact that self.binary was None, causing the misleading traceback.
Tighten up the logic in XmlLint.
Bypass the PkgMetadata check for manifest mode.

floppym@naomi btrfs-progs % repoman manifest
>>> Creating Manifest for /home/floppym/repos/gentoo/sys-fs/btrfs-progs
Traceback (most recent call last):
  File "/home/floppym/bin/repoman", line 37, in <module>
    repoman_main(sys.argv[1:])
  File "/home/floppym/src/portage/pym/repoman/main.py", line 111, in repoman_main
    can_force = scanner.scan_pkgs(can_force)
  File "/home/floppym/src/portage/pym/repoman/scanner.py", line 242, in scan_pkgs
    rdata = func(**dynamic_data)
  File "/home/floppym/src/portage/pym/repoman/modules/scan/metadata/pkgmetadata.py", line 180, in check
    if not self.xmllint.check(checkdir, repolevel):
  File "/home/floppym/src/portage/pym/repoman/_xml.py", line 98, in check
    os.path.join(checkdir, "metadata.xml"))))
TypeError: unsupported operand type(s) for +: 'NoneType' and 'str'

 pym/repoman/_xml.py                              | 2 +-
 pym/repoman/modules/scan/metadata/pkgmetadata.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py
index 43fc930..334377d 100644
--- a/pym/repoman/_xml.py
+++ b/pym/repoman/_xml.py
@@ -58,6 +58,7 @@ class XmlLint(object):
 		self.repoman_settings = repoman_settings
 		self._is_capable = metadata_dtd is not None
 		self.binary = None
+		self._is_capable = False
 		self._check_capable()
 
 	def _check_capable(self):
@@ -66,7 +67,6 @@ class XmlLint(object):
 		self.binary = find_binary('xmllint')
 		if not self.binary:
 			print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
-			self._is_capable = False
 		elif not self._is_capable:
 			if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings):
 				sys.exit(1)

diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py
index 220fd23..8e93457 100644
--- a/pym/repoman/modules/scan/metadata/pkgmetadata.py
+++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py
@@ -71,6 +71,9 @@ class PkgMetadata(ScanBase):
 		repolevel = kwargs.get('repolevel')
 
 		self.musedict = {}
+		if self.options.mode in ['manifest']:
+			return {'continue': False, 'muselist': frozenset(self.musedict)}
+
 		# metadata.xml file check
 		if "metadata.xml" not in checkdirlist:
 			self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     4602b7a0c896840ddfa53e92462050aca89e8574
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 21 19:28:19 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=4602b7a0

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml

 pym/repoman/modules/scan/metadata/__init__.py |  8 ++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 6ab44f6..ed4a967 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -50,6 +50,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 1cd37d0..4cc2e67 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -269,7 +269,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -317,11 +316,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     02395b0dea31388acbc5ae58ebc5d90f64b2d10a
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 21 02:53:44 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=02395b0d

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml

 pym/repoman/modules/scan/metadata/__init__.py |  8 ++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 6ab44f6..ed4a967 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -50,6 +50,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 1cd37d0..4cc2e67 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -269,7 +269,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -317,11 +316,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     18d5f07ffda73e32a083baf5b9f4944bcf733718
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:56:25 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 21 00:35:22 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=18d5f07f

repoman: Migrate more metadata checks to ebuild_metadata.py

 .../modules/scan/metadata/ebuild_metadata.py       | 32 ++++++++++++++++++++--
 pym/repoman/scanner.py                             | 17 ------------
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 2dc1db2..77c947e 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+from repoman.qa_data import missingvars
+
 if sys.hexversion >= 0x3000000:
 	basestring = str
 
@@ -16,7 +18,7 @@ class EbuildMetadata(object):
 	def __init__(self, **kwargs):
 		self.qatracker = kwargs.get('qatracker')
 
-	def check(self, **kwargs):
+	def invalidchar(self, **kwargs):
 		ebuild = kwargs.get('ebuild')
 		for k, v in ebuild.metadata.items():
 			if not isinstance(v, basestring):
@@ -28,9 +30,35 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	def missing(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for pos, missing_var in enumerate(missingvars):
+			if not ebuild.metadata.get(missing_var):
+				if kwargs.get('catdir') == "virtual" and \
+					missing_var in ("HOMEPAGE", "LICENSE"):
+					continue
+				if kwargs.get('live_ebuild') and missing_var == "KEYWORDS":
+					continue
+				myqakey = missingvars[pos] + ".missing"
+				self.qatracker.add_error(myqakey, '%s/%s.ebuild'
+					% (kwargs.get('xpkg'), kwargs.get('y_ebuild')))
+		return {'continue': False}
+
+	def old_virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
 		if ebuild.metadata.get("PROVIDE"):
 			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+		return {'continue': False}
 
+	def virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		if kwargs.get('catdir') == "virtual":
+			for var in ("HOMEPAGE", "LICENSE"):
+				if ebuild.metadata.get(var):
+					myqakey = var + ".virtual"
+					self.qatracker.add_error(myqakey, ebuild.relative_path)
 		return {'continue': False}
 
 	@property
@@ -39,4 +67,4 @@ class EbuildMetadata(object):
 
 	@property
 	def runInEbuilds(self):
-		return (True, [self.check])
+		return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 46f46f5..d42fd33 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -324,23 +324,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for pos, missing_var in enumerate(missingvars):
-				if not dynamic_data['ebuild'].metadata.get(missing_var):
-					if dynamic_data['catdir'] == "virtual" and \
-						missing_var in ("HOMEPAGE", "LICENSE"):
-						continue
-					if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS":
-						continue
-					myqakey = missingvars[pos] + ".missing"
-					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
-
-			if dynamic_data['catdir'] == "virtual":
-				for var in ("HOMEPAGE", "LICENSE"):
-					if dynamic_data['ebuild'].metadata.get(var):
-						myqakey = var + ".virtual"
-						self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['live_ebuild'] and self.repo_settings.repo_config.name == "gentoo":
 				self.liveeclasscheck.check(
 					dynamic_data['pkg'], xpkg, dynamic_data['ebuild'], y_ebuild, dynamic_data['ebuild'].keywords, self.repo_metadata['pmaskdict'])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     04cc3461557369a1ba864280925282771232c7fe
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:31:26 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 21 00:35:22 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=04cc3461

scanner.py: Migrate another metadata check to ebuild_metadata

 pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++
 pym/repoman/scanner.py                               | 2 --
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 143a40e..2dc1db2 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -28,6 +28,9 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		if ebuild.metadata.get("PROVIDE"):
+			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+
 		return {'continue': False}
 
 	@property

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index e6a17cd..46f46f5 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -324,8 +324,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
-				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 
 			for pos, missing_var in enumerate(missingvars):
 				if not dynamic_data['ebuild'].metadata.get(missing_var):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     20d723057fe75d128debdb50705d18a3c85449f0
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 10:35:49 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 21 00:35:22 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=20d72305

repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check

 pym/repoman/modules/scan/metadata/__init__.py      | 10 +++++-
 .../modules/scan/metadata/ebuild_metadata.py       | 39 ++++++++++++++++++++++
 pym/repoman/scanner.py                             | 27 +--------------
 3 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 7327ec0..eba6565 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -10,7 +10,7 @@ module_spec = {
 	'name': 'metadata',
 	'description': doc,
 	'provides':{
-		'metadata-module': {
+		'pkg-metadata': {
 			'name': "pkgmetadata",
 			'class': "PkgMetadata",
 			'description': doc,
@@ -18,6 +18,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'ebuild-metadata': {
+			'name': "ebuild_metadata",
+			'class': "EbuildMetadata",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
new file mode 100644
index 0000000..143a40e
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -0,0 +1,39 @@
+# -*- coding:utf-8 -*-
+
+'''Ebuild Metadata Checks'''
+
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+class EbuildMetadata(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for k, v in ebuild.metadata.items():
+			if not isinstance(v, basestring):
+				continue
+			m = NON_ASCII_RE.search(v)
+			if m is not None:
+				self.qatracker.add_error(
+					"variable.invalidchar",
+					"%s: %s variable contains non-ASCII "
+					"character at position %s" %
+					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index a8aa2f3..6f3fb53 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals
 import copy
 import io
 import logging
-import re
-import sys
 from itertools import chain
 from pprint import pformat
 
@@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
 MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
 
 
-
-if sys.hexversion >= 0x3000000:
-	basestring = str
-
-NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
-
-
 def sort_key(item):
 	return item[2].sub_path
 
 
-
 class Scanner(object):
 	'''Primary scan class.  Operates all the small Q/A tests and checks'''
 
@@ -311,7 +301,7 @@ class Scanner(object):
 			# initialize per ebuild plugin checks here
 			# need to set it up for ==> self.modules_list or some other ordered list
 			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
-				('eapi', 'EAPIChecks')]:
+				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]:
 				if mod[0]:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])
 					logging.debug("Initializing class name: %s", mod_class.__name__)
@@ -338,21 +328,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for k, v in dynamic_data['ebuild'].metadata.items():
-				if not isinstance(v, basestring):
-					continue
-				m = NON_ASCII_RE.search(v)
-				if m is not None:
-					self.qatracker.add_error(
-						"variable.invalidchar",
-						"%s: %s variable contains non-ASCII "
-						"character at position %s" %
-						(dynamic_data['ebuild'].relative_path, k, m.start() + 1))
-
-			if not dynamic_data['src_uri_error']:
-				self.thirdparty.check(dynamic_data['ebuild'].metadata, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
 				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     e0d88104006627f24165ac88ef121025dc844952
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 10:35:49 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Jan 18 19:20:03 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e0d88104

repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check

 pym/repoman/modules/scan/metadata/__init__.py      | 10 +++++-
 .../modules/scan/metadata/ebuild_metadata.py       | 39 ++++++++++++++++++++++
 pym/repoman/scanner.py                             | 27 +--------------
 3 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 7327ec0..eba6565 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -10,7 +10,7 @@ module_spec = {
 	'name': 'metadata',
 	'description': doc,
 	'provides':{
-		'metadata-module': {
+		'pkg-metadata': {
 			'name': "pkgmetadata",
 			'class': "PkgMetadata",
 			'description': doc,
@@ -18,6 +18,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'ebuild-metadata': {
+			'name': "ebuild_metadata",
+			'class': "EbuildMetadata",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
new file mode 100644
index 0000000..143a40e
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -0,0 +1,39 @@
+# -*- coding:utf-8 -*-
+
+'''Ebuild Metadata Checks'''
+
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+class EbuildMetadata(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for k, v in ebuild.metadata.items():
+			if not isinstance(v, basestring):
+				continue
+			m = NON_ASCII_RE.search(v)
+			if m is not None:
+				self.qatracker.add_error(
+					"variable.invalidchar",
+					"%s: %s variable contains non-ASCII "
+					"character at position %s" %
+					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index a8aa2f3..6f3fb53 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals
 import copy
 import io
 import logging
-import re
-import sys
 from itertools import chain
 from pprint import pformat
 
@@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
 MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
 
 
-
-if sys.hexversion >= 0x3000000:
-	basestring = str
-
-NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
-
-
 def sort_key(item):
 	return item[2].sub_path
 
 
-
 class Scanner(object):
 	'''Primary scan class.  Operates all the small Q/A tests and checks'''
 
@@ -311,7 +301,7 @@ class Scanner(object):
 			# initialize per ebuild plugin checks here
 			# need to set it up for ==> self.modules_list or some other ordered list
 			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
-				('eapi', 'EAPIChecks')]:
+				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]:
 				if mod[0]:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])
 					logging.debug("Initializing class name: %s", mod_class.__name__)
@@ -338,21 +328,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for k, v in dynamic_data['ebuild'].metadata.items():
-				if not isinstance(v, basestring):
-					continue
-				m = NON_ASCII_RE.search(v)
-				if m is not None:
-					self.qatracker.add_error(
-						"variable.invalidchar",
-						"%s: %s variable contains non-ASCII "
-						"character at position %s" %
-						(dynamic_data['ebuild'].relative_path, k, m.start() + 1))
-
-			if not dynamic_data['src_uri_error']:
-				self.thirdparty.check(dynamic_data['ebuild'].metadata, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
 				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     63d8ca7ebc4c93b16640ce3c7ca65032520457f1
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 10:35:49 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Jan 11 08:00:15 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=63d8ca7e

repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check

 pym/repoman/modules/scan/metadata/__init__.py      | 10 +++++-
 .../modules/scan/metadata/ebuild_metadata.py       | 39 ++++++++++++++++++++++
 pym/repoman/scanner.py                             | 27 +--------------
 3 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 7327ec0..eba6565 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -10,7 +10,7 @@ module_spec = {
 	'name': 'metadata',
 	'description': doc,
 	'provides':{
-		'metadata-module': {
+		'pkg-metadata': {
 			'name': "pkgmetadata",
 			'class': "PkgMetadata",
 			'description': doc,
@@ -18,6 +18,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'ebuild-metadata': {
+			'name': "ebuild_metadata",
+			'class': "EbuildMetadata",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
new file mode 100644
index 0000000..143a40e
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -0,0 +1,39 @@
+# -*- coding:utf-8 -*-
+
+'''Ebuild Metadata Checks'''
+
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+class EbuildMetadata(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for k, v in ebuild.metadata.items():
+			if not isinstance(v, basestring):
+				continue
+			m = NON_ASCII_RE.search(v)
+			if m is not None:
+				self.qatracker.add_error(
+					"variable.invalidchar",
+					"%s: %s variable contains non-ASCII "
+					"character at position %s" %
+					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index a8aa2f3..6f3fb53 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals
 import copy
 import io
 import logging
-import re
-import sys
 from itertools import chain
 from pprint import pformat
 
@@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
 MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
 
 
-
-if sys.hexversion >= 0x3000000:
-	basestring = str
-
-NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
-
-
 def sort_key(item):
 	return item[2].sub_path
 
 
-
 class Scanner(object):
 	'''Primary scan class.  Operates all the small Q/A tests and checks'''
 
@@ -311,7 +301,7 @@ class Scanner(object):
 			# initialize per ebuild plugin checks here
 			# need to set it up for ==> self.modules_list or some other ordered list
 			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
-				('eapi', 'EAPIChecks')]:
+				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]:
 				if mod[0]:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])
 					logging.debug("Initializing class name: %s", mod_class.__name__)
@@ -338,21 +328,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for k, v in dynamic_data['ebuild'].metadata.items():
-				if not isinstance(v, basestring):
-					continue
-				m = NON_ASCII_RE.search(v)
-				if m is not None:
-					self.qatracker.add_error(
-						"variable.invalidchar",
-						"%s: %s variable contains non-ASCII "
-						"character at position %s" %
-						(dynamic_data['ebuild'].relative_path, k, m.start() + 1))
-
-			if not dynamic_data['src_uri_error']:
-				self.thirdparty.check(dynamic_data['ebuild'].metadata, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
 				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     91883ecc397925167ce18cd6f0baed863b775837
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 22:59:38 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=91883ecc

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml

 pym/repoman/modules/scan/metadata/__init__.py |  8 ++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 6ab44f6..ed4a967 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -50,6 +50,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 89eaa57..50dd259 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -269,7 +269,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -317,11 +316,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     b273922a63f9b18b86d3dcdcaac8492c1e60ac69
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:31:26 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 22:59:35 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b273922a

scanner.py: Migrate another metadata check to ebuild_metadata

 pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++
 pym/repoman/scanner.py                               | 2 --
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 143a40e..2dc1db2 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -28,6 +28,9 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		if ebuild.metadata.get("PROVIDE"):
+			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+
 		return {'continue': False}
 
 	@property

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index e6a17cd..46f46f5 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -324,8 +324,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
-				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 
 			for pos, missing_var in enumerate(missingvars):
 				if not dynamic_data['ebuild'].metadata.get(missing_var):

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     891dd1e6bda1a5368562629c2864ec043ee9ec76
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 10:35:49 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 22:59:32 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=891dd1e6

repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check

 pym/repoman/modules/scan/metadata/__init__.py      | 10 +++++-
 .../modules/scan/metadata/ebuild_metadata.py       | 39 ++++++++++++++++++++++
 pym/repoman/scanner.py                             | 27 +--------------
 3 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 7327ec0..eba6565 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -10,7 +10,7 @@ module_spec = {
 	'name': 'metadata',
 	'description': doc,
 	'provides':{
-		'metadata-module': {
+		'pkg-metadata': {
 			'name': "pkgmetadata",
 			'class': "PkgMetadata",
 			'description': doc,
@@ -18,6 +18,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'ebuild-metadata': {
+			'name': "ebuild_metadata",
+			'class': "EbuildMetadata",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
new file mode 100644
index 0000000..143a40e
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -0,0 +1,39 @@
+# -*- coding:utf-8 -*-
+
+'''Ebuild Metadata Checks'''
+
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+class EbuildMetadata(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for k, v in ebuild.metadata.items():
+			if not isinstance(v, basestring):
+				continue
+			m = NON_ASCII_RE.search(v)
+			if m is not None:
+				self.qatracker.add_error(
+					"variable.invalidchar",
+					"%s: %s variable contains non-ASCII "
+					"character at position %s" %
+					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index a8aa2f3..6f3fb53 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals
 import copy
 import io
 import logging
-import re
-import sys
 from itertools import chain
 from pprint import pformat
 
@@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
 MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
 
 
-
-if sys.hexversion >= 0x3000000:
-	basestring = str
-
-NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
-
-
 def sort_key(item):
 	return item[2].sub_path
 
 
-
 class Scanner(object):
 	'''Primary scan class.  Operates all the small Q/A tests and checks'''
 
@@ -311,7 +301,7 @@ class Scanner(object):
 			# initialize per ebuild plugin checks here
 			# need to set it up for ==> self.modules_list or some other ordered list
 			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
-				('eapi', 'EAPIChecks')]:
+				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]:
 				if mod[0]:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])
 					logging.debug("Initializing class name: %s", mod_class.__name__)
@@ -338,21 +328,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for k, v in dynamic_data['ebuild'].metadata.items():
-				if not isinstance(v, basestring):
-					continue
-				m = NON_ASCII_RE.search(v)
-				if m is not None:
-					self.qatracker.add_error(
-						"variable.invalidchar",
-						"%s: %s variable contains non-ASCII "
-						"character at position %s" %
-						(dynamic_data['ebuild'].relative_path, k, m.start() + 1))
-
-			if not dynamic_data['src_uri_error']:
-				self.thirdparty.check(dynamic_data['ebuild'].metadata, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
 				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     ae123d5af1e2c33e22e1d501b252ea9954b384dc
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:56:25 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 20:15:08 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=ae123d5a

repoman: Migrate more metadata checks to ebuild_metadata.py

 .../modules/scan/metadata/ebuild_metadata.py       | 32 ++++++++++++++++++++--
 pym/repoman/scanner.py                             | 17 ------------
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 2dc1db2..77c947e 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+from repoman.qa_data import missingvars
+
 if sys.hexversion >= 0x3000000:
 	basestring = str
 
@@ -16,7 +18,7 @@ class EbuildMetadata(object):
 	def __init__(self, **kwargs):
 		self.qatracker = kwargs.get('qatracker')
 
-	def check(self, **kwargs):
+	def invalidchar(self, **kwargs):
 		ebuild = kwargs.get('ebuild')
 		for k, v in ebuild.metadata.items():
 			if not isinstance(v, basestring):
@@ -28,9 +30,35 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	def missing(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for pos, missing_var in enumerate(missingvars):
+			if not ebuild.metadata.get(missing_var):
+				if kwargs.get('catdir') == "virtual" and \
+					missing_var in ("HOMEPAGE", "LICENSE"):
+					continue
+				if kwargs.get('live_ebuild') and missing_var == "KEYWORDS":
+					continue
+				myqakey = missingvars[pos] + ".missing"
+				self.qatracker.add_error(myqakey, '%s/%s.ebuild'
+					% (kwargs.get('xpkg'), kwargs.get('y_ebuild')))
+		return {'continue': False}
+
+	def old_virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
 		if ebuild.metadata.get("PROVIDE"):
 			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+		return {'continue': False}
 
+	def virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		if kwargs.get('catdir') == "virtual":
+			for var in ("HOMEPAGE", "LICENSE"):
+				if ebuild.metadata.get(var):
+					myqakey = var + ".virtual"
+					self.qatracker.add_error(myqakey, ebuild.relative_path)
 		return {'continue': False}
 
 	@property
@@ -39,4 +67,4 @@ class EbuildMetadata(object):
 
 	@property
 	def runInEbuilds(self):
-		return (True, [self.check])
+		return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 46f46f5..d42fd33 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -324,23 +324,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for pos, missing_var in enumerate(missingvars):
-				if not dynamic_data['ebuild'].metadata.get(missing_var):
-					if dynamic_data['catdir'] == "virtual" and \
-						missing_var in ("HOMEPAGE", "LICENSE"):
-						continue
-					if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS":
-						continue
-					myqakey = missingvars[pos] + ".missing"
-					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
-
-			if dynamic_data['catdir'] == "virtual":
-				for var in ("HOMEPAGE", "LICENSE"):
-					if dynamic_data['ebuild'].metadata.get(var):
-						myqakey = var + ".virtual"
-						self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['live_ebuild'] and self.repo_settings.repo_config.name == "gentoo":
 				self.liveeclasscheck.check(
 					dynamic_data['pkg'], xpkg, dynamic_data['ebuild'], y_ebuild, dynamic_data['ebuild'].keywords, self.repo_metadata['pmaskdict'])

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     5e273483758272413387c1dadd403a8f1bffdc4a
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 03:23:53 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5e273483

repoman: Create a metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml

 pym/repoman/modules/scan/metadata/__init__.py |  8 ++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++++
 pym/repoman/scanner.py                        | 36 ++++++++++++++++++++-------
 3 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 6ab44f6..ed4a967 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -50,6 +50,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 89eaa57..50dd259 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -269,7 +269,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -317,11 +316,30 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata.get('continue', False):
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     f2e20a96e9f1d9073bb6c8722170667ba96915a0
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 10:35:49 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 03:23:49 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f2e20a96

repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check

 pym/repoman/modules/scan/metadata/__init__.py      | 10 +++++-
 .../modules/scan/metadata/ebuild_metadata.py       | 39 ++++++++++++++++++++++
 pym/repoman/scanner.py                             | 27 +--------------
 3 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 7327ec0..eba6565 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -10,7 +10,7 @@ module_spec = {
 	'name': 'metadata',
 	'description': doc,
 	'provides':{
-		'metadata-module': {
+		'pkg-metadata': {
 			'name': "pkgmetadata",
 			'class': "PkgMetadata",
 			'description': doc,
@@ -18,6 +18,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'ebuild-metadata': {
+			'name': "ebuild_metadata",
+			'class': "EbuildMetadata",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
new file mode 100644
index 0000000..143a40e
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -0,0 +1,39 @@
+# -*- coding:utf-8 -*-
+
+'''Ebuild Metadata Checks'''
+
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+class EbuildMetadata(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for k, v in ebuild.metadata.items():
+			if not isinstance(v, basestring):
+				continue
+			m = NON_ASCII_RE.search(v)
+			if m is not None:
+				self.qatracker.add_error(
+					"variable.invalidchar",
+					"%s: %s variable contains non-ASCII "
+					"character at position %s" %
+					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index a8aa2f3..6f3fb53 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals
 import copy
 import io
 import logging
-import re
-import sys
 from itertools import chain
 from pprint import pformat
 
@@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
 MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
 
 
-
-if sys.hexversion >= 0x3000000:
-	basestring = str
-
-NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
-
-
 def sort_key(item):
 	return item[2].sub_path
 
 
-
 class Scanner(object):
 	'''Primary scan class.  Operates all the small Q/A tests and checks'''
 
@@ -311,7 +301,7 @@ class Scanner(object):
 			# initialize per ebuild plugin checks here
 			# need to set it up for ==> self.modules_list or some other ordered list
 			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
-				('eapi', 'EAPIChecks')]:
+				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]:
 				if mod[0]:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])
 					logging.debug("Initializing class name: %s", mod_class.__name__)
@@ -338,21 +328,6 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-
-			for k, v in dynamic_data['ebuild'].metadata.items():
-				if not isinstance(v, basestring):
-					continue
-				m = NON_ASCII_RE.search(v)
-				if m is not None:
-					self.qatracker.add_error(
-						"variable.invalidchar",
-						"%s: %s variable contains non-ASCII "
-						"character at position %s" %
-						(dynamic_data['ebuild'].relative_path, k, m.start() + 1))
-
-			if not dynamic_data['src_uri_error']:
-				self.thirdparty.check(dynamic_data['ebuild'].metadata, dynamic_data['ebuild'].relative_path)
-
 			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
 				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
 

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     0acb069f91d0e400f2182c3965462850f7d02da0
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 08:37:22 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Jan  6 04:08:25 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0acb069f

repoman: Create an metadata UnusedCheck and final pkg checks

Create a plugin loop for any final pkg checks.
Create the one plugin for the unused use-descriptions in mteadata.xml

 pym/repoman/modules/scan/metadata/__init__.py |  8 ++++++
 pym/repoman/modules/scan/metadata/unused.py   | 32 ++++++++++++++++++++++
 pym/repoman/scanner.py                        | 39 ++++++++++++++++++++-------
 3 files changed, 69 insertions(+), 10 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 6ab44f6..ed4a967 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -50,6 +50,14 @@ module_spec = {
 			'func_desc': {
 			},
 		},
+		'unused-metadata': {
+			'name': "unused",
+			'class': "UnusedCheck",
+			'description': doc,
+			'functions': ['check'],
+			'func_desc': {
+			},
+		},
 	}
 }
 

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
new file mode 100644
index 0000000..5eb6716
--- /dev/null
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -0,0 +1,32 @@
+
+
+class UnusedCheck(object):
+
+	def __init__(self, **kwargs):
+		self.qatracker = kwargs.get('qatracker')
+
+	def check(self, **kwargs):
+		xpkg = kwargs.get('xpkg')
+		muselist = kwargs.get('muselist')
+		used_useflags = kwargs.get('used_useflags')
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if kwargs.get('allvalid'):
+			for myflag in muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))
+		return {'continue': False}
+
+	@property
+	def runInPkgs(self):
+		return (False, [])
+
+	@property
+	def runInEbuilds(self):
+		return (False, [])
+
+	@property
+	def runInFinal(self):
+		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 08e53ac..cfc6802 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -271,7 +271,6 @@ class Scanner(object):
 
 
 	def _scan_ebuilds(self, ebuildlist, dynamic_data):
-		xpkg = dynamic_data['xpkg']
 		# detect unused local USE-descriptions
 		dynamic_data['used_useflags'] = set()
 
@@ -320,13 +319,33 @@ class Scanner(object):
 			if y_ebuild_continue:
 				continue
 
-			print("**** finished plugin loop, continuing...")
+			print("**** finished ebuild plugin loop, continuing...")
+
+		# Final checks
+		# initialize per pkg plugin final checks here
+		# need to set it up for ==> self.modules_list or some other ordered list
+		xpkg_complete = False
+		for mod in [('unused', 'UnusedChecks')]:
+			if mod[0]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				print("Initializing class name:", mod_class.__name__)
+				self.modules[mod[1]] = mod_class(**self.kwargs)
+			print("scan_ebuilds final checks: module:", mod[1])
+			do_it, functions = self.modules[mod[1]].runInFinal
+			# print("do_it", do_it, "functions", functions)
+			if do_it:
+				for func in functions:
+					print("\tRunning function:", func)
+					rdata = func(**dynamic_data)
+					if rdata['continue']:
+						xpkg_complete = True
+						print("\t>>> Continuing")
+						break
+					#print("rdata:", rdata)
+					dynamic_data.update(rdata)
+					#print("dynamic_data", dynamic_data)
+
+		if xpkg_complete:
+			return
+		return
 
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if dynamic_data['allvalid']:
-			for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']):
-				self.qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))

[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/

[Brian Dolbec]

commit:     aebf5722a1bc262b8bd843348c947d06efa6713c
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  3 11:56:25 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Jan  6 04:08:21 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=aebf5722

repoman: Migrate more metadata checks to ebuild_metadata.py

 .../modules/scan/metadata/ebuild_metadata.py       | 32 ++++++++++++++++++++--
 pym/repoman/scanner.py                             | 19 -------------
 2 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
index 2dc1db2..77c947e 100644
--- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py
+++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+from repoman.qa_data import missingvars
+
 if sys.hexversion >= 0x3000000:
 	basestring = str
 
@@ -16,7 +18,7 @@ class EbuildMetadata(object):
 	def __init__(self, **kwargs):
 		self.qatracker = kwargs.get('qatracker')
 
-	def check(self, **kwargs):
+	def invalidchar(self, **kwargs):
 		ebuild = kwargs.get('ebuild')
 		for k, v in ebuild.metadata.items():
 			if not isinstance(v, basestring):
@@ -28,9 +30,35 @@ class EbuildMetadata(object):
 					"%s: %s variable contains non-ASCII "
 					"character at position %s" %
 					(ebuild.relative_path, k, m.start() + 1))
+		return {'continue': False}
+
+	def missing(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		for pos, missing_var in enumerate(missingvars):
+			if not ebuild.metadata.get(missing_var):
+				if kwargs.get('catdir') == "virtual" and \
+					missing_var in ("HOMEPAGE", "LICENSE"):
+					continue
+				if kwargs.get('live_ebuild') and missing_var == "KEYWORDS":
+					continue
+				myqakey = missingvars[pos] + ".missing"
+				self.qatracker.add_error(myqakey, '%s/%s.ebuild'
+					% (kwargs.get('xpkg'), kwargs.get('y_ebuild')))
+		return {'continue': False}
+
+	def old_virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
 		if ebuild.metadata.get("PROVIDE"):
 			self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+		return {'continue': False}
 
+	def virtual(self, **kwargs):
+		ebuild = kwargs.get('ebuild')
+		if kwargs.get('catdir') == "virtual":
+			for var in ("HOMEPAGE", "LICENSE"):
+				if ebuild.metadata.get(var):
+					myqakey = var + ".virtual"
+					self.qatracker.add_error(myqakey, ebuild.relative_path)
 		return {'continue': False}
 
 	@property
@@ -39,4 +67,4 @@ class EbuildMetadata(object):
 
 	@property
 	def runInEbuilds(self):
-		return (True, [self.check])
+		return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index ca540a7..20c6460 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -330,25 +330,6 @@ class Scanner(object):
 
 			print("**** finished plugin loop, continuing...")
 
-			if dynamic_data['ebuild'].metadata.get("PROVIDE"):
-				self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path)
-
-			for pos, missing_var in enumerate(missingvars):
-				if not dynamic_data['ebuild'].metadata.get(missing_var):
-					if dynamic_data['catdir'] == "virtual" and \
-						missing_var in ("HOMEPAGE", "LICENSE"):
-						continue
-					if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS":
-						continue
-					myqakey = missingvars[pos] + ".missing"
-					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
-
-			if dynamic_data['catdir'] == "virtual":
-				for var in ("HOMEPAGE", "LICENSE"):
-					if dynamic_data['ebuild'].metadata.get(var):
-						myqakey = var + ".virtual"
-						self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path)
-
 			self.descriptioncheck.check(dynamic_data['pkg'], dynamic_data['ebuild'])