[gentoo-commits] proj/portage:master commit in: pym/repoman/

["Brian Dolbec" <dolsen@gentoo.org>]

commit:     9f63c395ee23b00d77d00e667a28624de5baff49
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Sep 17 15:29:11 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Sep 21 23:42:46 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9f63c395

repoman: Move the primary checks loop to it's own class and file

Only minimal changes were done for this initial move.
The _scan_ebuilds() needs major hacking up into manageable chunks.
Clean out code separation demarcation lines
These lines were originally used to mark places where code was removed.
And replaced with a class instance and/or function call.

Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/repoman/main.py    | 756 ++-----------------------------------------------
 pym/repoman/scanner.py | 715 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 743 insertions(+), 728 deletions(-)

diff --git a/pym/repoman/main.py b/pym/repoman/main.py
index e3d0472..2b2f91d 100755
--- a/pym/repoman/main.py
+++ b/pym/repoman/main.py
@@ -4,7 +4,6 @@
 
 from __future__ import print_function, unicode_literals
 
-import copy
 import errno
 import io
 import logging
@@ -15,7 +14,6 @@ import sys
 import tempfile
 import platform
 from itertools import chain
-from pprint import pformat
 
 from os import path as osp
 if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")):
@@ -30,14 +28,12 @@ portage._disable_legacy_globals()
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
-from _emerge.Package import Package
 from _emerge.UserQuery import UserQuery
 import portage.checksum
 import portage.const
 import portage.repository.config
-from portage import cvstree, normalize_path
+from portage import cvstree
 from portage import util
-from portage.dep import Atom
 from portage.process import find_binary, spawn
 from portage.output import (
 	bold, create_color_func, green, nocolor, red)
@@ -47,40 +43,18 @@ from portage.util import writemsg_level
 from portage.package.ebuild.digestgen import digestgen
 
 from repoman.argparser import parse_args
-from repoman.checks.directories.files import FileChecks
-from repoman.checks.ebuilds.checks import run_checks, checks_init
-from repoman.checks.ebuilds.eclasses.live import LiveEclassChecks
-from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks
-from repoman.checks.ebuilds.fetches import FetchChecks
-from repoman.checks.ebuilds.keywords import KeywordChecks
-from repoman.checks.ebuilds.isebuild import IsEbuild
-from repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors
-from repoman.checks.ebuilds.manifests import Manifests
-from repoman.check_missingslot import check_missingslot
-from repoman.checks.ebuilds.misc import bad_split_check, pkg_invalid
-from repoman.checks.ebuilds.pkgmetadata import PkgMetadata
-from repoman.checks.ebuilds.use_flags import USEFlagChecks
-from repoman.checks.ebuilds.variables.description import DescriptionChecks
-from repoman.checks.ebuilds.variables.eapi import EAPIChecks
-from repoman.checks.ebuilds.variables.license import LicenseChecks
-from repoman.checks.ebuilds.variables.restrict import RestrictChecks
-from repoman.ebuild import Ebuild
+from repoman.checks.ebuilds.checks import checks_init
 from repoman.errors import err
 from repoman.gpg import gpgsign, need_signature
-from repoman.modules.commit import repochecks
-from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
 from repoman.qa_data import (
 	format_qa_output, format_qa_output_column, qahelp,
-	qawarnings, qacats, missingvars,
-	suspect_virtual, suspect_rdepend)
-from repoman.qa_tracker import QATracker
-from repoman.repos import RepoSettings, repo_metadata
-from repoman.scan import Changes, scan
+	qawarnings, qacats)
+from repoman.repos import RepoSettings
+from repoman.scanner import Scanner
 from repoman._subprocess import repoman_popen, repoman_getstatusoutput
 from repoman import utilities
 from repoman.vcs.vcs import (
 	git_supports_gpg_sign, vcs_files_to_cps, VCSSettings)
-from repoman.vcs.vcsstatus import VCSStatus
 
 
 if sys.hexversion >= 0x3000000:
@@ -90,21 +64,11 @@ util.initialize_logger()
 
 bad = create_color_func("BAD")
 
-live_eclasses = portage.const.LIVE_ECLASSES
-non_ascii_re = re.compile(r'[^\x00-\x7f]')
-
 # A sane umask is needed for files that portage creates.
 os.umask(0o22)
 
-def sort_key(item):
-	return item[2].sub_path
-
 
 def repoman_main(argv):
-	# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
-	# behave incrementally.
-	repoman_incrementals = tuple(
-		x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 	config_root = os.environ.get("PORTAGE_CONFIGROOT")
 	repoman_settings = portage.config(config_root=config_root, local_config=False)
 
@@ -142,30 +106,9 @@ def repoman_main(argv):
 	repo_settings = RepoSettings(
 		config_root, portdir, portdir_overlay,
 		repoman_settings, vcs_settings, options, qawarnings)
-
 	repoman_settings = repo_settings.repoman_settings
-
 	portdb = repo_settings.portdb
 
-	if options.echangelog is None and repo_settings.repo_config.update_changelog:
-		options.echangelog = 'y'
-
-	if vcs_settings.vcs is None:
-		options.echangelog = 'n'
-
-	# The --echangelog option causes automatic ChangeLog generation,
-	# which invalidates changelog.ebuildadded and changelog.missing
-	# checks.
-	# Note: Some don't use ChangeLogs in distributed SCMs.
-	# It will be generated on server side from scm log,
-	# before package moves to the rsync server.
-	# This is needed because they try to avoid merge collisions.
-	# Gentoo's Council decided to always use the ChangeLog file.
-	# TODO: shouldn't this just be switched on the repo, iso the VCS?
-	is_echangelog_enabled = options.echangelog in ('y', 'force')
-	vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
-	check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn
-
 	if 'digest' in repoman_settings.features and options.digest != 'n':
 		options.digest = 'y'
 
@@ -178,663 +121,16 @@ def repoman_main(argv):
 	env = os.environ.copy()
 	env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 
-	categories = []
-	for path in repo_settings.repo_config.eclass_db.porttrees:
-		categories.extend(portage.util.grabfile(
-			os.path.join(path, 'profiles', 'categories')))
-	repoman_settings.categories = frozenset(
-		portage.util.stack_lists([categories], incremental=1))
-	categories = repoman_settings.categories
-
-	portdb.settings = repoman_settings
-	# We really only need to cache the metadata that's necessary for visibility
-	# filtering. Anything else can be discarded to reduce memory consumption.
-	portdb._aux_cache_keys.clear()
-	portdb._aux_cache_keys.update(
-		["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
-
-	reposplit = myreporoot.split(os.path.sep)
-	repolevel = len(reposplit)
-
-	###################
-	commitmessage = None
-	if options.mode == 'commit':
-		repochecks.commit_check(repolevel, reposplit)
-		repochecks.conflict_check(vcs_settings, options)
-
-	###################
-
-	# Make startdir relative to the canonical repodir, so that we can pass
-	# it to digestgen and it won't have to be canonicalized again.
-	if repolevel == 1:
-		startdir = repo_settings.repodir
-	else:
-		startdir = normalize_path(mydir)
-		startdir = os.path.join(
-			repo_settings.repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
-	###################
-
-
-	# get lists of valid keywords, licenses, and use
-	new_data = repo_metadata(repo_settings.portdb, repoman_settings)
-	kwlist, liclist, uselist, profile_list, \
-		global_pmaskdict, liclist_deprecated = new_data
-
-	repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
-	repoman_settings.backup_changes('PORTAGE_ARCHLIST')
-
-	####################
-
-	profiles = setup_profile(profile_list)
-
-	####################
-
-	check_profiles(profiles, repoman_settings.archlist())
-
-	####################
-
-	scanlist = scan(repolevel, reposplit, startdir, categories, repo_settings)
-
-	####################
-
-	dev_keywords = dev_profile_keywords(profiles)
-
-	qatracker = QATracker()
-
-
-	if options.mode == "manifest":
-		pass
-	elif options.pretend:
-		print(green("\nRepoMan does a once-over of the neighborhood..."))
-	else:
-		print(green("\nRepoMan scours the neighborhood..."))
-
-	#####################
-
-	changed = Changes(options)
-	changed.scan(vcs_settings)
-
-	######################
-
-	have_pmasked = False
-	have_dev_keywords = False
-	dofail = 0
-
-	# NOTE: match-all caches are not shared due to potential
-	# differences between profiles in _get_implicit_iuse.
-	arch_caches = {}
-	arch_xmatch_caches = {}
-	shared_xmatch_caches = {"cp-list": {}}
-
-	include_arches = None
-	if options.include_arches:
-		include_arches = set()
-		include_arches.update(*[x.split() for x in options.include_arches])
-
-	# Disable the "ebuild.notadded" check when not in commit mode and
-	# running `svn status` in every package dir will be too expensive.
-
-	check_ebuild_notadded = not \
-		(vcs_settings.vcs == "svn" and repolevel < 3 and options.mode != "commit")
-
-	effective_scanlist = scanlist
-	if options.if_modified == "y":
-		effective_scanlist = sorted(vcs_files_to_cps(
-			chain(changed.changed, changed.new, changed.removed),
-			repolevel, reposplit, categories))
-
-	######################
-	# initialize our checks classes here before the big xpkg loop
-	manifester = Manifests(options, qatracker, repoman_settings)
-	is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker)
-	filescheck = FileChecks(
-		qatracker, repoman_settings, repo_settings, portdb, vcs_settings)
-	status_check = VCSStatus(vcs_settings, qatracker)
-	fetchcheck = FetchChecks(
-		qatracker, repoman_settings, repo_settings, portdb, vcs_settings)
-	pkgmeta = PkgMetadata(options, qatracker, repoman_settings)
-	thirdparty = ThirdPartyMirrors(repoman_settings, qatracker)
-	use_flag_checks = USEFlagChecks(qatracker, uselist)
-	keywordcheck = KeywordChecks(qatracker, options)
-	liveeclasscheck = LiveEclassChecks(qatracker)
-	rubyeclasscheck = RubyEclassChecks(qatracker)
-	eapicheck = EAPIChecks(qatracker, repo_settings)
-	descriptioncheck = DescriptionChecks(qatracker)
-	licensecheck = LicenseChecks(qatracker, liclist, liclist_deprecated)
-	restrictcheck = RestrictChecks(qatracker)
-	######################
-
-	for xpkg in effective_scanlist:
-		# ebuilds and digests added to cvs respectively.
-		logging.info("checking package %s" % xpkg)
-		# save memory by discarding xmatch caches from previous package(s)
-		arch_xmatch_caches.clear()
-		eadded = []
-		catdir, pkgdir = xpkg.split("/")
-		checkdir = repo_settings.repodir + "/" + xpkg
-		checkdir_relative = ""
-		if repolevel < 3:
-			checkdir_relative = os.path.join(pkgdir, checkdir_relative)
-		if repolevel < 2:
-			checkdir_relative = os.path.join(catdir, checkdir_relative)
-		checkdir_relative = os.path.join(".", checkdir_relative)
-
-	#####################
-		if manifester.run(checkdir, portdb):
-			continue
-		if not manifester.generated_manifest:
-			manifester.digest_check(xpkg, checkdir)
-	######################
-
-		if options.mode == 'manifest-check':
-			continue
-
-		checkdirlist = os.listdir(checkdir)
-
-	######################
-		pkgs, allvalid = is_ebuild.check(checkdirlist, checkdir, xpkg)
-		if is_ebuild.continue_:
-			# If we can't access all the metadata then it's totally unsafe to
-			# commit since there's no way to generate a correct Manifest.
-			# Do not try to do any more QA checks on this package since missing
-			# metadata leads to false positives for several checks, and false
-			# positives confuse users.
-			can_force = False
-			continue
-	######################
-
-		keywordcheck.prepare()
-
-		# Sort ebuilds in ascending order for the KEYWORDS.dropped check.
-		ebuildlist = sorted(pkgs.values())
-		ebuildlist = [pkg.pf for pkg in ebuildlist]
-	#######################
-		filescheck.check(
-			checkdir, checkdirlist, checkdir_relative, changed.changed, changed.new)
-	#######################
-		status_check.check(check_ebuild_notadded, checkdir, checkdir_relative, xpkg)
-		eadded.extend(status_check.eadded)
-
-	#################
-		fetchcheck.check(
-			xpkg, checkdir, checkdir_relative, changed.changed, changed.new)
-	#################
-
-		if check_changelog and "ChangeLog" not in checkdirlist:
-			qatracker.add_error("changelog.missing", xpkg + "/ChangeLog")
-	#################
-		pkgmeta.check(xpkg, checkdir, checkdirlist, repolevel)
-		muselist = frozenset(pkgmeta.musedict)
-	#################
-
-		changelog_path = os.path.join(checkdir_relative, "ChangeLog")
-		changelog_modified = changelog_path in changed.changelogs
-
-		# detect unused local USE-descriptions
-		used_useflags = set()
-
-		for y_ebuild in ebuildlist:
-			##################
-			ebuild = Ebuild(
-				repo_settings, repolevel, pkgdir, catdir, vcs_settings,
-				xpkg, y_ebuild)
-			##################
-
-			if check_changelog and not changelog_modified \
-				and ebuild.ebuild_path in changed.new_ebuilds:
-				qatracker.add_error('changelog.ebuildadded', ebuild.relative_path)
-
-			if ebuild.untracked(check_ebuild_notadded, y_ebuild, eadded):
-				# ebuild not added to vcs
-				qatracker.add_error(
-					"ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild")
-
-	##################
-			if bad_split_check(xpkg, y_ebuild, pkgdir, qatracker):
-				continue
-	###################
-			pkg = pkgs[y_ebuild]
-			if pkg_invalid(pkg, qatracker, ebuild):
-				allvalid = False
-				continue
-
-			myaux = pkg._metadata
-			eapi = myaux["EAPI"]
-			inherited = pkg.inherited
-			live_ebuild = live_eclasses.intersection(inherited)
-
-			#######################
-			eapicheck.check(pkg, ebuild)
-			#######################
-
-			for k, v in myaux.items():
-				if not isinstance(v, basestring):
-					continue
-				m = non_ascii_re.search(v)
-				if m is not None:
-					qatracker.add_error(
-						"variable.invalidchar",
-						"%s: %s variable contains non-ASCII "
-						"character at position %s" %
-						(ebuild.relative_path, k, m.start() + 1))
-
-			if not fetchcheck.src_uri_error:
-				#######################
-				thirdparty.check(myaux, ebuild.relative_path)
-				#######################
-			if myaux.get("PROVIDE"):
-				qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
-
-			for pos, missing_var in enumerate(missingvars):
-				if not myaux.get(missing_var):
-					if catdir == "virtual" and \
-						missing_var in ("HOMEPAGE", "LICENSE"):
-						continue
-					if live_ebuild and missing_var == "KEYWORDS":
-						continue
-					myqakey = missingvars[pos] + ".missing"
-					qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
-
-			if catdir == "virtual":
-				for var in ("HOMEPAGE", "LICENSE"):
-					if myaux.get(var):
-						myqakey = var + ".virtual"
-						qatracker.add_error(myqakey, ebuild.relative_path)
-
-			#######################
-			descriptioncheck.check(pkg, ebuild)
-			#######################
-
-			keywords = myaux["KEYWORDS"].split()
-
-			ebuild_archs = set(
-				kw.lstrip("~") for kw in keywords if not kw.startswith("-"))
-
-			#######################
-			keywordcheck.check(
-				pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, changed,
-				live_ebuild, kwlist, profiles)
-			#######################
-
-			if live_ebuild and repo_settings.repo_config.name == "gentoo":
-				#######################
-				liveeclasscheck.check(
-					pkg, xpkg, ebuild, y_ebuild, keywords, global_pmaskdict)
-				#######################
-
-			if options.ignore_arches:
-				arches = [[
-					repoman_settings["ARCH"], repoman_settings["ARCH"],
-					repoman_settings["ACCEPT_KEYWORDS"].split()]]
-			else:
-				arches = set()
-				for keyword in keywords:
-					if keyword[0] == "-":
-						continue
-					elif keyword[0] == "~":
-						arch = keyword[1:]
-						if arch == "*":
-							for expanded_arch in profiles:
-								if expanded_arch == "**":
-									continue
-								arches.add(
-									(keyword, expanded_arch, (
-										expanded_arch, "~" + expanded_arch)))
-						else:
-							arches.add((keyword, arch, (arch, keyword)))
-					else:
-						if keyword == "*":
-							for expanded_arch in profiles:
-								if expanded_arch == "**":
-									continue
-								arches.add(
-									(keyword, expanded_arch, (expanded_arch,)))
-						else:
-							arches.add((keyword, keyword, (keyword,)))
-				if not arches:
-					# Use an empty profile for checking dependencies of
-					# packages that have empty KEYWORDS.
-					arches.add(('**', '**', ('**',)))
-
-			unknown_pkgs = set()
-			baddepsyntax = False
-			badlicsyntax = False
-			badprovsyntax = False
-			# catpkg = catdir + "/" + y_ebuild
-
-			inherited_java_eclass = "java-pkg-2" in inherited or \
-				"java-pkg-opt-2" in inherited
-			inherited_wxwidgets_eclass = "wxwidgets" in inherited
-			# operator_tokens = set(["||", "(", ")"])
-			type_list, badsyntax = [], []
-			for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
-				mydepstr = myaux[mytype]
-
-				buildtime = mytype in Package._buildtime_keys
-				runtime = mytype in Package._runtime_keys
-				token_class = None
-				if mytype.endswith("DEPEND"):
-					token_class = portage.dep.Atom
+	# Perform the main checks
+	scanner = Scanner(repo_settings, myreporoot, config_root, options,
+					vcs_settings, mydir, env)
+	qatracker, can_force = scanner.scan_pkgs(can_force)
 
-				try:
-					atoms = portage.dep.use_reduce(
-						mydepstr, matchall=1, flat=True,
-						is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
-				except portage.exception.InvalidDependString as e:
-					atoms = None
-					badsyntax.append(str(e))
-
-				if atoms and mytype.endswith("DEPEND"):
-					if runtime and \
-						"test?" in mydepstr.split():
-						qatracker.add_error(
-							mytype + '.suspect',
-							"%s: 'test?' USE conditional in %s" %
-							(ebuild.relative_path, mytype))
-
-					for atom in atoms:
-						if atom == "||":
-							continue
-
-						is_blocker = atom.blocker
-
-						# Skip dependency.unknown for blockers, so that we
-						# don't encourage people to remove necessary blockers,
-						# as discussed in bug 382407. We use atom.without_use
-						# due to bug 525376.
-						if not is_blocker and \
-							not portdb.xmatch("match-all", atom.without_use) and \
-							not atom.cp.startswith("virtual/"):
-							unknown_pkgs.add((mytype, atom.unevaluated_atom))
-
-						if catdir != "virtual":
-							if not is_blocker and \
-								atom.cp in suspect_virtual:
-								qatracker.add_error(
-									'virtual.suspect', ebuild.relative_path +
-									": %s: consider using '%s' instead of '%s'" %
-									(mytype, suspect_virtual[atom.cp], atom))
-							if not is_blocker and \
-								atom.cp.startswith("perl-core/"):
-								qatracker.add_error('dependency.perlcore',
-									ebuild.relative_path +
-									": %s: please use '%s' instead of '%s'" %
-									(mytype,
-									atom.replace("perl-core/","virtual/perl-"),
-									atom))
-
-						if buildtime and \
-							not is_blocker and \
-							not inherited_java_eclass and \
-							atom.cp == "virtual/jdk":
-							qatracker.add_error(
-								'java.eclassesnotused', ebuild.relative_path)
-						elif buildtime and \
-							not is_blocker and \
-							not inherited_wxwidgets_eclass and \
-							atom.cp == "x11-libs/wxGTK":
-							qatracker.add_error(
-								'wxwidgets.eclassnotused',
-								"%s: %ss on x11-libs/wxGTK without inheriting"
-								" wxwidgets.eclass" % (ebuild.relative_path, mytype))
-						elif runtime:
-							if not is_blocker and \
-								atom.cp in suspect_rdepend:
-								qatracker.add_error(
-									mytype + '.suspect',
-									ebuild.relative_path + ": '%s'" % atom)
-
-						if atom.operator == "~" and \
-							portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
-							qacat = 'dependency.badtilde'
-							qatracker.add_error(
-								qacat, "%s: %s uses the ~ operator"
-								" with a non-zero revision: '%s'" %
-								(ebuild.relative_path, mytype, atom))
-
-						check_missingslot(atom, mytype, eapi, portdb, qatracker,
-							ebuild.relative_path, myaux)
-
-				type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
-
-			for m, b in zip(type_list, badsyntax):
-				if m.endswith("DEPEND"):
-					qacat = "dependency.syntax"
-				else:
-					qacat = m + ".syntax"
-				qatracker.add_error(
-					qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
-
-			badlicsyntax = len([z for z in type_list if z == "LICENSE"])
-			badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
-			baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
-			badlicsyntax = badlicsyntax > 0
-			badprovsyntax = badprovsyntax > 0
-
-			#################
-			use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, muselist)
-
-			ebuild_used_useflags = use_flag_checks.getUsedUseFlags()
-			used_useflags = used_useflags.union(ebuild_used_useflags)
-			#################
-			rubyeclasscheck.check(pkg, ebuild)
-			#################
-
-			# license checks
-			if not badlicsyntax:
-				#################
-				licensecheck.check(pkg, xpkg, ebuild, y_ebuild)
-				#################
-
-			#################
-			restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
-			#################
-
-			# Syntax Checks
-
-			if not vcs_settings.vcs_preserves_mtime:
-				if ebuild.ebuild_path not in changed.new_ebuilds and \
-					ebuild.ebuild_path not in changed.ebuilds:
-					pkg.mtime = None
-			try:
-				# All ebuilds should have utf_8 encoding.
-				f = io.open(
-					_unicode_encode(
-						ebuild.full_path, encoding=_encodings['fs'], errors='strict'),
-					mode='r', encoding=_encodings['repo.content'])
-				try:
-					for check_name, e in run_checks(f, pkg):
-						qatracker.add_error(
-							check_name, ebuild.relative_path + ': %s' % e)
-				finally:
-					f.close()
-			except UnicodeDecodeError:
-				# A file.UTF8 failure will have already been recorded above.
-				pass
-
-			if options.force:
-				# The dep_check() calls are the most expensive QA test. If --force
-				# is enabled, there's no point in wasting time on these since the
-				# user is intent on forcing the commit anyway.
-				continue
-
-			relevant_profiles = []
-			for keyword, arch, groups in arches:
-				if arch not in profiles:
-					# A missing profile will create an error further down
-					# during the KEYWORDS verification.
-					continue
-
-				if include_arches is not None:
-					if arch not in include_arches:
-						continue
-
-				relevant_profiles.extend(
-					(keyword, groups, prof) for prof in profiles[arch])
-
-			relevant_profiles.sort(key=sort_key)
-
-			for keyword, groups, prof in relevant_profiles:
-
-				is_stable_profile = prof.status == "stable"
-				is_dev_profile = prof.status == "dev" and \
-					options.include_dev
-				is_exp_profile = prof.status == "exp" and \
-					options.include_exp_profiles == 'y'
-				if not (is_stable_profile or is_dev_profile or is_exp_profile):
-					continue
+	commitmessage = None
 
-				dep_settings = arch_caches.get(prof.sub_path)
-				if dep_settings is None:
-					dep_settings = portage.config(
-						config_profile_path=prof.abs_path,
-						config_incrementals=repoman_incrementals,
-						config_root=config_root,
-						local_config=False,
-						_unmatched_removal=options.unmatched_removal,
-						env=env, repositories=repoman_settings.repositories)
-					dep_settings.categories = repoman_settings.categories
-					if options.without_mask:
-						dep_settings._mask_manager_obj = \
-							copy.deepcopy(dep_settings._mask_manager)
-						dep_settings._mask_manager._pmaskdict.clear()
-					arch_caches[prof.sub_path] = dep_settings
-
-				xmatch_cache_key = (prof.sub_path, tuple(groups))
-				xcache = arch_xmatch_caches.get(xmatch_cache_key)
-				if xcache is None:
-					portdb.melt()
-					portdb.freeze()
-					xcache = portdb.xcache
-					xcache.update(shared_xmatch_caches)
-					arch_xmatch_caches[xmatch_cache_key] = xcache
-
-				repo_settings.trees[repo_settings.root]["porttree"].settings = dep_settings
-				portdb.settings = dep_settings
-				portdb.xcache = xcache
-
-				dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
-				# just in case, prevent config.reset() from nuking these.
-				dep_settings.backup_changes("ACCEPT_KEYWORDS")
-
-				# This attribute is used in dbapi._match_use() to apply
-				# use.stable.{mask,force} settings based on the stable
-				# status of the parent package. This is required in order
-				# for USE deps of unstable packages to be resolved correctly,
-				# since otherwise use.stable.{mask,force} settings of
-				# dependencies may conflict (see bug #456342).
-				dep_settings._parent_stable = dep_settings._isStable(pkg)
-
-				# Handle package.use*.{force,mask) calculation, for use
-				# in dep_check.
-				dep_settings.useforce = dep_settings._use_manager.getUseForce(
-					pkg, stable=dep_settings._parent_stable)
-				dep_settings.usemask = dep_settings._use_manager.getUseMask(
-					pkg, stable=dep_settings._parent_stable)
-
-				if not baddepsyntax:
-					ismasked = not ebuild_archs or \
-						pkg.cpv not in portdb.xmatch("match-visible",
-						Atom("%s::%s" % (pkg.cp, repo_settings.repo_config.name)))
-					if ismasked:
-						if not have_pmasked:
-							have_pmasked = bool(dep_settings._getMaskAtom(
-								pkg.cpv, pkg._metadata))
-						if options.ignore_masked:
-							continue
-						# we are testing deps for a masked package; give it some lee-way
-						suffix = "masked"
-						matchmode = "minimum-all"
-					else:
-						suffix = ""
-						matchmode = "minimum-visible"
-
-					if not have_dev_keywords:
-						have_dev_keywords = \
-							bool(dev_keywords.intersection(keywords))
-
-					if prof.status == "dev":
-						suffix = suffix + "indev"
-
-					for mytype in Package._dep_keys:
-
-						mykey = "dependency.bad" + suffix
-						myvalue = myaux[mytype]
-						if not myvalue:
-							continue
-
-						success, atoms = portage.dep_check(
-							myvalue, portdb, dep_settings,
-							use="all", mode=matchmode, trees=repo_settings.trees)
-
-						if success:
-							if atoms:
-
-								# Don't bother with dependency.unknown for
-								# cases in which *DEPEND.bad is triggered.
-								for atom in atoms:
-									# dep_check returns all blockers and they
-									# aren't counted for *DEPEND.bad, so we
-									# ignore them here.
-									if not atom.blocker:
-										unknown_pkgs.discard(
-											(mytype, atom.unevaluated_atom))
-
-								if not prof.sub_path:
-									# old-style virtuals currently aren't
-									# resolvable with empty profile, since
-									# 'virtuals' mappings are unavailable
-									# (it would be expensive to search
-									# for PROVIDE in all ebuilds)
-									atoms = [
-										atom for atom in atoms if not (
-											atom.cp.startswith('virtual/')
-											and not portdb.cp_list(atom.cp))]
-
-								# we have some unsolvable deps
-								# remove ! deps, which always show up as unsatisfiable
-								atoms = [
-									str(atom.unevaluated_atom)
-									for atom in atoms if not atom.blocker]
-
-								# if we emptied out our list, continue:
-								if not atoms:
-									continue
-								qatracker.add_error(mykey,
-									"%s: %s: %s(%s)\n%s"
-									% (ebuild.relative_path, mytype, keyword, prof,
-										pformat(atoms, indent=6)))
-						else:
-							qatracker.add_error(mykey,
-								"%s: %s: %s(%s)\n%s"
-								% (ebuild.relative_path, mytype, keyword, prof,
-									pformat(atoms, indent=6)))
-
-			if not baddepsyntax and unknown_pkgs:
-				type_map = {}
-				for mytype, atom in unknown_pkgs:
-					type_map.setdefault(mytype, set()).add(atom)
-				for mytype, atoms in type_map.items():
-					qatracker.add_error(
-						"dependency.unknown", "%s: %s: %s"
-						% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))
-
-		# check if there are unused local USE-descriptions in metadata.xml
-		# (unless there are any invalids, to avoid noise)
-		if allvalid:
-			for myflag in muselist.difference(used_useflags):
-				qatracker.add_error(
-					"metadata.warning",
-					"%s/metadata.xml: unused local USE-description: '%s'"
-					% (xpkg, myflag))
-
-
-	if options.if_modified == "y" and len(effective_scanlist) < 1:
+	if options.if_modified == "y" and len(scanner.effective_scanlist) < 1:
 		logging.warning("--if-modified is enabled, but no modified packages were found!")
 
-	if options.mode == "manifest":
-		sys.exit(dofail)
-
 	# dofail will be true if we have failed in at least one non-warning category
 	dofail = 0
 	# dowarn will be true if we tripped any warnings
@@ -842,6 +138,10 @@ def repoman_main(argv):
 	# dofull will be true if we should print a "repoman full" informational message
 	dofull = options.mode != 'full'
 
+	# early out for manifest generation
+	if options.mode == "manifest":
+		sys.exit(dofail)
+
 	for x in qacats:
 		if x not in qatracker.fails:
 			continue
@@ -884,9 +184,9 @@ def repoman_main(argv):
 	suggest_ignore_masked = False
 	suggest_include_dev = False
 
-	if have_pmasked and not (options.without_mask or options.ignore_masked):
+	if scanner.have['pmasked'] and not (options.without_mask or options.ignore_masked):
 		suggest_ignore_masked = True
-	if have_dev_keywords and not options.include_dev:
+	if scanner.have['dev_keywords'] and not options.include_dev:
 		suggest_include_dev = True
 
 	if suggest_ignore_masked or suggest_include_dev:
@@ -1164,8 +464,8 @@ def repoman_main(argv):
 			commitmessagefile = None
 		if not commitmessage or not commitmessage.strip():
 			msg_prefix = ""
-			if repolevel > 1:
-				msg_prefix = "/".join(reposplit[1:]) + ": "
+			if scanner.repolevel > 1:
+				msg_prefix = "/".join(scanner.reposplit[1:]) + ": "
 
 			try:
 				editor = os.environ.get("EDITOR")
@@ -1196,10 +496,10 @@ def repoman_main(argv):
 			report_options.append("--force")
 		if options.ignore_arches:
 			report_options.append("--ignore-arches")
-		if include_arches is not None:
+		if scanner.include_arches is not None:
 			report_options.append(
 				"--include-arches=\"%s\"" %
-				" ".join(sorted(include_arches)))
+				" ".join(sorted(scanner.include_arches)))
 
 		if vcs_settings.vcs == "git":
 			# Use new footer only for git (see bug #438364).
@@ -1238,18 +538,18 @@ def repoman_main(argv):
 			committer_name = utilities.get_committer_name(env=repoman_settings)
 			for x in sorted(vcs_files_to_cps(
 				chain(myupdates, mymanifests, myremoved),
-				repolevel, reposplit, categories)):
+				scanner.repolevel, scanner.reposplit, scanner.categories)):
 				catdir, pkgdir = x.split("/")
 				checkdir = repo_settings.repodir + "/" + x
 				checkdir_relative = ""
-				if repolevel < 3:
+				if scanner.repolevel < 3:
 					checkdir_relative = os.path.join(pkgdir, checkdir_relative)
-				if repolevel < 2:
+				if scanner.repolevel < 2:
 					checkdir_relative = os.path.join(catdir, checkdir_relative)
 				checkdir_relative = os.path.join(".", checkdir_relative)
 
 				changelog_path = os.path.join(checkdir_relative, "ChangeLog")
-				changelog_modified = changelog_path in changed.changelogs
+				changelog_modified = changelog_path in scanner.changed.changelogs
 				if changelog_modified and options.echangelog != 'force':
 					continue
 
@@ -1459,7 +759,7 @@ def repoman_main(argv):
 			if modified:
 				portage.util.write_atomic(x, b''.join(mylines), mode='wb')
 
-		if repolevel == 1:
+		if scanner.repolevel == 1:
 			utilities.repoman_sez(
 				"\"You're rather crazy... "
 				"doing the entire repository.\"\n")
@@ -1467,7 +767,7 @@ def repoman_main(argv):
 		if vcs_settings.vcs in ('cvs', 'svn') and (myupdates or myremoved):
 			for x in sorted(vcs_files_to_cps(
 				chain(myupdates, myremoved, mymanifests),
-				repolevel, reposplit, categories)):
+				scanner.repolevel, scanner.reposplit, scanner.categories)):
 				repoman_settings["O"] = os.path.join(repo_settings.repodir, x)
 				digestgen(mysettings=repoman_settings, myportdb=portdb)
 
@@ -1480,7 +780,7 @@ def repoman_main(argv):
 			try:
 				for x in sorted(vcs_files_to_cps(
 					chain(myupdates, myremoved, mymanifests),
-					repolevel, reposplit, categories)):
+					scanner.repolevel, scanner.reposplit, scanner.categories)):
 					repoman_settings["O"] = os.path.join(repo_settings.repodir, x)
 					manifest_path = os.path.join(repoman_settings["O"], "Manifest")
 					if not need_signature(manifest_path):

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
new file mode 100644
index 0000000..44ff33b
--- /dev/null
+++ b/pym/repoman/scanner.py
@@ -0,0 +1,715 @@
+
+import copy
+import io
+import logging
+import re
+import sys
+from itertools import chain
+from pprint import pformat
+
+from _emerge.Package import Package
+
+import portage
+from portage import normalize_path
+from portage import os
+from portage import _encodings
+from portage import _unicode_encode
+from portage.dep import Atom
+from portage.output import green
+from repoman.checks.directories.files import FileChecks
+from repoman.checks.ebuilds.checks import run_checks
+from repoman.checks.ebuilds.eclasses.live import LiveEclassChecks
+from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks
+from repoman.checks.ebuilds.fetches import FetchChecks
+from repoman.checks.ebuilds.keywords import KeywordChecks
+from repoman.checks.ebuilds.isebuild import IsEbuild
+from repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors
+from repoman.checks.ebuilds.manifests import Manifests
+from repoman.check_missingslot import check_missingslot
+from repoman.checks.ebuilds.misc import bad_split_check, pkg_invalid
+from repoman.checks.ebuilds.pkgmetadata import PkgMetadata
+from repoman.checks.ebuilds.use_flags import USEFlagChecks
+from repoman.checks.ebuilds.variables.description import DescriptionChecks
+from repoman.checks.ebuilds.variables.eapi import EAPIChecks
+from repoman.checks.ebuilds.variables.license import LicenseChecks
+from repoman.checks.ebuilds.variables.restrict import RestrictChecks
+from repoman.ebuild import Ebuild
+from repoman.modules.commit import repochecks
+from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
+from repoman.qa_data import missingvars, suspect_virtual, suspect_rdepend
+from repoman.qa_tracker import QATracker
+from repoman.repos import repo_metadata
+from repoman.scan import Changes, scan
+from repoman.vcs.vcsstatus import VCSStatus
+from repoman.vcs.vcs import vcs_files_to_cps
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+def sort_key(item):
+	return item[2].sub_path
+
+
+
+class Scanner(object):
+	'''Primary scan class.  Operates all the small Q/A tests and checks'''
+
+	def __init__(self, repo_settings, myreporoot, config_root, options,
+				vcs_settings, mydir, env):
+		'''Class __init__'''
+		self.repo_settings = repo_settings
+		self.config_root = config_root
+		self.options = options
+		self.vcs_settings = vcs_settings
+		self.env = env
+
+		# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
+		# behave incrementally.
+		self.repoman_incrementals = tuple(
+			x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
+
+		self.categories = []
+		for path in self.repo_settings.repo_config.eclass_db.porttrees:
+			self.categories.extend(portage.util.grabfile(
+				os.path.join(path, 'profiles', 'categories')))
+		self.repo_settings.repoman_settings.categories = frozenset(
+			portage.util.stack_lists([self.categories], incremental=1))
+		self.categories = self.repo_settings.repoman_settings.categories
+
+		self.portdb = repo_settings.portdb
+		self.portdb.settings = self.repo_settings.repoman_settings
+		# We really only need to cache the metadata that's necessary for visibility
+		# filtering. Anything else can be discarded to reduce memory consumption.
+		self.portdb._aux_cache_keys.clear()
+		self.portdb._aux_cache_keys.update(
+			["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
+
+		self.reposplit = myreporoot.split(os.path.sep)
+		self.repolevel = len(self.reposplit)
+
+		if self.options.mode == 'commit':
+			repochecks.commit_check(self.repolevel, self.reposplit)
+			repochecks.conflict_check(self.vcs_settings, self.options)
+
+		# Make startdir relative to the canonical repodir, so that we can pass
+		# it to digestgen and it won't have to be canonicalized again.
+		if self.repolevel == 1:
+			startdir = self.repo_settings.repodir
+		else:
+			startdir = normalize_path(mydir)
+			startdir = os.path.join(
+				self.repo_settings.repodir, *startdir.split(os.sep)[-2 - self.repolevel + 3:])
+
+		# get lists of valid keywords, licenses, and use
+		new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings)
+		kwlist, liclist, uselist, profile_list, \
+			global_pmaskdict, liclist_deprecated = new_data
+		self.repo_metadata = {
+			'kwlist': kwlist,
+			'liclist': liclist,
+			'uselist': uselist,
+			'profile_list': profile_list,
+			'pmaskdict': global_pmaskdict,
+			'lic_deprecated': liclist_deprecated,
+		}
+
+		self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
+		self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')
+
+		self.profiles = setup_profile(profile_list)
+
+		check_profiles(self.profiles, self.repo_settings.repoman_settings.archlist())
+
+		scanlist = scan(self.repolevel, self.reposplit, startdir, self.categories, self.repo_settings)
+
+		self.dev_keywords = dev_profile_keywords(self.profiles)
+
+		self.qatracker = QATracker()
+
+		if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog:
+			self.options.echangelog = 'y'
+
+		if self.vcs_settings.vcs is None:
+			self.options.echangelog = 'n'
+
+		self.check = {}
+		# The --echangelog option causes automatic ChangeLog generation,
+		# which invalidates changelog.ebuildadded and changelog.missing
+		# checks.
+		# Note: Some don't use ChangeLogs in distributed SCMs.
+		# It will be generated on server side from scm log,
+		# before package moves to the rsync server.
+		# This is needed because they try to avoid merge collisions.
+		# Gentoo's Council decided to always use the ChangeLog file.
+		# TODO: shouldn't this just be switched on the repo, iso the VCS?
+		is_echangelog_enabled = self.options.echangelog in ('y', 'force')
+		self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs', 'svn')
+		self.check['changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn
+
+		if self.options.mode == "manifest":
+			pass
+		elif self.options.pretend:
+			print(green("\nRepoMan does a once-over of the neighborhood..."))
+		else:
+			print(green("\nRepoMan scours the neighborhood..."))
+
+		self.changed = Changes(self.options)
+		self.changed.scan(self.vcs_settings)
+
+		self.have = {
+			'pmasked': False,
+			'dev_keywords': False,
+		}
+
+		# NOTE: match-all caches are not shared due to potential
+		# differences between profiles in _get_implicit_iuse.
+		self.caches = {
+			'arch': {},
+			'arch_xmatch': {},
+			'shared_xmatch': {"cp-list": {}},
+		}
+
+		self.include_arches = None
+		if self.options.include_arches:
+			self.include_arches = set()
+			self.include_arches.update(*[x.split() for x in self.options.include_arches])
+
+		# Disable the "ebuild.notadded" check when not in commit mode and
+		# running `svn status` in every package dir will be too expensive.
+		self.check['ebuild_notadded'] = not \
+			(self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit")
+
+		self.effective_scanlist = scanlist
+		if self.options.if_modified == "y":
+			self.effective_scanlist = sorted(vcs_files_to_cps(
+				chain(self.changed.changed, self.changed.new, self.changed.removed),
+				self.repolevel, self.reposplit, self.categories))
+
+		self.live_eclasses = portage.const.LIVE_ECLASSES
+
+		# initialize our checks classes here before the big xpkg loop
+		self.manifester = Manifests(self.options, self.qatracker, self.repo_settings.repoman_settings)
+		self.is_ebuild = IsEbuild(self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.qatracker)
+		self.filescheck = FileChecks(
+			self.qatracker, self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.vcs_settings)
+		self.status_check = VCSStatus(self.vcs_settings, self.qatracker)
+		self.fetchcheck = FetchChecks(
+			self.qatracker, self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.vcs_settings)
+		self.pkgmeta = PkgMetadata(self.options, self.qatracker, self.repo_settings.repoman_settings)
+		self.thirdparty = ThirdPartyMirrors(self.repo_settings.repoman_settings, self.qatracker)
+		self.use_flag_checks = USEFlagChecks(self.qatracker, uselist)
+		self.keywordcheck = KeywordChecks(self.qatracker, self.options)
+		self.liveeclasscheck = LiveEclassChecks(self.qatracker)
+		self.rubyeclasscheck = RubyEclassChecks(self.qatracker)
+		self.eapicheck = EAPIChecks(self.qatracker, self.repo_settings)
+		self.descriptioncheck = DescriptionChecks(self.qatracker)
+		self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated)
+		self.restrictcheck = RestrictChecks(self.qatracker)
+
+
+	def scan_pkgs(self, can_force):
+		for xpkg in self.effective_scanlist:
+			# ebuilds and digests added to cvs respectively.
+			logging.info("checking package %s" % xpkg)
+			# save memory by discarding xmatch caches from previous package(s)
+			self.caches['arch_xmatch'].clear()
+			self.eadded = []
+			catdir, pkgdir = xpkg.split("/")
+			checkdir = self.repo_settings.repodir + "/" + xpkg
+			checkdir_relative = ""
+			if self.repolevel < 3:
+				checkdir_relative = os.path.join(pkgdir, checkdir_relative)
+			if self.repolevel < 2:
+				checkdir_relative = os.path.join(catdir, checkdir_relative)
+			checkdir_relative = os.path.join(".", checkdir_relative)
+
+			if self.manifester.run(checkdir, self.portdb):
+				continue
+			if not self.manifester.generated_manifest:
+				self.manifester.digest_check(xpkg, checkdir)
+			if self.options.mode == 'manifest-check':
+				continue
+
+			checkdirlist = os.listdir(checkdir)
+
+			self.pkgs, self.allvalid = self.is_ebuild.check(checkdirlist, checkdir, xpkg)
+			if self.is_ebuild.continue_:
+				# If we can't access all the metadata then it's totally unsafe to
+				# commit since there's no way to generate a correct Manifest.
+				# Do not try to do any more QA checks on this package since missing
+				# metadata leads to false positives for several checks, and false
+				# positives confuse users.
+				can_force = False
+				continue
+
+			self.keywordcheck.prepare()
+
+			# Sort ebuilds in ascending order for the KEYWORDS.dropped check.
+			ebuildlist = sorted(self.pkgs.values())
+			ebuildlist = [pkg.pf for pkg in ebuildlist]
+
+			self.filescheck.check(
+				checkdir, checkdirlist, checkdir_relative, self.changed.changed, self.changed.new)
+
+			self.status_check.check(self.check['ebuild_notadded'], checkdir, checkdir_relative, xpkg)
+			self.eadded.extend(self.status_check.eadded)
+
+			self.fetchcheck.check(
+				xpkg, checkdir, checkdir_relative, self.changed.changed, self.changed.new)
+
+			if self.check['changelog'] and "ChangeLog" not in checkdirlist:
+				self.qatracker.add_error("changelog.missing", xpkg + "/ChangeLog")
+
+			self.pkgmeta.check(xpkg, checkdir, checkdirlist, self.repolevel)
+			self.muselist = frozenset(self.pkgmeta.musedict)
+
+			changelog_path = os.path.join(checkdir_relative, "ChangeLog")
+			self.changelog_modified = changelog_path in self.changed.changelogs
+
+			self._scan_ebuilds(ebuildlist, xpkg, catdir, pkgdir)
+		return self.qatracker, can_force
+
+
+	def _scan_ebuilds(self, ebuildlist, xpkg, catdir, pkgdir):
+		# detect unused local USE-descriptions
+		used_useflags = set()
+
+		for y_ebuild in ebuildlist:
+
+			ebuild = Ebuild(
+				self.repo_settings, self.repolevel, pkgdir, catdir, self.vcs_settings,
+				xpkg, y_ebuild)
+
+			if self.check['changelog'] and not self.changelog_modified \
+				and ebuild.ebuild_path in self.changed.new_ebuilds:
+				self.qatracker.add_error('changelog.ebuildadded', ebuild.relative_path)
+
+			if ebuild.untracked(self.check['ebuild_notadded'], y_ebuild, self.eadded):
+				# ebuild not added to vcs
+				self.qatracker.add_error(
+					"ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild")
+
+			if bad_split_check(xpkg, y_ebuild, pkgdir, self.qatracker):
+				continue
+
+			pkg = self.pkgs[y_ebuild]
+			if pkg_invalid(pkg, self.qatracker, ebuild):
+				self.allvalid = False
+				continue
+
+			myaux = pkg._metadata
+			eapi = myaux["EAPI"]
+			inherited = pkg.inherited
+			live_ebuild = self.live_eclasses.intersection(inherited)
+
+			self.eapicheck.check(pkg, ebuild)
+
+			for k, v in myaux.items():
+				if not isinstance(v, basestring):
+					continue
+				m = NON_ASCII_RE.search(v)
+				if m is not None:
+					self.qatracker.add_error(
+						"variable.invalidchar",
+						"%s: %s variable contains non-ASCII "
+						"character at position %s" %
+						(ebuild.relative_path, k, m.start() + 1))
+
+			if not self.fetchcheck.src_uri_error:
+				self.thirdparty.check(myaux, ebuild.relative_path)
+
+			if myaux.get("PROVIDE"):
+				self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
+
+			for pos, missing_var in enumerate(missingvars):
+				if not myaux.get(missing_var):
+					if catdir == "virtual" and \
+						missing_var in ("HOMEPAGE", "LICENSE"):
+						continue
+					if live_ebuild and missing_var == "KEYWORDS":
+						continue
+					myqakey = missingvars[pos] + ".missing"
+					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
+
+			if catdir == "virtual":
+				for var in ("HOMEPAGE", "LICENSE"):
+					if myaux.get(var):
+						myqakey = var + ".virtual"
+						self.qatracker.add_error(myqakey, ebuild.relative_path)
+
+			self.descriptioncheck.check(pkg, ebuild)
+
+			keywords = myaux["KEYWORDS"].split()
+
+			ebuild_archs = set(
+				kw.lstrip("~") for kw in keywords if not kw.startswith("-"))
+
+			self.keywordcheck.check(
+				pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, self.changed,
+				live_ebuild, self.repo_metadata['kwlist'], self.profiles)
+
+			if live_ebuild and self.repo_settings.repo_config.name == "gentoo":
+				self.liveeclasscheck.check(
+					pkg, xpkg, ebuild, y_ebuild, keywords, self.repo_metadata['pmaskdict'])
+
+			if self.options.ignore_arches:
+				arches = [[
+					self.repo_settings.repoman_settings["ARCH"], self.repo_settings.repoman_settings["ARCH"],
+					self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
+			else:
+				arches = set()
+				for keyword in keywords:
+					if keyword[0] == "-":
+						continue
+					elif keyword[0] == "~":
+						arch = keyword[1:]
+						if arch == "*":
+							for expanded_arch in self.profiles:
+								if expanded_arch == "**":
+									continue
+								arches.add(
+									(keyword, expanded_arch, (
+										expanded_arch, "~" + expanded_arch)))
+						else:
+							arches.add((keyword, arch, (arch, keyword)))
+					else:
+						if keyword == "*":
+							for expanded_arch in self.profiles:
+								if expanded_arch == "**":
+									continue
+								arches.add(
+									(keyword, expanded_arch, (expanded_arch,)))
+						else:
+							arches.add((keyword, keyword, (keyword,)))
+				if not arches:
+					# Use an empty profile for checking dependencies of
+					# packages that have empty KEYWORDS.
+					arches.add(('**', '**', ('**',)))
+
+			unknown_pkgs = set()
+			baddepsyntax = False
+			badlicsyntax = False
+			badprovsyntax = False
+			# catpkg = catdir + "/" + y_ebuild
+
+			inherited_java_eclass = "java-pkg-2" in inherited or \
+				"java-pkg-opt-2" in inherited
+			inherited_wxwidgets_eclass = "wxwidgets" in inherited
+			# operator_tokens = set(["||", "(", ")"])
+			type_list, badsyntax = [], []
+			for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
+				mydepstr = myaux[mytype]
+
+				buildtime = mytype in Package._buildtime_keys
+				runtime = mytype in Package._runtime_keys
+				token_class = None
+				if mytype.endswith("DEPEND"):
+					token_class = portage.dep.Atom
+
+				try:
+					atoms = portage.dep.use_reduce(
+						mydepstr, matchall=1, flat=True,
+						is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
+				except portage.exception.InvalidDependString as e:
+					atoms = None
+					badsyntax.append(str(e))
+
+				if atoms and mytype.endswith("DEPEND"):
+					if runtime and \
+						"test?" in mydepstr.split():
+						self.qatracker.add_error(
+							mytype + '.suspect',
+							"%s: 'test?' USE conditional in %s" %
+							(ebuild.relative_path, mytype))
+
+					for atom in atoms:
+						if atom == "||":
+							continue
+
+						is_blocker = atom.blocker
+
+						# Skip dependency.unknown for blockers, so that we
+						# don't encourage people to remove necessary blockers,
+						# as discussed in bug 382407. We use atom.without_use
+						# due to bug 525376.
+						if not is_blocker and \
+							not self.portdb.xmatch("match-all", atom.without_use) and \
+							not atom.cp.startswith("virtual/"):
+							unknown_pkgs.add((mytype, atom.unevaluated_atom))
+
+						if catdir != "virtual":
+							if not is_blocker and \
+								atom.cp in suspect_virtual:
+								self.qatracker.add_error(
+									'virtual.suspect', ebuild.relative_path +
+									": %s: consider using '%s' instead of '%s'" %
+									(mytype, suspect_virtual[atom.cp], atom))
+							if not is_blocker and \
+								atom.cp.startswith("perl-core/"):
+								self.qatracker.add_error('dependency.perlcore',
+									ebuild.relative_path +
+									": %s: please use '%s' instead of '%s'" %
+									(mytype,
+									atom.replace("perl-core/","virtual/perl-"),
+									atom))
+
+						if buildtime and \
+							not is_blocker and \
+							not inherited_java_eclass and \
+							atom.cp == "virtual/jdk":
+							self.qatracker.add_error(
+								'java.eclassesnotused', ebuild.relative_path)
+						elif buildtime and \
+							not is_blocker and \
+							not inherited_wxwidgets_eclass and \
+							atom.cp == "x11-libs/wxGTK":
+							self.qatracker.add_error(
+								'wxwidgets.eclassnotused',
+								"%s: %ss on x11-libs/wxGTK without inheriting"
+								" wxwidgets.eclass" % (ebuild.relative_path, mytype))
+						elif runtime:
+							if not is_blocker and \
+								atom.cp in suspect_rdepend:
+								self.qatracker.add_error(
+									mytype + '.suspect',
+									ebuild.relative_path + ": '%s'" % atom)
+
+						if atom.operator == "~" and \
+							portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
+							qacat = 'dependency.badtilde'
+							self.qatracker.add_error(
+								qacat, "%s: %s uses the ~ operator"
+								" with a non-zero revision: '%s'" %
+								(ebuild.relative_path, mytype, atom))
+
+						check_missingslot(atom, mytype, eapi, self.portdb, self.qatracker,
+							ebuild.relative_path, myaux)
+
+				type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
+
+			for m, b in zip(type_list, badsyntax):
+				if m.endswith("DEPEND"):
+					qacat = "dependency.syntax"
+				else:
+					qacat = m + ".syntax"
+				self.qatracker.add_error(
+					qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
+
+			badlicsyntax = len([z for z in type_list if z == "LICENSE"])
+			badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
+			baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
+			badlicsyntax = badlicsyntax > 0
+			badprovsyntax = badprovsyntax > 0
+
+			self.use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, self.muselist)
+
+			ebuild_used_useflags = self.use_flag_checks.getUsedUseFlags()
+			used_useflags = used_useflags.union(ebuild_used_useflags)
+
+			self.rubyeclasscheck.check(pkg, ebuild)
+
+			# license checks
+			if not badlicsyntax:
+				self.licensecheck.check(pkg, xpkg, ebuild, y_ebuild)
+
+			self.restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
+
+			# Syntax Checks
+			if not self.vcs_settings.vcs_preserves_mtime:
+				if ebuild.ebuild_path not in self.changed.new_ebuilds and \
+					ebuild.ebuild_path not in self.changed.ebuilds:
+					pkg.mtime = None
+			try:
+				# All ebuilds should have utf_8 encoding.
+				f = io.open(
+					_unicode_encode(
+						ebuild.full_path, encoding=_encodings['fs'], errors='strict'),
+					mode='r', encoding=_encodings['repo.content'])
+				try:
+					for check_name, e in run_checks(f, pkg):
+						self.qatracker.add_error(
+							check_name, ebuild.relative_path + ': %s' % e)
+				finally:
+					f.close()
+			except UnicodeDecodeError:
+				# A file.UTF8 failure will have already been recorded above.
+				pass
+
+			if self.options.force:
+				# The dep_check() calls are the most expensive QA test. If --force
+				# is enabled, there's no point in wasting time on these since the
+				# user is intent on forcing the commit anyway.
+				continue
+
+			relevant_profiles = []
+			for keyword, arch, groups in arches:
+				if arch not in self.profiles:
+					# A missing profile will create an error further down
+					# during the KEYWORDS verification.
+					continue
+
+				if self.include_arches is not None:
+					if arch not in self.include_arches:
+						continue
+
+				relevant_profiles.extend(
+					(keyword, groups, prof) for prof in self.profiles[arch])
+
+			relevant_profiles.sort(key=sort_key)
+
+			for keyword, groups, prof in relevant_profiles:
+
+				is_stable_profile = prof.status == "stable"
+				is_dev_profile = prof.status == "dev" and \
+					self.options.include_dev
+				is_exp_profile = prof.status == "exp" and \
+					self.options.include_exp_profiles == 'y'
+				if not (is_stable_profile or is_dev_profile or is_exp_profile):
+					continue
+
+				dep_settings = self.caches['arch'].get(prof.sub_path)
+				if dep_settings is None:
+					dep_settings = portage.config(
+						config_profile_path=prof.abs_path,
+						config_incrementals=self.repoman_incrementals,
+						config_root=self.config_root,
+						local_config=False,
+						_unmatched_removal=self.options.unmatched_removal,
+						env=self.env, repositories=self.repo_settings.repoman_settings.repositories)
+					dep_settings.categories = self.repo_settings.repoman_settings.categories
+					if self.options.without_mask:
+						dep_settings._mask_manager_obj = \
+							copy.deepcopy(dep_settings._mask_manager)
+						dep_settings._mask_manager._pmaskdict.clear()
+					self.caches['arch'][prof.sub_path] = dep_settings
+
+				xmatch_cache_key = (prof.sub_path, tuple(groups))
+				xcache = self.caches['arch_xmatch'].get(xmatch_cache_key)
+				if xcache is None:
+					self.portdb.melt()
+					self.portdb.freeze()
+					xcache = self.portdb.xcache
+					xcache.update(self.caches['shared_xmatch'])
+					self.caches['arch_xmatch'][xmatch_cache_key] = xcache
+
+				self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings
+				self.portdb.settings = dep_settings
+				self.portdb.xcache = xcache
+
+				dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
+				# just in case, prevent config.reset() from nuking these.
+				dep_settings.backup_changes("ACCEPT_KEYWORDS")
+
+				# This attribute is used in dbapi._match_use() to apply
+				# use.stable.{mask,force} settings based on the stable
+				# status of the parent package. This is required in order
+				# for USE deps of unstable packages to be resolved correctly,
+				# since otherwise use.stable.{mask,force} settings of
+				# dependencies may conflict (see bug #456342).
+				dep_settings._parent_stable = dep_settings._isStable(pkg)
+
+				# Handle package.use*.{force,mask) calculation, for use
+				# in dep_check.
+				dep_settings.useforce = dep_settings._use_manager.getUseForce(
+					pkg, stable=dep_settings._parent_stable)
+				dep_settings.usemask = dep_settings._use_manager.getUseMask(
+					pkg, stable=dep_settings._parent_stable)
+
+				if not baddepsyntax:
+					ismasked = not ebuild_archs or \
+						pkg.cpv not in self.portdb.xmatch("match-visible",
+						Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name)))
+					if ismasked:
+						if not self.have['pmasked']:
+							self.have['pmasked'] = bool(dep_settings._getMaskAtom(
+								pkg.cpv, pkg._metadata))
+						if self.options.ignore_masked:
+							continue
+						# we are testing deps for a masked package; give it some lee-way
+						suffix = "masked"
+						matchmode = "minimum-all"
+					else:
+						suffix = ""
+						matchmode = "minimum-visible"
+
+					if not self.have['dev_keywords']:
+						self.have['dev_keywords'] = \
+							bool(self.dev_keywords.intersection(keywords))
+
+					if prof.status == "dev":
+						suffix = suffix + "indev"
+
+					for mytype in Package._dep_keys:
+
+						mykey = "dependency.bad" + suffix
+						myvalue = myaux[mytype]
+						if not myvalue:
+							continue
+
+						success, atoms = portage.dep_check(
+							myvalue, self.portdb, dep_settings,
+							use="all", mode=matchmode, trees=self.repo_settings.trees)
+
+						if success:
+							if atoms:
+
+								# Don't bother with dependency.unknown for
+								# cases in which *DEPEND.bad is triggered.
+								for atom in atoms:
+									# dep_check returns all blockers and they
+									# aren't counted for *DEPEND.bad, so we
+									# ignore them here.
+									if not atom.blocker:
+										unknown_pkgs.discard(
+											(mytype, atom.unevaluated_atom))
+
+								if not prof.sub_path:
+									# old-style virtuals currently aren't
+									# resolvable with empty profile, since
+									# 'virtuals' mappings are unavailable
+									# (it would be expensive to search
+									# for PROVIDE in all ebuilds)
+									atoms = [
+										atom for atom in atoms if not (
+											atom.cp.startswith('virtual/')
+											and not self.portdb.cp_list(atom.cp))]
+
+								# we have some unsolvable deps
+								# remove ! deps, which always show up as unsatisfiable
+								atoms = [
+									str(atom.unevaluated_atom)
+									for atom in atoms if not atom.blocker]
+
+								# if we emptied out our list, continue:
+								if not atoms:
+									continue
+								self.qatracker.add_error(mykey,
+									"%s: %s: %s(%s)\n%s"
+									% (ebuild.relative_path, mytype, keyword, prof,
+										pformat(atoms, indent=6)))
+						else:
+							self.qatracker.add_error(mykey,
+								"%s: %s: %s(%s)\n%s"
+								% (ebuild.relative_path, mytype, keyword, prof,
+									pformat(atoms, indent=6)))
+
+			if not baddepsyntax and unknown_pkgs:
+				type_map = {}
+				for mytype, atom in unknown_pkgs:
+					type_map.setdefault(mytype, set()).add(atom)
+				for mytype, atoms in type_map.items():
+					self.qatracker.add_error(
+						"dependency.unknown", "%s: %s: %s"
+						% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))
+
+		# check if there are unused local USE-descriptions in metadata.xml
+		# (unless there are any invalids, to avoid noise)
+		if self.allvalid:
+			for myflag in self.muselist.difference(used_useflags):
+				self.qatracker.add_error(
+					"metadata.warning",
+					"%s/metadata.xml: unused local USE-description: '%s'"
+					% (xpkg, myflag))