From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id C5CAA1381FA for ; Fri, 30 May 2014 19:26:01 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id C5B63E092E; Fri, 30 May 2014 19:25:59 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 33EC3E092E for ; Fri, 30 May 2014 19:25:59 +0000 (UTC) Received: from spoonbill.gentoo.org (spoonbill.gentoo.org [81.93.255.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id D1C2C33F195 for ; Fri, 30 May 2014 19:25:57 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by spoonbill.gentoo.org (Postfix) with ESMTP id 30311182D3 for ; Fri, 30 May 2014 19:25:56 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1401477771.807324e57f63918e0ee64837608caaa28a0e53a6.dol-sen@gentoo> Subject: [gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/vcs/, pym/repoman/checks/ebuilds/ X-VCS-Repository: proj/portage X-VCS-Files: pym/repoman/checks/ebuilds/thirdpartymirrors.py pym/repoman/main.py pym/repoman/qa_data.py pym/repoman/qa_tracker.py pym/repoman/vcs/vcsstatus.py X-VCS-Directories: pym/repoman/ pym/repoman/vcs/ pym/repoman/checks/ebuilds/ X-VCS-Committer: dol-sen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: 807324e57f63918e0ee64837608caaa28a0e53a6 X-VCS-Branch: repoman Date: Fri, 30 May 2014 19:25:56 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: cac8083a-ff6e-4fac-b601-9095ef82ffd8 X-Archives-Hash: 79c096a3be243d782443b2abcebacb41 commit: 807324e57f63918e0ee64837608caaa28a0e53a6 Author: Brian Dolbec gentoo org> AuthorDate: Fri May 30 18:36:11 2014 +0000 Commit: Brian Dolbec gmail com> CommitDate: Fri May 30 19:22:51 2014 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=807324e5 Repoman: Create a new QATracker class Make use of the new QATracker class to track all fails. Use the length of each tracker for stats count. Only store trackers if they occurred. QATracker roughed in for tracking warnings as well as errors. --- pym/repoman/checks/ebuilds/thirdpartymirrors.py | 11 +- pym/repoman/main.py | 261 +++++++++--------------- pym/repoman/qa_data.py | 16 +- pym/repoman/qa_tracker.py | 46 +++++ pym/repoman/vcs/vcsstatus.py | 23 +-- 5 files changed, 161 insertions(+), 196 deletions(-) diff --git a/pym/repoman/checks/ebuilds/thirdpartymirrors.py b/pym/repoman/checks/ebuilds/thirdpartymirrors.py index 976a62c..cce61f6 100644 --- a/pym/repoman/checks/ebuilds/thirdpartymirrors.py +++ b/pym/repoman/checks/ebuilds/thirdpartymirrors.py @@ -4,7 +4,7 @@ import portage class ThirdPartyMirrors(object): - def __init__(self, repoman_settings): + def __init__(self, repoman_settings, qatracker): # Build a regex from thirdpartymirrors for the SRC_URI.mirror check. self.thirdpartymirrors = {} for k, v in repoman_settings.thirdpartymirrors().items(): @@ -12,14 +12,10 @@ class ThirdPartyMirrors(object): if not v.endswith("/"): v += "/" self.thirdpartymirrors[v] = k - self.stats = 0 - self.fails = [] + self.qatracker = qatracker def check(self, myaux, relative_path): - # reset our stats in case this is a repeat run - self.stats = 0 - self.fails = [] # Check that URIs don't reference a server from thirdpartymirrors. for uri in portage.dep.use_reduce( myaux["SRC_URI"], matchall=True, is_src_uri=True, @@ -33,8 +29,7 @@ class ThirdPartyMirrors(object): continue new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):]) - self.stats += 1 - self.fails.append( + self.qatracker.add_error("SRC_URI.mirror", "%s: '%s' found in thirdpartymirrors, use '%s'" % (relative_path, mirror, new_uri)) return diff --git a/pym/repoman/main.py b/pym/repoman/main.py index f48c8ba..3a1ff7a 100755 --- a/pym/repoman/main.py +++ b/pym/repoman/main.py @@ -77,6 +77,7 @@ from repoman.profile import check_profiles, dev_keywords, setup_profile from repoman.qa_data import (format_qa_output, format_qa_output_column, qahelp, qawarnings, qacats, no_exec, allvars, max_desc_len, missingvars, ruby_deprecated, suspect_virtual, suspect_rdepend, valid_restrict) +from qa_tracker import QATracker from repoman.repos import has_global_mask, RepoSettings, repo_metadata from repoman.scan import Changes, scan from repoman._subprocess import repoman_popen, repoman_getstatusoutput @@ -258,12 +259,7 @@ scanlist = scan(repolevel, reposplit, startdir, categories, repo_settings) dev_keywords = dev_keywords(profiles) -stats = {} -fails = {} - -for x in qacats: - stats[x] = 0 - fails[x] = [] +qatracker = QATracker() #################### @@ -348,8 +344,7 @@ for xpkg in effective_scanlist: repoman_settings['O'] = checkdir repoman_settings['PORTAGE_QUIET'] = '1' if not portage.digestcheck([], repoman_settings, strict=1): - stats["manifest.bad"] += 1 - fails["manifest.bad"].append(os.path.join(xpkg, 'Manifest')) + qatracker.add_error("manifest.bad", os.path.join(xpkg, 'Manifest')) repoman_settings.pop('PORTAGE_QUIET', None) if options.mode == 'manifest-check': @@ -368,8 +363,7 @@ for xpkg in effective_scanlist: os.stat(os.path.join(checkdir, y)).st_mode) & 0o111 if file_is_executable: - stats["file.executable"] += 1 - fails["file.executable"].append(os.path.join(checkdir, y)) + qatracker.add_error("file.executable", os.path.join(checkdir, y)) if file_is_ebuild: pf = y[:-7] ebuildlist.append(pf) @@ -378,18 +372,15 @@ for xpkg in effective_scanlist: myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars))) except KeyError: allvalid = False - stats["ebuild.syntax"] += 1 - fails["ebuild.syntax"].append(os.path.join(xpkg, y)) + qatracker.add_error("ebuild.syntax", os.path.join(xpkg, y)) continue except IOError: allvalid = False - stats["ebuild.output"] += 1 - fails["ebuild.output"].append(os.path.join(xpkg, y)) + qatracker.add_error("ebuild.output", os.path.join(xpkg, y)) continue if not portage.eapi_is_supported(myaux["EAPI"]): allvalid = False - stats["EAPI.unsupported"] += 1 - fails["EAPI.unsupported"].append(os.path.join(xpkg, y)) + qatracker.add_error("EAPI.unsupported", os.path.join(xpkg, y)) continue pkgs[pf] = Package( cpv=cpv, metadata=myaux, root_config=root_config, @@ -421,8 +412,7 @@ for xpkg in effective_scanlist: # prohibited characters). See bug #406877. index = -1 if index != -1: - stats["file.name"] += 1 - fails["file.name"].append( + qatracker.add_error("file.name", "%s/%s: char '%s'" % (checkdir, y, y[index])) if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")): @@ -438,25 +428,21 @@ for xpkg in effective_scanlist: for l in f: line += 1 except UnicodeDecodeError as ue: - stats["file.UTF8"] += 1 s = ue.object[:ue.start] l2 = s.count("\n") line += l2 if l2 != 0: s = s[s.rfind("\n") + 1:] - fails["file.UTF8"].append( + qatracker.add_error("file.UTF8", "%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s)) finally: if f is not None: f.close() ############### - status_check = VCSStatus(vcs_settings, checkdir, checkdir_relative, xpkg) + status_check = VCSStatus(vcs_settings, checkdir, checkdir_relative, xpkg, qatracker) status_check.check(check_ebuild_notadded) eadded.extend(status_check.eadded) - for key in list(status_check.stats): - stats[key] += status_check.stats[key] - fails[key].extend(status_check.fails[key]) ############### mf = repoman_settings.repositories.get_repo_for_location( @@ -478,8 +464,7 @@ for xpkg in effective_scanlist: # This will be reported as an "ebuild.syntax" error. pass else: - stats["SRC_URI.syntax"] += 1 - fails["SRC_URI.syntax"].append( + qatracker.add_error("SRC_URI.syntax", "%s.ebuild SRC_URI: %s" % (mykey, e)) del fetchlist_dict if not src_uri_error: @@ -491,12 +476,10 @@ for xpkg in effective_scanlist: myfiles_all = set(myfiles_all) for entry in mydigests: if entry not in myfiles_all: - stats["digest.unused"] += 1 - fails["digest.unused"].append(checkdir + "::" + entry) + qatracker.add_error("digest.unused", checkdir + "::" + entry) for entry in myfiles_all: if entry not in mydigests: - stats["digest.missing"] += 1 - fails["digest.missing"].append(checkdir + "::" + entry) + qatracker.add_error("digest.missing", checkdir + "::" + entry) del myfiles_all if os.path.exists(checkdir + "/files"): @@ -529,12 +512,10 @@ for xpkg in effective_scanlist: # File size between 20 KiB and 60 KiB causes a warning, # while file size over 60 KiB causes an error. elif mystat.st_size > 61440: - stats["file.size.fatal"] += 1 - fails["file.size.fatal"].append( + qatracker.add_error("file.size.fatal", "(%d KiB) %s/files/%s" % (mystat.st_size // 1024, xpkg, y)) elif mystat.st_size > 20480: - stats["file.size"] += 1 - fails["file.size"].append( + qatracker.add_error("file.size", "(%d KiB) %s/files/%s" % (mystat.st_size // 1024, xpkg, y)) index = repo_settings.repo_config.find_invalid_path_char(y) @@ -547,20 +528,17 @@ for xpkg in effective_scanlist: # prohibited characters). See bug #406877. index = -1 if index != -1: - stats["file.name"] += 1 - fails["file.name"].append( + qatracker.add_error("file.name", "%s/files/%s: char '%s'" % (checkdir, y, y[index])) del mydigests if check_changelog and "ChangeLog" not in checkdirlist: - stats["changelog.missing"] += 1 - fails["changelog.missing"].append(xpkg + "/ChangeLog") + qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") musedict = {} # metadata.xml file check if "metadata.xml" not in checkdirlist: - stats["metadata.missing"] += 1 - fails["metadata.missing"].append(xpkg + "/metadata.xml") + qatracker.add_error("metadata.missing", xpkg + "/metadata.xml") # metadata.xml parse check else: metadata_bad = False @@ -576,8 +554,7 @@ for xpkg in effective_scanlist: parser=xml_parser) except (ExpatError, SyntaxError, EnvironmentError) as e: metadata_bad = True - stats["metadata.bad"] += 1 - fails["metadata.bad"].append("%s/metadata.xml: %s" % (xpkg, e)) + qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) del e else: if not hasattr(xml_parser, 'parser') or \ @@ -587,8 +564,7 @@ for xpkg in effective_scanlist: pass else: if "XML_DECLARATION" not in xml_info: - stats["metadata.bad"] += 1 - fails["metadata.bad"].append( + qatracker.add_error("metadata.bad", "%s/metadata.xml: " "xml declaration is missing on first line, " "should be '%s'" % (xpkg, metadata_xml_declaration)) @@ -597,38 +573,34 @@ for xpkg in effective_scanlist: xml_info["XML_DECLARATION"] if xml_encoding is None or \ xml_encoding.upper() != metadata_xml_encoding: - stats["metadata.bad"] += 1 if xml_encoding is None: encoding_problem = "but it is undefined" else: encoding_problem = "not '%s'" % xml_encoding - fails["metadata.bad"].append( + qatracker.add_error("metadata.bad", "%s/metadata.xml: " "xml declaration encoding should be '%s', %s" % (xpkg, metadata_xml_encoding, encoding_problem)) if "DOCTYPE" not in xml_info: metadata_bad = True - stats["metadata.bad"] += 1 - fails["metadata.bad"].append( + qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) else: doctype_name, doctype_system, doctype_pubid = \ xml_info["DOCTYPE"] if doctype_system != metadata_dtd_uri: - stats["metadata.bad"] += 1 if doctype_system is None: system_problem = "but it is undefined" else: system_problem = "not '%s'" % doctype_system - fails["metadata.bad"].append( + qatracker.add_error("metadata.bad", "%s/metadata.xml: " "DOCTYPE: SYSTEM should refer to '%s', %s" % (xpkg, metadata_dtd_uri, system_problem)) if doctype_name != metadata_doctype_name: - stats["metadata.bad"] += 1 - fails["metadata.bad"].append( + qatracker.add_error("metadata.bad", "%s/metadata.xml: " "DOCTYPE: name should be '%s', not '%s'" % (xpkg, metadata_doctype_name, doctype_name)) @@ -638,8 +610,8 @@ for xpkg in effective_scanlist: musedict = utilities.parse_metadata_use(_metadata_xml) except portage.exception.ParseError as e: metadata_bad = True - stats["metadata.bad"] += 1 - fails["metadata.bad"].append("%s/metadata.xml: %s" % (xpkg, e)) + qatracker.add_error("metadata.bad", + "%s/metadata.xml: %s" % (xpkg, e)) else: for atom in chain(*musedict.values()): if atom is None: @@ -647,13 +619,11 @@ for xpkg in effective_scanlist: try: atom = Atom(atom) except InvalidAtom as e: - stats["metadata.bad"] += 1 - fails["metadata.bad"].append( + qatracker.add_error("metadata.bad", "%s/metadata.xml: Invalid atom: %s" % (xpkg, e)) else: if atom.cp != xpkg: - stats["metadata.bad"] += 1 - fails["metadata.bad"].append( + qatracker.add_error("metadata.bad", "%s/metadata.xml: Atom contains " "unexpected cat/pn: %s" % (xpkg, atom)) @@ -662,16 +632,15 @@ for xpkg in effective_scanlist: utilities.check_metadata(_metadata_xml, herd_base) except (utilities.UnknownHerdsError, ) as e: metadata_bad = True - stats["metadata.bad"] += 1 - fails["metadata.bad"].append("%s/metadata.xml: %s" % (xpkg, e)) + qatracker.add_error("metadata.bad", + "%s/metadata.xml: %s" % (xpkg, e)) del e ################# # Only carry out if in package directory or check forced if not metadata_bad: if not xmllint.check(checkdir): - stats["metadata.bad"] += 1 - fails["metadata.bad"].append(xpkg + "/metadata.xml") + qatracker.add_error("metadata.bad", xpkg + "/metadata.xml") ################# del metadata_bad @@ -690,13 +659,12 @@ for xpkg in effective_scanlist: if check_changelog and not changelog_modified \ and ebuild.ebuild_path in changed.new_ebuilds: - stats['changelog.ebuildadded'] += 1 - fails['changelog.ebuildadded'].append(ebuild.relative_path) + qatracker.add_error('changelog.ebuildadded', ebuild.relative_path) if ebuild.untracked(check_ebuild_notadded, y_ebuild, eadded): # ebuild not added to vcs - stats["ebuild.notadded"] += 1 - fails["ebuild.notadded"].append(xpkg + "/" + y_ebuild + ".ebuild") + qatracker.add_error("ebuild.notadded", + xpkg + "/" + y_ebuild + ".ebuild") myesplit = portage.pkgsplit(y_ebuild) is_bad_split = myesplit is None or myesplit[0] != xpkg.split("/")[-1] @@ -706,13 +674,13 @@ for xpkg in effective_scanlist: is_pv_toolong2 = pv_toolong_re.search(myesplit[2]) if is_pv_toolong or is_pv_toolong2: - stats["ebuild.invalidname"] += 1 - fails["ebuild.invalidname"].append(xpkg + "/" + y_ebuild + ".ebuild") + qatracker.add_error("ebuild.invalidname", + xpkg + "/" + y_ebuild + ".ebuild") continue elif myesplit[0] != pkgdir: print(pkgdir, myesplit[0]) - stats["ebuild.namenomatch"] += 1 - fails["ebuild.namenomatch"].append(xpkg + "/" + y_ebuild + ".ebuild") + qatracker.add_error("ebuild.namenomatch", + xpkg + "/" + y_ebuild + ".ebuild") continue pkg = pkgs[y_ebuild] @@ -721,8 +689,8 @@ for xpkg in effective_scanlist: allvalid = False for k, msgs in pkg.invalid.items(): for msg in msgs: - stats[k] += 1 - fails[k].append("%s: %s" % (ebuild.relative_path, msg)) + qatracker.add_error(k, + "%s: %s" % (ebuild.relative_path, msg)) continue myaux = pkg._metadata @@ -731,13 +699,11 @@ for xpkg in effective_scanlist: live_ebuild = live_eclasses.intersection(inherited) if repo_settings.repo_config.eapi_is_banned(eapi): - stats["repo.eapi.banned"] += 1 - fails["repo.eapi.banned"].append( + qatracker.add_error("repo.eapi.banned", "%s: %s" % (ebuild.relative_path, eapi)) elif repo_settings.repo_config.eapi_is_deprecated(eapi): - stats["repo.eapi.deprecated"] += 1 - fails["repo.eapi.deprecated"].append( + qatracker.add_error("repo.eapi.deprecated", "%s: %s" % (ebuild.relative_path, eapi)) for k, v in myaux.items(): @@ -745,22 +711,18 @@ for xpkg in effective_scanlist: continue m = non_ascii_re.search(v) if m is not None: - stats["variable.invalidchar"] += 1 - fails["variable.invalidchar"].append( + qatracker.add_error("variable.invalidchar", "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) if not src_uri_error: ####################### - thirdparty = ThirdPartyMirrors(repoman_settings) + thirdparty = ThirdPartyMirrors(repoman_settings, qatracker) thirdparty.check(myaux, ebuild.relative_path) - stats["SRC_URI.mirror"] = thirdparty.stats - fails["SRC_URI.mirror"] = thirdparty.fails ####################### if myaux.get("PROVIDE"): - stats["virtual.oldstyle"] += 1 - fails["virtual.oldstyle"].append(ebuild.relative_path) + qatracker.add_error("virtual.oldstyle", ebuild.relative_path) for pos, missing_var in enumerate(missingvars): if not myaux.get(missing_var): @@ -770,26 +732,22 @@ for xpkg in effective_scanlist: if live_ebuild and missing_var == "KEYWORDS": continue myqakey = missingvars[pos] + ".missing" - stats[myqakey] += 1 - fails[myqakey].append(xpkg + "/" + y_ebuild + ".ebuild") + qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") if catdir == "virtual": for var in ("HOMEPAGE", "LICENSE"): if myaux.get(var): myqakey = var + ".virtual" - stats[myqakey] += 1 - fails[myqakey].append(ebuild.relative_path) + qatracker.add_error(myqakey, ebuild.relative_path) if myaux['DESCRIPTION'][-1:] in ['.']: - stats['DESCRIPTION.punctuation'] += 1 - fails['DESCRIPTION.punctuation'].append( + qatracker.add_error('DESCRIPTION.punctuation', "%s: DESCRIPTION ends with a '%s' character" % (ebuild.relative_path, myaux['DESCRIPTION'][-1:])) # 14 is the length of DESCRIPTION="" if len(myaux['DESCRIPTION']) > max_desc_len: - stats['DESCRIPTION.toolong'] += 1 - fails['DESCRIPTION.toolong'].append( + qatracker.add_error('DESCRIPTION.toolong', "%s: DESCRIPTION is %d characters (max %d)" % (ebuild.relative_path, len(myaux['DESCRIPTION']), max_desc_len)) @@ -802,8 +760,7 @@ for xpkg in effective_scanlist: if stable_keywords: if ebuild.ebuild_path in changed.new_ebuilds and catdir != "virtual": stable_keywords.sort() - stats["KEYWORDS.stable"] += 1 - fails["KEYWORDS.stable"].append( + qatracker.add_error("KEYWORDS.stable", "%s/%s.ebuild added with stable keywords: %s" % (xpkg, y_ebuild, " ".join(stable_keywords))) @@ -816,8 +773,7 @@ for xpkg in effective_scanlist: elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild: dropped_keywords = previous_keywords.difference(ebuild_archs) if dropped_keywords: - stats["KEYWORDS.dropped"] += 1 - fails["KEYWORDS.dropped"].append( + qatracker.add_error("KEYWORDS.dropped", "%s: %s" % (ebuild.relative_path, " ".join(sorted(dropped_keywords)))) @@ -833,8 +789,8 @@ for xpkg in effective_scanlist: if kw in kwlist: haskeyword = True if not haskeyword: - stats["KEYWORDS.stupid"] += 1 - fails["KEYWORDS.stupid"].append(xpkg + "/" + y_ebuild + ".ebuild") + qatracker.add_error("KEYWORDS.stupid", + xpkg + "/" + y_ebuild + ".ebuild") """ Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should @@ -848,15 +804,13 @@ for xpkg in effective_scanlist: bad_stable_keywords.append(keyword) del keyword if bad_stable_keywords: - stats["LIVEVCS.stable"] += 1 - fails["LIVEVCS.stable"].append( + qatracker.add_error("LIVEVCS.stable", "%s/%s.ebuild with stable keywords:%s " % (xpkg, y_ebuild, bad_stable_keywords)) del bad_stable_keywords if keywords and not has_global_mask(pkg): - stats["LIVEVCS.unmasked"] += 1 - fails["LIVEVCS.unmasked"].append(ebuild.relative_path) + qatracker.add_error("LIVEVCS.unmasked", ebuild.relative_path) if options.ignore_arches: arches = [[ @@ -923,8 +877,7 @@ for xpkg in effective_scanlist: if atoms and mytype.endswith("DEPEND"): if runtime and \ "test?" in mydepstr.split(): - stats[mytype + '.suspect'] += 1 - fails[mytype + '.suspect'].append( + qatracker.add_error(mytype + '.suspect', "%s: 'test?' USE conditional in %s" % (ebuild.relative_path, mytype)) @@ -945,8 +898,7 @@ for xpkg in effective_scanlist: if catdir != "virtual": if not is_blocker and \ atom.cp in suspect_virtual: - stats['virtual.suspect'] += 1 - fails['virtual.suspect'].append( + qatracker.add_error('virtual.suspect', ebuild.relative_path + ": %s: consider using '%s' instead of '%s'" % (mytype, suspect_virtual[atom.cp], atom)) @@ -955,28 +907,25 @@ for xpkg in effective_scanlist: not is_blocker and \ not inherited_java_eclass and \ atom.cp == "virtual/jdk": - stats['java.eclassesnotused'] += 1 - fails['java.eclassesnotused'].append(ebuild.relative_path) + qatracker.add_error('java.eclassesnotused', + ebuild.relative_path) elif buildtime and \ not is_blocker and \ not inherited_wxwidgets_eclass and \ atom.cp == "x11-libs/wxGTK": - stats['wxwidgets.eclassnotused'] += 1 - fails['wxwidgets.eclassnotused'].append( + qatracker.add_error('wxwidgets.eclassnotused', "%s: %ss on x11-libs/wxGTK without inheriting" " wxwidgets.eclass" % (ebuild.relative_path, mytype)) elif runtime: if not is_blocker and \ atom.cp in suspect_rdepend: - stats[mytype + '.suspect'] += 1 - fails[mytype + '.suspect'].append( + qatracker.add_error(mytype + '.suspect', ebuild.relative_path + ": '%s'" % atom) if atom.operator == "~" and \ portage.versions.catpkgsplit(atom.cpv)[3] != "r0": qacat = 'dependency.badtilde' - stats[qacat] += 1 - fails[qacat].append( + qatracker.add_error(qacat, "%s: %s uses the ~ operator" " with a non-zero revision: '%s'" % (ebuild.relative_path, mytype, atom)) @@ -988,8 +937,8 @@ for xpkg in effective_scanlist: qacat = "dependency.syntax" else: qacat = m + ".syntax" - stats[qacat] += 1 - fails[qacat].append("%s: %s: %s" % (ebuild.relative_path, m, b)) + qatracker.add_error(qacat, + "%s: %s: %s" % (ebuild.relative_path, m, b)) badlicsyntax = len([z for z in type_list if z == "LICENSE"]) badprovsyntax = len([z for z in type_list if z == "PROVIDE"]) @@ -1015,15 +964,14 @@ for xpkg in effective_scanlist: if default_use and not eapi_has_iuse_defaults(eapi): for myflag in default_use: - stats['EAPI.incompatible'] += 1 - fails['EAPI.incompatible'].append( + qatracker.add_error('EAPI.incompatible', "%s: IUSE defaults" " not supported with EAPI='%s': '%s'" % (ebuild.relative_path, eapi, myflag)) for mypos in range(len(myuse)): - stats["IUSE.invalid"] += 1 - fails["IUSE.invalid"].append(xpkg + "/" + y_ebuild + ".ebuild: %s" % myuse[mypos]) + qatracker.add_error("IUSE.invalid", + xpkg + "/" + y_ebuild + ".ebuild: %s" % myuse[mypos]) # Check for outdated RUBY targets old_ruby_eclasses = ["ruby-ng", "ruby-fakegem", "ruby"] @@ -1033,9 +981,9 @@ for xpkg in effective_scanlist: ruby_intersection = pkg.iuse.all.intersection(ruby_deprecated) if ruby_intersection: for myruby in ruby_intersection: - stats["IUSE.rubydeprecated"] += 1 - fails["IUSE.rubydeprecated"].append( - (ebuild.relative_path + ": Deprecated ruby target: %s") % myruby) + qatracker.add_error("IUSE.rubydeprecated", + (ebuild.relative_path + ": Deprecated ruby target: %s") + % myruby) # license checks if not badlicsyntax: @@ -1048,11 +996,11 @@ for xpkg in effective_scanlist: # Need to check for "||" manually as no portage # function will remove it without removing values. if lic not in liclist and lic != "||": - stats["LICENSE.invalid"] += 1 - fails["LICENSE.invalid"].append(xpkg + "/" + y_ebuild + ".ebuild: %s" % lic) + qatracker.add_error("LICENSE.invalid", + xpkg + "/" + y_ebuild + ".ebuild: %s" % lic) elif lic in liclist_deprecated: - stats["LICENSE.deprecated"] += 1 - fails["LICENSE.deprecated"].append("%s: %s" % (ebuild.relative_path, lic)) + qatracker.add_error("LICENSE.deprecated", + "%s: %s" % (ebuild.relative_path, lic)) # keyword checks myuse = myaux["KEYWORDS"].split() @@ -1064,13 +1012,12 @@ for xpkg in effective_scanlist: if myskey[:1] == "~": myskey = myskey[1:] if myskey not in kwlist: - stats["KEYWORDS.invalid"] += 1 - fails["KEYWORDS.invalid"].append( + qatracker.add_error("KEYWORDS.invalid", "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mykey)) elif myskey not in profiles: - stats["KEYWORDS.invalid"] += 1 - fails["KEYWORDS.invalid"].append( - "%s/%s.ebuild: %s (profile invalid)" % (xpkg, y_ebuild, mykey)) + qatracker.add_error("KEYWORDS.invalid", + "%s/%s.ebuild: %s (profile invalid)" + % (xpkg, y_ebuild, mykey)) # restrict checks myrestrict = None @@ -1078,31 +1025,29 @@ for xpkg in effective_scanlist: myrestrict = portage.dep.use_reduce( myaux["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - stats["RESTRICT.syntax"] += 1 - fails["RESTRICT.syntax"].append( + qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e if myrestrict: myrestrict = set(myrestrict) mybadrestrict = myrestrict.difference(valid_restrict) if mybadrestrict: - stats["RESTRICT.invalid"] += len(mybadrestrict) for mybad in mybadrestrict: - fails["RESTRICT.invalid"].append(xpkg + "/" + y_ebuild + ".ebuild: %s" % mybad) + qatracker.add_error("RESTRICT.invalid", + xpkg + "/" + y_ebuild + ".ebuild: %s" % mybad) # REQUIRED_USE check required_use = myaux["REQUIRED_USE"] if required_use: if not eapi_has_required_use(eapi): - stats['EAPI.incompatible'] += 1 - fails['EAPI.incompatible'].append( + qatracker.add_error('EAPI.incompatible', "%s: REQUIRED_USE" - " not supported with EAPI='%s'" % (ebuild.relative_path, eapi,)) + " not supported with EAPI='%s'" + % (ebuild.relative_path, eapi,)) try: portage.dep.check_required_use( required_use, (), pkg.iuse.is_valid_flag, eapi=eapi) except portage.exception.InvalidDependString as e: - stats["REQUIRED_USE.syntax"] += 1 - fails["REQUIRED_USE.syntax"].append( + qatracker.add_error("REQUIRED_USE.syntax", "%s: REQUIRED_USE: %s" % (ebuild.relative_path, e)) del e @@ -1120,8 +1065,8 @@ for xpkg in effective_scanlist: mode='r', encoding=_encodings['repo.content']) try: for check_name, e in run_checks(f, pkg): - stats[check_name] += 1 - fails[check_name].append(ebuild.relative_path + ': %s' % e) + qatracker.add_error(check_name, + ebuild.relative_path + ': %s' % e) finally: f.close() except UnicodeDecodeError: @@ -1278,16 +1223,14 @@ for xpkg in effective_scanlist: # if we emptied out our list, continue: if not atoms: continue - stats[mykey] += 1 - fails[mykey].append( - "%s: %s: %s(%s) %s" % ( - ebuild.relative_path, mytype, keyword, prof, + qatracker.add_error(mykey, + "%s: %s: %s(%s) %s" + % (ebuild.relative_path, mytype, keyword, prof, repr(atoms))) else: - stats[mykey] += 1 - fails[mykey].append( - "%s: %s: %s(%s) %s" % ( - ebuild.relative_path, mytype, keyword, prof, + qatracker.add_error(mykey, + "%s: %s: %s(%s) %s" + % (ebuild.relative_path, mytype, keyword, prof, repr(atoms))) if not baddepsyntax and unknown_pkgs: @@ -1295,19 +1238,17 @@ for xpkg in effective_scanlist: for mytype, atom in unknown_pkgs: type_map.setdefault(mytype, set()).add(atom) for mytype, atoms in type_map.items(): - stats["dependency.unknown"] += 1 - fails["dependency.unknown"].append( - "%s: %s: %s" % ( - ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) + qatracker.add_error("dependency.unknown", + "%s: %s: %s" + % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) # check if there are unused local USE-descriptions in metadata.xml # (unless there are any invalids, to avoid noise) if allvalid: for myflag in muselist.difference(used_useflags): - stats["metadata.warning"] += 1 - fails["metadata.warning"].append( - "%s/metadata.xml: unused local USE-description: '%s'" % - (xpkg, myflag)) + qatracker.add_error("metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) if options.if_modified == "y" and len(effective_scanlist) < 1: logging.warn("--if-modified is enabled, but no modified packages were found!") @@ -1323,7 +1264,7 @@ dowarn = 0 dofull = options.mode != 'full' for x in qacats: - if not stats[x]: + if x not in qatracker.fails: continue dowarn = 1 if x not in qawarnings: @@ -1354,7 +1295,7 @@ format_outputs = { format_output = format_outputs.get( options.output_style, format_outputs['default']) -format_output(f, stats, fails, dofull, dofail, options, qawarnings) +format_output(f, qatracker.fails, dofull, dofail, options, qawarnings) style_file.flush() del console_writer, f, style_file diff --git a/pym/repoman/qa_data.py b/pym/repoman/qa_data.py index 2929762..d7f0ca9 100644 --- a/pym/repoman/qa_data.py +++ b/pym/repoman/qa_data.py @@ -345,12 +345,11 @@ no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"]) def format_qa_output( - formatter, stats, fails, dofull, dofail, options, qawarnings): + formatter, fails, dofull, dofail, options, qawarnings): """Helper function that formats output properly Args: formatter - a subclass of Formatter - stats - a dict of qa status items fails - a dict of qa status failures dofull - boolean to print full results or a summary dofail - boolean to decide if failure was hard or soft @@ -360,8 +359,8 @@ def format_qa_output( """ full = options.mode == 'full' # we only want key value pairs where value > 0 - for category, number in \ - filter(lambda myitem: myitem[1] > 0, sorted(stats.items())): + for category in sorted(fails): + number = len(fails[category]) formatter.add_literal_data(" " + category.ljust(30)) if category in qawarnings: formatter.push_style("WARN") @@ -383,7 +382,7 @@ def format_qa_output( def format_qa_output_column( - formatter, stats, fails, dofull, dofail, options, qawarnings): + formatter, fails, dofull, dofail, options, qawarnings): """Helper function that formats output in a machine-parseable column format @param formatter: an instance of Formatter @@ -403,11 +402,8 @@ def format_qa_output_column( @return: None (modifies formatter) """ full = options.mode == 'full' - for category, number in stats.items(): - # we only want key value pairs where value > 0 - if number < 1: - continue - + for category in fails.items(): + number = len(fails[category]) formatter.add_literal_data("NumberOf " + category + " ") if category in qawarnings: formatter.push_style("WARN") diff --git a/pym/repoman/qa_tracker.py b/pym/repoman/qa_tracker.py new file mode 100644 index 0000000..48a7241 --- /dev/null +++ b/pym/repoman/qa_tracker.py @@ -0,0 +1,46 @@ + +import logging +import sys + +from repoman.qa_data import qacats, qawarnings + + +class QATracker(object): + '''Track all occurrances of Q/A problems detected''' + + def __init__(self): + self.fails = {} + self.warns = {} + + + def add_error(self, detected_qa, info): + '''Add the Q/A error to the database of detected problems + + @param detected_qa: string, member of qa_data.qacats list + @param info: string, details of the detected problem + ''' + if detected_qa not in qacats: + logging.error('QATracker: Exiting on error. unknown detected_qa type passed in ' + 'to add_error(): %s, %s' % (detected_qa, info)) + sys.exit(1) + try: + self.fails[detected_qa].append(info) + except KeyError: + self.fails[detected_qa] = [info] + + + def add_warning(self, detected_qa, info): + '''Add the Q/A warning to the database of detected problems + + @param detected_qa: string, member of qa_data.qawarnings list + @param info: string, details of the detected problem + ''' + if detected_qa not in qawarnings: + logging.error('QATracker: Exiting on error. unknown detected_qa type passed in ' + 'to add_warning(): %s, %s' % (detected_qa, info)) + sys.exit(1) + try: + self.warns[detected_qa].append(info) + except KeyError: + self.warns[detected_qa] = [info] + diff --git a/pym/repoman/vcs/vcsstatus.py b/pym/repoman/vcs/vcsstatus.py index 346c20e..eedf866 100644 --- a/pym/repoman/vcs/vcsstatus.py +++ b/pym/repoman/vcs/vcsstatus.py @@ -11,15 +11,14 @@ class VCSStatus(object): '''Determines the status of the vcs repositories to determine if files are not added''' - def __init__(self, vcs_settings, checkdir, checkdir_relative, xpkg): + def __init__(self, vcs_settings, checkdir, checkdir_relative, xpkg, qatracker): self.vcs_settings = vcs_settings self.vcs = vcs_settings.vcs self.eadded = [] self.checkdir = checkdir self.checkdir_relative = checkdir_relative self.xpkg = xpkg - self.stats = {} - self.fails = {} + self.qatracker = qatracker def check(self, check_not_added): @@ -31,14 +30,8 @@ class VCSStatus(object): def post_git_hg(self, myf): for l in myf: if l[:-1][-7:] == ".ebuild": - if "ebuild.notadded" in list(self.fails): - self.stats["ebuild.notadded"] += 1 - self.fails["ebuild.notadded"].append( + self.qatracker.add_error("ebuild.notadded", os.path.join(self.xpkg, os.path.basename(l[:-1]))) - else: - self.stats["ebuild.notadded"] = 1 - self.fails["ebuild.notadded"] = [os.path.join( - self.xpkg, os.path.basename(l[:-1]))] myf.close() @@ -62,14 +55,8 @@ class VCSStatus(object): myl = myf.readlines() myf.close() except IOError: - if "CVS/Entries.IO_error" in list(self.fails): - self.stats["CVS/Entries.IO_error"] += 1 - self.fails["CVS/Entries.IO_error"].append( - self.checkdir + "/CVS/Entries") - else: - self.stats["CVS/Entries.IO_error"] = 1 - self.fails["CVS/Entries.IO_error"] = [ - self.checkdir + "/CVS/Entries"] + self.qatracker.add_error("CVS/Entries.IO_error", + self.checkdir + "/CVS/Entries") return True for l in myl: if l[0] != "/":