From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 3E1051393E9 for ; Mon, 2 Jun 2014 06:05:06 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 5A069E0998; Mon, 2 Jun 2014 06:05:05 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 932C5E0986 for ; Mon, 2 Jun 2014 06:05:04 +0000 (UTC) Received: from spoonbill.gentoo.org (spoonbill.gentoo.org [81.93.255.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 7861433FF71 for ; Mon, 2 Jun 2014 06:05:03 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by spoonbill.gentoo.org (Postfix) with ESMTP id 47B5E181A9 for ; Mon, 2 Jun 2014 06:05:01 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1401687105.fc8b6617dfc742e683af929de7ad6d8ab70d9dc6.dol-sen@gentoo> Subject: [gentoo-commits] proj/portage:repoman commit in: pym/repoman/, pym/repoman/checks/ebuilds/ X-VCS-Repository: proj/portage X-VCS-Files: pym/repoman/checks/ebuilds/fetches.py pym/repoman/main.py X-VCS-Directories: pym/repoman/ pym/repoman/checks/ebuilds/ X-VCS-Committer: dol-sen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: fc8b6617dfc742e683af929de7ad6d8ab70d9dc6 X-VCS-Branch: repoman Date: Mon, 2 Jun 2014 06:05:01 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 85523bc1-ab5b-40d4-a8d2-d6dff28c808b X-Archives-Hash: 7588f1ee1892e9bbe5a017659de51b0a commit: fc8b6617dfc742e683af929de7ad6d8ab70d9dc6 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jun 2 05:31:45 2014 +0000 Commit: Brian Dolbec gmail com> CommitDate: Mon Jun 2 05:31:45 2014 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fc8b6617 repoman/main.py: Create FetchChecks class Create the new class in checks/ebuilds/fetches.py. --- pym/repoman/checks/ebuilds/fetches.py | 132 ++++++++++++++++++++++++++++++++++ pym/repoman/main.py | 96 ++----------------------- 2 files changed, 139 insertions(+), 89 deletions(-) diff --git a/pym/repoman/checks/ebuilds/fetches.py b/pym/repoman/checks/ebuilds/fetches.py new file mode 100644 index 0000000..3d59339 --- /dev/null +++ b/pym/repoman/checks/ebuilds/fetches.py @@ -0,0 +1,132 @@ + +'''fetches.py +Performs the src_uri fetchlist and files checks +''' + +from stat import S_ISDIR + +import portage +from portage import os + + +class FetchChecks(object): + '''Performs checks on the files needed for the ebuild''' + + def __init__(self, qatracker, repoman_settings, repo_settings, portdb, + vcs_settings, vcs_new_changed): + ''' + @param qatracker: QATracker instance + @param repoman_settings: settings instance + @param repo_settings: repository settings instance + @param portdb: portdb instance + ''' + self.portdb = portdb + self.qatracker = qatracker + self.repo_settings = repo_settings + self.repoman_settings = repoman_settings + self.vcs_settings = vcs_settings + self.vcs_new_changed = vcs_new_changed + self._digests = None + + + def check(self, xpkg, checkdir, checkdir_relative): + '''Checks the ebuild sources and files for errors + + @param xpkg: the pacakge being checked + @param checkdir: string, directory path + @param checkdir_relative: repolevel determined path + ''' + self.checkdir = checkdir + fetchlist_dict = portage.FetchlistDict(checkdir, self.repoman_settings, self.portdb) + myfiles_all = [] + self.src_uri_error = False + for mykey in fetchlist_dict: + try: + myfiles_all.extend(fetchlist_dict[mykey]) + except portage.exception.InvalidDependString as e: + self.src_uri_error = True + try: + self.portdb.aux_get(mykey, ["SRC_URI"]) + except KeyError: + # This will be reported as an "ebuild.syntax" error. + pass + else: + self.qatracker.add_error("SRC_URI.syntax", + "%s.ebuild SRC_URI: %s" % (mykey, e)) + del fetchlist_dict + if not self.src_uri_error: + # This test can produce false positives if SRC_URI could not + # be parsed for one or more ebuilds. There's no point in + # producing a false error here since the root cause will + # produce a valid error elsewhere, such as "SRC_URI.syntax" + # or "ebuild.sytax". + myfiles_all = set(myfiles_all) + for entry in self.digests: + if entry not in myfiles_all: + self.qatracker.add_error("digest.unused", checkdir + "::" + entry) + for entry in myfiles_all: + if entry not in self.digests: + self.qatracker.add_error("digest.missing", checkdir + "::" + entry) + del myfiles_all + + if os.path.exists(checkdir + "/files"): + filesdirlist = os.listdir(checkdir + "/files") + + # Recurse through files directory, use filesdirlist as a stack; + # appending directories as needed, + # so people can't hide > 20k files in a subdirectory. + while filesdirlist: + y = filesdirlist.pop(0) + relative_path = os.path.join(xpkg, "files", y) + full_path = os.path.join(self.repo_settings.repodir, relative_path) + try: + mystat = os.stat(full_path) + except OSError as oe: + if oe.errno == 2: + # don't worry about it. it likely was removed via fix above. + continue + else: + raise oe + if S_ISDIR(mystat.st_mode): + # !!! VCS "portability" alert! Need some function isVcsDir() or alike !!! + if y == "CVS" or y == ".svn": + continue + for z in os.listdir(checkdir + "/files/" + y): + if z == "CVS" or z == ".svn": + continue + filesdirlist.append(y + "/" + z) + # Current policy is no files over 20 KiB, these are the checks. + # File size between 20 KiB and 60 KiB causes a warning, + # while file size over 60 KiB causes an error. + elif mystat.st_size > 61440: + self.qatracker.add_error("file.size.fatal", + "(%d KiB) %s/files/%s" % (mystat.st_size // 1024, xpkg, y)) + elif mystat.st_size > 20480: + self.qatracker.add_error("file.size", + "(%d KiB) %s/files/%s" % (mystat.st_size // 1024, xpkg, y)) + + index = self.repo_settings.repo_config.find_invalid_path_char(y) + if index != -1: + y_relative = os.path.join(checkdir_relative, "files", y) + if self.vcs_settings.vcs is not None and not self.vcs_new_changed(y_relative): + # If the file isn't in the VCS new or changed set, then + # assume that it's an irrelevant temporary file (Manifest + # entries are not generated for file names containing + # prohibited characters). See bug #406877. + index = -1 + if index != -1: + self.qatracker.add_error("file.name", + "%s/files/%s: char '%s'" % (checkdir, y, y[index])) + + + @property + def digests(self): + '''Property function, returns the saved digests or + loads them for the test''' + if not self._digests: + mf = self.repoman_settings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(self.checkdir))) + mf = mf.load_manifest(self.checkdir, self.repoman_settings["DISTDIR"]) + self._digests = mf.getTypeDigests("DIST") + del mf + return self._digests diff --git a/pym/repoman/main.py b/pym/repoman/main.py index d80cf59..ffb9929 100755 --- a/pym/repoman/main.py +++ b/pym/repoman/main.py @@ -16,7 +16,6 @@ import sys import tempfile import platform from itertools import chain -from stat import S_ISDIR from os import path as osp pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__)))) #, "pym") @@ -47,6 +46,7 @@ from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use from repoman.argparser import parse_args from repoman.checks.ebuilds.checks import run_checks, checks_init +from repoman.checks.ebuilds.fetches import FetchChecks from repoman.checks.ebuilds.isebuild import IsEbuild from repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors from repoman.checks.ebuilds.manifests import Manifests @@ -369,93 +369,11 @@ for xpkg in effective_scanlist: status_check.check(check_ebuild_notadded) eadded.extend(status_check.eadded) ############### - - mf = repoman_settings.repositories.get_repo_for_location( - os.path.dirname(os.path.dirname(checkdir))) - mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"]) - mydigests = mf.getTypeDigests("DIST") - - fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb) - myfiles_all = [] - src_uri_error = False - for mykey in fetchlist_dict: - try: - myfiles_all.extend(fetchlist_dict[mykey]) - except portage.exception.InvalidDependString as e: - src_uri_error = True - try: - portdb.aux_get(mykey, ["SRC_URI"]) - except KeyError: - # This will be reported as an "ebuild.syntax" error. - pass - else: - qatracker.add_error("SRC_URI.syntax", - "%s.ebuild SRC_URI: %s" % (mykey, e)) - del fetchlist_dict - if not src_uri_error: - # This test can produce false positives if SRC_URI could not - # be parsed for one or more ebuilds. There's no point in - # producing a false error here since the root cause will - # produce a valid error elsewhere, such as "SRC_URI.syntax" - # or "ebuild.sytax". - myfiles_all = set(myfiles_all) - for entry in mydigests: - if entry not in myfiles_all: - qatracker.add_error("digest.unused", checkdir + "::" + entry) - for entry in myfiles_all: - if entry not in mydigests: - qatracker.add_error("digest.missing", checkdir + "::" + entry) - del myfiles_all - - if os.path.exists(checkdir + "/files"): - filesdirlist = os.listdir(checkdir + "/files") - - # Recurse through files directory, use filesdirlist as a stack; - # appending directories as needed, - # so people can't hide > 20k files in a subdirectory. - while filesdirlist: - y = filesdirlist.pop(0) - relative_path = os.path.join(xpkg, "files", y) - full_path = os.path.join(repo_settings.repodir, relative_path) - try: - mystat = os.stat(full_path) - except OSError as oe: - if oe.errno == 2: - # don't worry about it. it likely was removed via fix above. - continue - else: - raise oe - if S_ISDIR(mystat.st_mode): - # !!! VCS "portability" alert! Need some function isVcsDir() or alike !!! - if y == "CVS" or y == ".svn": - continue - for z in os.listdir(checkdir + "/files/" + y): - if z == "CVS" or z == ".svn": - continue - filesdirlist.append(y + "/" + z) - # Current policy is no files over 20 KiB, these are the checks. - # File size between 20 KiB and 60 KiB causes a warning, - # while file size over 60 KiB causes an error. - elif mystat.st_size > 61440: - qatracker.add_error("file.size.fatal", - "(%d KiB) %s/files/%s" % (mystat.st_size // 1024, xpkg, y)) - elif mystat.st_size > 20480: - qatracker.add_error("file.size", - "(%d KiB) %s/files/%s" % (mystat.st_size // 1024, xpkg, y)) - - index = repo_settings.repo_config.find_invalid_path_char(y) - if index != -1: - y_relative = os.path.join(checkdir_relative, "files", y) - if vcs_settings.vcs is not None and not vcs_new_changed(y_relative): - # If the file isn't in the VCS new or changed set, then - # assume that it's an irrelevant temporary file (Manifest - # entries are not generated for file names containing - # prohibited characters). See bug #406877. - index = -1 - if index != -1: - qatracker.add_error("file.name", - "%s/files/%s: char '%s'" % (checkdir, y, y[index])) - del mydigests +################# + fetchcheck = FetchChecks(qatracker, repoman_settings, repo_settings, portdb, + vcs_settings, vcs_new_changed) + fetchcheck.check(xpkg, checkdir, checkdir_relative) +################# if check_changelog and "ChangeLog" not in checkdirlist: qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") @@ -535,7 +453,7 @@ for xpkg in effective_scanlist: "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) - if not src_uri_error: + if not fetchcheck.src_uri_error: ####################### thirdparty = ThirdPartyMirrors(repoman_settings, qatracker) thirdparty.check(myaux, ebuild.relative_path)