From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 76B461384A7 for ; Tue, 28 Jul 2015 23:20:55 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id B6F131404F; Tue, 28 Jul 2015 23:20:53 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 49B761404F for ; Tue, 28 Jul 2015 23:20:53 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id D7C57340845 for ; Tue, 28 Jul 2015 23:20:51 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 04097AA for ; Tue, 28 Jul 2015 23:20:49 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1438125604.2bdca5ed5769380d7f839aa521f95afb51d7dbed.zorry@gentoo> Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/repoman/, tbc/pym/ X-VCS-Repository: proj/tinderbox-cluster X-VCS-Files: tbc/pym/repoman/main.py tbc/pym/updatedb.py X-VCS-Directories: tbc/pym/ tbc/pym/repoman/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 2bdca5ed5769380d7f839aa521f95afb51d7dbed X-VCS-Branch: master Date: Tue, 28 Jul 2015 23:20:49 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 2df71406-3e8d-4341-9bc3-d36d57fc7d31 X-Archives-Hash: ceb201a584dbe99c400c4f8c421318ed commit: 2bdca5ed5769380d7f839aa521f95afb51d7dbed Author: Magnus Granberg gentoo org> AuthorDate: Tue Jul 28 23:20:04 2015 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Tue Jul 28 23:20:04 2015 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2bdca5ed add repoman_main() tbc/pym/repoman/main.py | 2832 +++++++++++++++++++++++------------------------ tbc/pym/updatedb.py | 3 +- 2 files changed, 1418 insertions(+), 1417 deletions(-) diff --git a/tbc/pym/repoman/main.py b/tbc/pym/repoman/main.py index b346b66..c49bd2c 100755 --- a/tbc/pym/repoman/main.py +++ b/tbc/pym/repoman/main.py @@ -94,1543 +94,1543 @@ non_ascii_re = re.compile(r'[^\x00-\x7f]') # A sane umask is needed for files that portage creates. os.umask(0o22) -# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to -# behave incrementally. -repoman_incrementals = tuple( - x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS') -config_root = os.environ.get("PORTAGE_CONFIGROOT") -repoman_settings = portage.config(config_root=config_root, local_config=False) - -if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \ - repoman_settings.get('TERM') == 'dumb' or \ - not sys.stdout.isatty(): - nocolor() - -options, arguments = parse_args( - sys.argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", "")) - -if options.version: - print("Portage", portage.VERSION) - sys.exit(0) -if options.experimental_inherit == 'y': - # This is experimental, so it's non-fatal. - qawarnings.add("inherit.missing") - checks_init(experimental_inherit=True) +def sort_key(item): + return item[2].sub_path + +# Setup the GPG commands +def gpgsign(filename): + gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND") + if gpgcmd is None: + raise MissingParameter( + "PORTAGE_GPG_SIGNING_COMMAND is unset!" + " Is make.globals missing?") + if "${PORTAGE_GPG_KEY}" in gpgcmd and \ + "PORTAGE_GPG_KEY" not in repoman_settings: + raise MissingParameter("PORTAGE_GPG_KEY is unset!") + if "${PORTAGE_GPG_DIR}" in gpgcmd: + if "PORTAGE_GPG_DIR" not in repoman_settings: + repoman_settings["PORTAGE_GPG_DIR"] = \ + os.path.expanduser("~/.gnupg") + logging.info( + "Automatically setting PORTAGE_GPG_DIR to '%s'" % + repoman_settings["PORTAGE_GPG_DIR"]) + else: + repoman_settings["PORTAGE_GPG_DIR"] = \ + os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"]) + if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK): + raise portage.exception.InvalidLocation( + "Unable to access directory: PORTAGE_GPG_DIR='%s'" % + repoman_settings["PORTAGE_GPG_DIR"]) + gpgvars = {"FILE": filename} + for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"): + v = repoman_settings.get(k) + if v is not None: + gpgvars[k] = v + gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars) + if options.pretend: + print("(" + gpgcmd + ")") + else: + # Encode unicode manually for bug #310789. + gpgcmd = portage.util.shlex_split(gpgcmd) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(gpgcmd[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(gpgcmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(gpgcmd[0]) + gpgcmd[0] = fullname + + gpgcmd = [ + _unicode_encode(arg, encoding=_encodings['fs'], errors='strict') + for arg in gpgcmd] + rValue = subprocess.call(gpgcmd) + if rValue == os.EX_OK: + os.rename(filename + ".asc", filename) + else: + raise portage.exception.PortageException( + "!!! gpg exited with '" + str(rValue) + "' status") + +def need_signature(filename): + try: + with open( + _unicode_encode( + filename, encoding=_encodings['fs'], errors='strict'), + 'rb') as f: + return b"BEGIN PGP SIGNED MESSAGE" not in f.readline() + except IOError as e: + if e.errno in (errno.ENOENT, errno.ESTALE): + return False + raise + +def repoman_main(argv): + # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to + # behave incrementally. + repoman_incrementals = tuple( + x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS') + config_root = os.environ.get("PORTAGE_CONFIGROOT") + repoman_settings = portage.config(config_root=config_root, local_config=False) + + if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \ + repoman_settings.get('TERM') == 'dumb' or \ + not sys.stdout.isatty(): + nocolor() + + options, arguments = parse_args( + argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", "")) + + if options.version: + print("Portage", portage.VERSION) + sys.exit(0) + + if options.experimental_inherit == 'y': + # This is experimental, so it's non-fatal. + qawarnings.add("inherit.missing") + checks_init(experimental_inherit=True) + + # Set this to False when an extraordinary issue (generally + # something other than a QA issue) makes it impossible to + # commit (like if Manifest generation fails). + can_force = True + + portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings) + if portdir is None: + sys.exit(1) -# Set this to False when an extraordinary issue (generally -# something other than a QA issue) makes it impossible to -# commit (like if Manifest generation fails). -can_force = True + myreporoot = os.path.basename(portdir_overlay) + myreporoot += mydir[len(portdir_overlay):] + ################## -portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings) -if portdir is None: - sys.exit(1) + vcs_settings = VCSSettings(options, repoman_settings) -myreporoot = os.path.basename(portdir_overlay) -myreporoot += mydir[len(portdir_overlay):] -################## + ################## -vcs_settings = VCSSettings(options, repoman_settings) + repo_settings = RepoSettings( + config_root, portdir, portdir_overlay, + repoman_settings, vcs_settings, options, qawarnings) + repoman_settings = repo_settings.repoman_settings -################## + portdb = repo_settings.portdb + ################## -repo_settings = RepoSettings( - config_root, portdir, portdir_overlay, - repoman_settings, vcs_settings, options, qawarnings) + if options.echangelog is None and repo_settings.repo_config.update_changelog: + options.echangelog = 'y' -repoman_settings = repo_settings.repoman_settings + if vcs_settings.vcs is None: + options.echangelog = 'n' -portdb = repo_settings.portdb -################## + # The --echangelog option causes automatic ChangeLog generation, + # which invalidates changelog.ebuildadded and changelog.missing + # checks. + # Note: Some don't use ChangeLogs in distributed SCMs. + # It will be generated on server side from scm log, + # before package moves to the rsync server. + # This is needed because they try to avoid merge collisions. + # Gentoo's Council decided to always use the ChangeLog file. + # TODO: shouldn't this just be switched on the repo, iso the VCS? + is_echangelog_enabled = options.echangelog in ('y', 'force') + vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn') + check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn + if 'digest' in repoman_settings.features and options.digest != 'n': + options.digest = 'y' -if options.echangelog is None and repo_settings.repo_config.update_changelog: - options.echangelog = 'y' + logging.debug("vcs: %s" % (vcs_settings.vcs,)) + logging.debug("repo config: %s" % (repo_settings.repo_config,)) + logging.debug("options: %s" % (options,)) -if vcs_settings.vcs is None: - options.echangelog = 'n' + # It's confusing if these warnings are displayed without the user + # being told which profile they come from, so disable them. + env = os.environ.copy() + env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn' -# The --echangelog option causes automatic ChangeLog generation, -# which invalidates changelog.ebuildadded and changelog.missing -# checks. -# Note: Some don't use ChangeLogs in distributed SCMs. -# It will be generated on server side from scm log, -# before package moves to the rsync server. -# This is needed because they try to avoid merge collisions. -# Gentoo's Council decided to always use the ChangeLog file. -# TODO: shouldn't this just be switched on the repo, iso the VCS? -is_echangelog_enabled = options.echangelog in ('y', 'force') -vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn') -check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn + categories = [] + for path in repo_settings.repo_config.eclass_db.porttrees: + categories.extend(portage.util.grabfile( + os.path.join(path, 'profiles', 'categories'))) + repoman_settings.categories = frozenset( + portage.util.stack_lists([categories], incremental=1)) + categories = repoman_settings.categories -if 'digest' in repoman_settings.features and options.digest != 'n': - options.digest = 'y' + portdb.settings = repoman_settings + # We really only need to cache the metadata that's necessary for visibility + # filtering. Anything else can be discarded to reduce memory consumption. + portdb._aux_cache_keys.clear() + portdb._aux_cache_keys.update( + ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"]) -logging.debug("vcs: %s" % (vcs_settings.vcs,)) -logging.debug("repo config: %s" % (repo_settings.repo_config,)) -logging.debug("options: %s" % (options,)) + reposplit = myreporoot.split(os.path.sep) + repolevel = len(reposplit) -# It's confusing if these warnings are displayed without the user -# being told which profile they come from, so disable them. -env = os.environ.copy() -env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn' + ################### -categories = [] -for path in repo_settings.repo_config.eclass_db.porttrees: - categories.extend(portage.util.grabfile( - os.path.join(path, 'profiles', 'categories'))) -repoman_settings.categories = frozenset( - portage.util.stack_lists([categories], incremental=1)) -categories = repoman_settings.categories + if options.mode == 'commit': + repochecks.commit_check(repolevel, reposplit) + repochecks.conflict_check(vcs_settings, options) -portdb.settings = repoman_settings -# We really only need to cache the metadata that's necessary for visibility -# filtering. Anything else can be discarded to reduce memory consumption. -portdb._aux_cache_keys.clear() -portdb._aux_cache_keys.update( - ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"]) + ################### -reposplit = myreporoot.split(os.path.sep) -repolevel = len(reposplit) + # Make startdir relative to the canonical repodir, so that we can pass + # it to digestgen and it won't have to be canonicalized again. + if repolevel == 1: + startdir = repo_settings.repodir + else: + startdir = normalize_path(mydir) + startdir = os.path.join( + repo_settings.repodir, *startdir.split(os.sep)[-2 - repolevel + 3:]) + ################### -################### + # get lists of valid keywords, licenses, and use + new_data = repo_metadata(repo_settings.portdb, repoman_settings) + kwlist, liclist, uselist, profile_list, \ + global_pmaskdict, liclist_deprecated = new_data -if options.mode == 'commit': - repochecks.commit_check(repolevel, reposplit) - repochecks.conflict_check(vcs_settings, options) + repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist)) + repoman_settings.backup_changes('PORTAGE_ARCHLIST') -################### + #################### -# Make startdir relative to the canonical repodir, so that we can pass -# it to digestgen and it won't have to be canonicalized again. -if repolevel == 1: - startdir = repo_settings.repodir -else: - startdir = normalize_path(mydir) - startdir = os.path.join( - repo_settings.repodir, *startdir.split(os.sep)[-2 - repolevel + 3:]) -################### - -# get lists of valid keywords, licenses, and use -new_data = repo_metadata(repo_settings.portdb, repoman_settings) -kwlist, liclist, uselist, profile_list, \ - global_pmaskdict, liclist_deprecated = new_data - -repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist)) -repoman_settings.backup_changes('PORTAGE_ARCHLIST') - -#################### - -profiles = setup_profile(profile_list) - -#################### - -check_profiles(profiles, repoman_settings.archlist()) - -#################### - -scanlist = scan(repolevel, reposplit, startdir, categories, repo_settings) - -#################### - -dev_keywords = dev_keywords(profiles) - -qatracker = QATracker() - - -if options.mode == "manifest": - pass -elif options.pretend: - print(green("\nRepoMan does a once-over of the neighborhood...")) -else: - print(green("\nRepoMan scours the neighborhood...")) - -##################### - -changed = Changes(options) -changed.scan(vcs_settings) - -###################### - -have_pmasked = False -have_dev_keywords = False -dofail = 0 - -# NOTE: match-all caches are not shared due to potential -# differences between profiles in _get_implicit_iuse. -arch_caches = {} -arch_xmatch_caches = {} -shared_xmatch_caches = {"cp-list": {}} - -include_arches = None -if options.include_arches: - include_arches = set() - include_arches.update(*[x.split() for x in options.include_arches]) - -# Disable the "ebuild.notadded" check when not in commit mode and -# running `svn status` in every package dir will be too expensive. - -check_ebuild_notadded = not \ - (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode != "commit") - -effective_scanlist = scanlist -if options.if_modified == "y": - effective_scanlist = sorted(vcs_files_to_cps( - chain(changed.changed, changed.new, changed.removed), - repolevel, reposplit, categories)) - -###################### -# initialize our checks classes here before the big xpkg loop -manifester = Manifests(options, qatracker, repoman_settings) -is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker) -filescheck = FileChecks( - qatracker, repoman_settings, repo_settings, portdb, vcs_settings) -status_check = VCSStatus(vcs_settings, qatracker) -fetchcheck = FetchChecks( - qatracker, repoman_settings, repo_settings, portdb, vcs_settings) -pkgmeta = PkgMetadata(options, qatracker, repoman_settings) -thirdparty = ThirdPartyMirrors(repoman_settings, qatracker) -use_flag_checks = USEFlagChecks(qatracker, uselist) -keywordcheck = KeywordChecks(qatracker) -liveeclasscheck = LiveEclassChecks(qatracker) -rubyeclasscheck = RubyEclassChecks(qatracker) -eapicheck = EAPIChecks(qatracker, repo_settings) -descriptioncheck = DescriptionChecks(qatracker) -licensecheck = LicenseChecks(qatracker, liclist, liclist_deprecated) -restrictcheck = RestrictChecks(qatracker) -###################### - -for xpkg in effective_scanlist: - # ebuilds and digests added to cvs respectively. - logging.info("checking package %s" % xpkg) - # save memory by discarding xmatch caches from previous package(s) - arch_xmatch_caches.clear() - eadded = [] - catdir, pkgdir = xpkg.split("/") - checkdir = repo_settings.repodir + "/" + xpkg - checkdir_relative = "" - if repolevel < 3: - checkdir_relative = os.path.join(pkgdir, checkdir_relative) - if repolevel < 2: - checkdir_relative = os.path.join(catdir, checkdir_relative) - checkdir_relative = os.path.join(".", checkdir_relative) - -##################### - if manifester.run(checkdir, portdb): - continue - if not manifester.generated_manifest: - manifester.digest_check(xpkg, checkdir) -###################### - - if options.mode == 'manifest-check': - continue - - checkdirlist = os.listdir(checkdir) - -###################### - pkgs, allvalid = is_ebuild.check(checkdirlist, checkdir, xpkg) - if is_ebuild.continue_: - # If we can't access all the metadata then it's totally unsafe to - # commit since there's no way to generate a correct Manifest. - # Do not try to do any more QA checks on this package since missing - # metadata leads to false positives for several checks, and false - # positives confuse users. - can_force = False - continue -###################### - - keywordcheck.prepare() - - # Sort ebuilds in ascending order for the KEYWORDS.dropped check. - ebuildlist = sorted(pkgs.values()) - ebuildlist = [pkg.pf for pkg in ebuildlist] -####################### - filescheck.check( - checkdir, checkdirlist, checkdir_relative, changed.changed, changed.new) -####################### - status_check.check(check_ebuild_notadded, checkdir, checkdir_relative, xpkg) - eadded.extend(status_check.eadded) - -################# - fetchcheck.check( - xpkg, checkdir, checkdir_relative, changed.changed, changed.new) -################# - - if check_changelog and "ChangeLog" not in checkdirlist: - qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") -################# - pkgmeta.check(xpkg, checkdir, checkdirlist, repolevel) - muselist = frozenset(pkgmeta.musedict) -################# - - changelog_path = os.path.join(checkdir_relative, "ChangeLog") - changelog_modified = changelog_path in changed.changelogs - - # detect unused local USE-descriptions - used_useflags = set() - - for y_ebuild in ebuildlist: - ################## - ebuild = Ebuild( - repo_settings, repolevel, pkgdir, catdir, vcs_settings, - xpkg, y_ebuild) - ################## - - if check_changelog and not changelog_modified \ - and ebuild.ebuild_path in changed.new_ebuilds: - qatracker.add_error('changelog.ebuildadded', ebuild.relative_path) - - if ebuild.untracked(check_ebuild_notadded, y_ebuild, eadded): - # ebuild not added to vcs - qatracker.add_error( - "ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild") - -################## - if bad_split_check(xpkg, y_ebuild, pkgdir, qatracker): - continue -################### - pkg = pkgs[y_ebuild] - if pkg_invalid(pkg, qatracker, ebuild): - allvalid = False + profiles = setup_profile(profile_list) + + #################### + + check_profiles(profiles, repoman_settings.archlist()) + + #################### + + scanlist = scan(repolevel, reposplit, startdir, categories, repo_settings) + + #################### + + dev_keywords = dev_keywords(profiles) + + qatracker = QATracker() + + + if options.mode == "manifest": + pass + elif options.pretend: + print(green("\nRepoMan does a once-over of the neighborhood...")) + else: + print(green("\nRepoMan scours the neighborhood...")) + + ##################### + + changed = Changes(options) + changed.scan(vcs_settings) + + ###################### + + have_pmasked = False + have_dev_keywords = False + dofail = 0 + + # NOTE: match-all caches are not shared due to potential + # differences between profiles in _get_implicit_iuse. + arch_caches = {} + arch_xmatch_caches = {} + shared_xmatch_caches = {"cp-list": {}} + + include_arches = None + if options.include_arches: + include_arches = set() + include_arches.update(*[x.split() for x in options.include_arches]) + + # Disable the "ebuild.notadded" check when not in commit mode and + # running `svn status` in every package dir will be too expensive. + + check_ebuild_notadded = not \ + (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode != "commit") + + effective_scanlist = scanlist + if options.if_modified == "y": + effective_scanlist = sorted(vcs_files_to_cps( + chain(changed.changed, changed.new, changed.removed), + repolevel, reposplit, categories)) + + ###################### + # initialize our checks classes here before the big xpkg loop + manifester = Manifests(options, qatracker, repoman_settings) + is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker) + filescheck = FileChecks( + qatracker, repoman_settings, repo_settings, portdb, vcs_settings) + status_check = VCSStatus(vcs_settings, qatracker) + fetchcheck = FetchChecks( + qatracker, repoman_settings, repo_settings, portdb, vcs_settings) + pkgmeta = PkgMetadata(options, qatracker, repoman_settings) + thirdparty = ThirdPartyMirrors(repoman_settings, qatracker) + use_flag_checks = USEFlagChecks(qatracker, uselist) + keywordcheck = KeywordChecks(qatracker) + liveeclasscheck = LiveEclassChecks(qatracker) + rubyeclasscheck = RubyEclassChecks(qatracker) + eapicheck = EAPIChecks(qatracker, repo_settings) + descriptioncheck = DescriptionChecks(qatracker) + licensecheck = LicenseChecks(qatracker, liclist, liclist_deprecated) + restrictcheck = RestrictChecks(qatracker) + ###################### + + for xpkg in effective_scanlist: + # ebuilds and digests added to cvs respectively. + logging.info("checking package %s" % xpkg) + # save memory by discarding xmatch caches from previous package(s) + arch_xmatch_caches.clear() + eadded = [] + catdir, pkgdir = xpkg.split("/") + checkdir = repo_settings.repodir + "/" + xpkg + checkdir_relative = "" + if repolevel < 3: + checkdir_relative = os.path.join(pkgdir, checkdir_relative) + if repolevel < 2: + checkdir_relative = os.path.join(catdir, checkdir_relative) + checkdir_relative = os.path.join(".", checkdir_relative) + + ##################### + if manifester.run(checkdir, portdb): continue + if not manifester.generated_manifest: + manifester.digest_check(xpkg, checkdir) + ###################### - myaux = pkg._metadata - eapi = myaux["EAPI"] - inherited = pkg.inherited - live_ebuild = live_eclasses.intersection(inherited) + if options.mode == 'manifest-check': + continue - ####################### - eapicheck.check(pkg, ebuild) - ####################### + checkdirlist = os.listdir(checkdir) + + ###################### + pkgs, allvalid = is_ebuild.check(checkdirlist, checkdir, xpkg) + if is_ebuild.continue_: + # If we can't access all the metadata then it's totally unsafe to + # commit since there's no way to generate a correct Manifest. + # Do not try to do any more QA checks on this package since missing + # metadata leads to false positives for several checks, and false + # positives confuse users. + can_force = False + continue + ###################### + + keywordcheck.prepare() + + # Sort ebuilds in ascending order for the KEYWORDS.dropped check. + ebuildlist = sorted(pkgs.values()) + ebuildlist = [pkg.pf for pkg in ebuildlist] + ####################### + filescheck.check( + checkdir, checkdirlist, checkdir_relative, changed.changed, changed.new) + ####################### + status_check.check(check_ebuild_notadded, checkdir, checkdir_relative, xpkg) + eadded.extend(status_check.eadded) + + ################# + fetchcheck.check( + xpkg, checkdir, checkdir_relative, changed.changed, changed.new) + ################# + + if check_changelog and "ChangeLog" not in checkdirlist: + qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") + ################# + pkgmeta.check(xpkg, checkdir, checkdirlist, repolevel) + muselist = frozenset(pkgmeta.musedict) + ################# + + changelog_path = os.path.join(checkdir_relative, "ChangeLog") + changelog_modified = changelog_path in changed.changelogs + + # detect unused local USE-descriptions + used_useflags = set() + + for y_ebuild in ebuildlist: + ################## + ebuild = Ebuild( + repo_settings, repolevel, pkgdir, catdir, vcs_settings, + xpkg, y_ebuild) + ################## + + if check_changelog and not changelog_modified \ + and ebuild.ebuild_path in changed.new_ebuilds: + qatracker.add_error('changelog.ebuildadded', ebuild.relative_path) + + if ebuild.untracked(check_ebuild_notadded, y_ebuild, eadded): + # ebuild not added to vcs + qatracker.add_error( + "ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild") - for k, v in myaux.items(): - if not isinstance(v, basestring): + ################## + if bad_split_check(xpkg, y_ebuild, pkgdir, qatracker): continue - m = non_ascii_re.search(v) - if m is not None: - qatracker.add_error( - "variable.invalidchar", - "%s: %s variable contains non-ASCII " - "character at position %s" % - (ebuild.relative_path, k, m.start() + 1)) + ################### + pkg = pkgs[y_ebuild] + if pkg_invalid(pkg, qatracker, ebuild): + allvalid = False + continue + + myaux = pkg._metadata + eapi = myaux["EAPI"] + inherited = pkg.inherited + live_ebuild = live_eclasses.intersection(inherited) - if not fetchcheck.src_uri_error: ####################### - thirdparty.check(myaux, ebuild.relative_path) + eapicheck.check(pkg, ebuild) ####################### - if myaux.get("PROVIDE"): - qatracker.add_error("virtual.oldstyle", ebuild.relative_path) - for pos, missing_var in enumerate(missingvars): - if not myaux.get(missing_var): - if catdir == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): + for k, v in myaux.items(): + if not isinstance(v, basestring): continue - if live_ebuild and missing_var == "KEYWORDS": - continue - myqakey = missingvars[pos] + ".missing" - qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - - if catdir == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if myaux.get(var): - myqakey = var + ".virtual" - qatracker.add_error(myqakey, ebuild.relative_path) + m = non_ascii_re.search(v) + if m is not None: + qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + + if not fetchcheck.src_uri_error: + ####################### + thirdparty.check(myaux, ebuild.relative_path) + ####################### + if myaux.get("PROVIDE"): + qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + + for pos, missing_var in enumerate(missingvars): + if not myaux.get(missing_var): + if catdir == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if live_ebuild and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - ####################### - descriptioncheck.check(pkg, ebuild) - ####################### + if catdir == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if myaux.get(var): + myqakey = var + ".virtual" + qatracker.add_error(myqakey, ebuild.relative_path) - keywords = myaux["KEYWORDS"].split() + ####################### + descriptioncheck.check(pkg, ebuild) + ####################### - ebuild_archs = set( - kw.lstrip("~") for kw in keywords if not kw.startswith("-")) + keywords = myaux["KEYWORDS"].split() - ####################### - keywordcheck.check( - pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, changed, - live_ebuild, kwlist, profiles) - ####################### + ebuild_archs = set( + kw.lstrip("~") for kw in keywords if not kw.startswith("-")) - if live_ebuild and repo_settings.repo_config.name == "gentoo": ####################### - liveeclasscheck.check( - pkg, xpkg, ebuild, y_ebuild, keywords, global_pmaskdict) + keywordcheck.check( + pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, changed, + live_ebuild, kwlist, profiles) ####################### - if options.ignore_arches: - arches = [[ - repoman_settings["ARCH"], repoman_settings["ARCH"], - repoman_settings["ACCEPT_KEYWORDS"].split()]] - else: - arches = set() - for keyword in keywords: - if keyword[0] == "-": - continue - elif keyword[0] == "~": - arch = keyword[1:] - if arch == "*": - for expanded_arch in profiles: - if expanded_arch == "**": - continue - arches.add( - (keyword, expanded_arch, ( - expanded_arch, "~" + expanded_arch))) - else: - arches.add((keyword, arch, (arch, keyword))) - else: - if keyword == "*": - for expanded_arch in profiles: - if expanded_arch == "**": - continue - arches.add( - (keyword, expanded_arch, (expanded_arch,))) - else: - arches.add((keyword, keyword, (keyword,))) - if not arches: - # Use an empty profile for checking dependencies of - # packages that have empty KEYWORDS. - arches.add(('**', '**', ('**',))) - - unknown_pkgs = set() - baddepsyntax = False - badlicsyntax = False - badprovsyntax = False - catpkg = catdir + "/" + y_ebuild - - inherited_java_eclass = "java-pkg-2" in inherited or \ - "java-pkg-opt-2" in inherited - inherited_wxwidgets_eclass = "wxwidgets" in inherited - operator_tokens = set(["||", "(", ")"]) - type_list, badsyntax = [], [] - for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"): - mydepstr = myaux[mytype] - - buildtime = mytype in Package._buildtime_keys - runtime = mytype in Package._runtime_keys - token_class = None - if mytype.endswith("DEPEND"): - token_class = portage.dep.Atom + if live_ebuild and repo_settings.repo_config.name == "gentoo": + ####################### + liveeclasscheck.check( + pkg, xpkg, ebuild, y_ebuild, keywords, global_pmaskdict) + ####################### - try: - atoms = portage.dep.use_reduce( - mydepstr, matchall=1, flat=True, - is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class) - except portage.exception.InvalidDependString as e: - atoms = None - badsyntax.append(str(e)) - - if atoms and mytype.endswith("DEPEND"): - if runtime and \ - "test?" in mydepstr.split(): - qatracker.add_error( - mytype + '.suspect', - "%s: 'test?' USE conditional in %s" % - (ebuild.relative_path, mytype)) - - for atom in atoms: - if atom == "||": + if options.ignore_arches: + arches = [[ + repoman_settings["ARCH"], repoman_settings["ARCH"], + repoman_settings["ACCEPT_KEYWORDS"].split()]] + else: + arches = set() + for keyword in keywords: + if keyword[0] == "-": continue + elif keyword[0] == "~": + arch = keyword[1:] + if arch == "*": + for expanded_arch in profiles: + if expanded_arch == "**": + continue + arches.add( + (keyword, expanded_arch, ( + expanded_arch, "~" + expanded_arch))) + else: + arches.add((keyword, arch, (arch, keyword))) + else: + if keyword == "*": + for expanded_arch in profiles: + if expanded_arch == "**": + continue + arches.add( + (keyword, expanded_arch, (expanded_arch,))) + else: + arches.add((keyword, keyword, (keyword,))) + if not arches: + # Use an empty profile for checking dependencies of + # packages that have empty KEYWORDS. + arches.add(('**', '**', ('**',))) + + unknown_pkgs = set() + baddepsyntax = False + badlicsyntax = False + badprovsyntax = False + catpkg = catdir + "/" + y_ebuild + + inherited_java_eclass = "java-pkg-2" in inherited or \ + "java-pkg-opt-2" in inherited + inherited_wxwidgets_eclass = "wxwidgets" in inherited + operator_tokens = set(["||", "(", ")"]) + type_list, badsyntax = [], [] + for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"): + mydepstr = myaux[mytype] + + buildtime = mytype in Package._buildtime_keys + runtime = mytype in Package._runtime_keys + token_class = None + if mytype.endswith("DEPEND"): + token_class = portage.dep.Atom + + try: + atoms = portage.dep.use_reduce( + mydepstr, matchall=1, flat=True, + is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class) + except portage.exception.InvalidDependString as e: + atoms = None + badsyntax.append(str(e)) + + if atoms and mytype.endswith("DEPEND"): + if runtime and \ + "test?" in mydepstr.split(): + qatracker.add_error( + mytype + '.suspect', + "%s: 'test?' USE conditional in %s" % + (ebuild.relative_path, mytype)) - is_blocker = atom.blocker + for atom in atoms: + if atom == "||": + continue - # Skip dependency.unknown for blockers, so that we - # don't encourage people to remove necessary blockers, - # as discussed in bug #382407. - if not is_blocker and \ - not portdb.xmatch("match-all", atom) and \ - not atom.cp.startswith("virtual/"): - unknown_pkgs.add((mytype, atom.unevaluated_atom)) + is_blocker = atom.blocker - if catdir != "virtual": + # Skip dependency.unknown for blockers, so that we + # don't encourage people to remove necessary blockers, + # as discussed in bug #382407. if not is_blocker and \ - atom.cp in suspect_virtual: + not portdb.xmatch("match-all", atom) and \ + not atom.cp.startswith("virtual/"): + unknown_pkgs.add((mytype, atom.unevaluated_atom)) + + if catdir != "virtual": + if not is_blocker and \ + atom.cp in suspect_virtual: + qatracker.add_error( + 'virtual.suspect', ebuild.relative_path + + ": %s: consider using '%s' instead of '%s'" % + (mytype, suspect_virtual[atom.cp], atom)) + if not is_blocker and \ + atom.cp.startswith("perl-core/"): + qatracker.add_error('dependency.perlcore', + ebuild.relative_path + + ": %s: please use '%s' instead of '%s'" % + (mytype, + atom.replace("perl-core/","virtual/perl-"), + atom)) + + if buildtime and \ + not is_blocker and \ + not inherited_java_eclass and \ + atom.cp == "virtual/jdk": qatracker.add_error( - 'virtual.suspect', ebuild.relative_path + - ": %s: consider using '%s' instead of '%s'" % - (mytype, suspect_virtual[atom.cp], atom)) - if not is_blocker and \ - atom.cp.startswith("perl-core/"): - qatracker.add_error('dependency.perlcore', - ebuild.relative_path + - ": %s: please use '%s' instead of '%s'" % - (mytype, - atom.replace("perl-core/","virtual/perl-"), - atom)) - - if buildtime and \ - not is_blocker and \ - not inherited_java_eclass and \ - atom.cp == "virtual/jdk": - qatracker.add_error( - 'java.eclassesnotused', ebuild.relative_path) - elif buildtime and \ - not is_blocker and \ - not inherited_wxwidgets_eclass and \ - atom.cp == "x11-libs/wxGTK": - qatracker.add_error( - 'wxwidgets.eclassnotused', - "%s: %ss on x11-libs/wxGTK without inheriting" - " wxwidgets.eclass" % (ebuild.relative_path, mytype)) - elif runtime: - if not is_blocker and \ - atom.cp in suspect_rdepend: + 'java.eclassesnotused', ebuild.relative_path) + elif buildtime and \ + not is_blocker and \ + not inherited_wxwidgets_eclass and \ + atom.cp == "x11-libs/wxGTK": + qatracker.add_error( + 'wxwidgets.eclassnotused', + "%s: %ss on x11-libs/wxGTK without inheriting" + " wxwidgets.eclass" % (ebuild.relative_path, mytype)) + elif runtime: + if not is_blocker and \ + atom.cp in suspect_rdepend: + qatracker.add_error( + mytype + '.suspect', + ebuild.relative_path + ": '%s'" % atom) + + if atom.operator == "~" and \ + portage.versions.catpkgsplit(atom.cpv)[3] != "r0": + qacat = 'dependency.badtilde' qatracker.add_error( - mytype + '.suspect', - ebuild.relative_path + ": '%s'" % atom) + qacat, "%s: %s uses the ~ operator" + " with a non-zero revision: '%s'" % + (ebuild.relative_path, mytype, atom)) - if atom.operator == "~" and \ - portage.versions.catpkgsplit(atom.cpv)[3] != "r0": - qacat = 'dependency.badtilde' - qatracker.add_error( - qacat, "%s: %s uses the ~ operator" - " with a non-zero revision: '%s'" % - (ebuild.relative_path, mytype, atom)) + type_list.extend([mytype] * (len(badsyntax) - len(type_list))) + + for m, b in zip(type_list, badsyntax): + if m.endswith("DEPEND"): + qacat = "dependency.syntax" + else: + qacat = m + ".syntax" + qatracker.add_error( + qacat, "%s: %s: %s" % (ebuild.relative_path, m, b)) - type_list.extend([mytype] * (len(badsyntax) - len(type_list))) + badlicsyntax = len([z for z in type_list if z == "LICENSE"]) + badprovsyntax = len([z for z in type_list if z == "PROVIDE"]) + baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax + badlicsyntax = badlicsyntax > 0 + badprovsyntax = badprovsyntax > 0 - for m, b in zip(type_list, badsyntax): - if m.endswith("DEPEND"): - qacat = "dependency.syntax" - else: - qacat = m + ".syntax" - qatracker.add_error( - qacat, "%s: %s: %s" % (ebuild.relative_path, m, b)) - - badlicsyntax = len([z for z in type_list if z == "LICENSE"]) - badprovsyntax = len([z for z in type_list if z == "PROVIDE"]) - baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax - badlicsyntax = badlicsyntax > 0 - badprovsyntax = badprovsyntax > 0 - - ################# - use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, muselist) - - ebuild_used_useflags = use_flag_checks.getUsedUseFlags() - used_useflags = used_useflags.union(ebuild_used_useflags) - ################# - rubyeclasscheck.check(pkg, ebuild) - ################# - - # license checks - if not badlicsyntax: ################# - licensecheck.check(pkg, xpkg, ebuild, y_ebuild) + use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, muselist) + + ebuild_used_useflags = use_flag_checks.getUsedUseFlags() + used_useflags = used_useflags.union(ebuild_used_useflags) + ################# + rubyeclasscheck.check(pkg, ebuild) ################# - ################# - restrictcheck.check(pkg, xpkg, ebuild, y_ebuild) - ################# + # license checks + if not badlicsyntax: + ################# + licensecheck.check(pkg, xpkg, ebuild, y_ebuild) + ################# - # Syntax Checks + ################# + restrictcheck.check(pkg, xpkg, ebuild, y_ebuild) + ################# - if not vcs_settings.vcs_preserves_mtime: - if ebuild.ebuild_path not in changed.new_ebuilds and \ - ebuild.ebuild_path not in changed.ebuilds: - pkg.mtime = None - try: - # All ebuilds should have utf_8 encoding. - f = io.open( - _unicode_encode( - ebuild.full_path, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content']) - try: - for check_name, e in run_checks(f, pkg): - qatracker.add_error( - check_name, ebuild.relative_path + ': %s' % e) - finally: - f.close() - except UnicodeDecodeError: - # A file.UTF8 failure will have already been recorded above. - pass + # Syntax Checks - if options.force: - # The dep_check() calls are the most expensive QA test. If --force - # is enabled, there's no point in wasting time on these since the - # user is intent on forcing the commit anyway. - continue + if not vcs_settings.vcs_preserves_mtime: + if ebuild.ebuild_path not in changed.new_ebuilds and \ + ebuild.ebuild_path not in changed.ebuilds: + pkg.mtime = None + try: + # All ebuilds should have utf_8 encoding. + f = io.open( + _unicode_encode( + ebuild.full_path, encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content']) + try: + for check_name, e in run_checks(f, pkg): + qatracker.add_error( + check_name, ebuild.relative_path + ': %s' % e) + finally: + f.close() + except UnicodeDecodeError: + # A file.UTF8 failure will have already been recorded above. + pass - relevant_profiles = [] - for keyword, arch, groups in arches: - if arch not in profiles: - # A missing profile will create an error further down - # during the KEYWORDS verification. + if options.force: + # The dep_check() calls are the most expensive QA test. If --force + # is enabled, there's no point in wasting time on these since the + # user is intent on forcing the commit anyway. continue - if include_arches is not None: - if arch not in include_arches: + relevant_profiles = [] + for keyword, arch, groups in arches: + if arch not in profiles: + # A missing profile will create an error further down + # during the KEYWORDS verification. continue - relevant_profiles.extend( - (keyword, groups, prof) for prof in profiles[arch]) - - def sort_key(item): - return item[2].sub_path - - relevant_profiles.sort(key=sort_key) - - for keyword, groups, prof in relevant_profiles: - - is_stable_profile = prof.status == "stable" - is_dev_profile = prof.status == "dev" and \ - options.include_dev - is_exp_profile = prof.status == "exp" and \ - options.include_exp_profiles == 'y' - if not (is_stable_profile or is_dev_profile or is_exp_profile): - continue - - dep_settings = arch_caches.get(prof.sub_path) - if dep_settings is None: - dep_settings = portage.config( - config_profile_path=prof.abs_path, - config_incrementals=repoman_incrementals, - config_root=config_root, - local_config=False, - _unmatched_removal=options.unmatched_removal, - env=env, repositories=repoman_settings.repositories) - dep_settings.categories = repoman_settings.categories - if options.without_mask: - dep_settings._mask_manager_obj = \ - copy.deepcopy(dep_settings._mask_manager) - dep_settings._mask_manager._pmaskdict.clear() - arch_caches[prof.sub_path] = dep_settings - - xmatch_cache_key = (prof.sub_path, tuple(groups)) - xcache = arch_xmatch_caches.get(xmatch_cache_key) - if xcache is None: - portdb.melt() - portdb.freeze() - xcache = portdb.xcache - xcache.update(shared_xmatch_caches) - arch_xmatch_caches[xmatch_cache_key] = xcache - - repo_settings.trees[repo_settings.root]["porttree"].settings = dep_settings - portdb.settings = dep_settings - portdb.xcache = xcache - - dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) - # just in case, prevent config.reset() from nuking these. - dep_settings.backup_changes("ACCEPT_KEYWORDS") - - # This attribute is used in dbapi._match_use() to apply - # use.stable.{mask,force} settings based on the stable - # status of the parent package. This is required in order - # for USE deps of unstable packages to be resolved correctly, - # since otherwise use.stable.{mask,force} settings of - # dependencies may conflict (see bug #456342). - dep_settings._parent_stable = dep_settings._isStable(pkg) - - # Handle package.use*.{force,mask) calculation, for use - # in dep_check. - dep_settings.useforce = dep_settings._use_manager.getUseForce( - pkg, stable=dep_settings._parent_stable) - dep_settings.usemask = dep_settings._use_manager.getUseMask( - pkg, stable=dep_settings._parent_stable) - - if not baddepsyntax: - ismasked = not ebuild_archs or \ - pkg.cpv not in portdb.xmatch("match-visible", pkg.cp) - if ismasked: - if not have_pmasked: - have_pmasked = bool(dep_settings._getMaskAtom( - pkg.cpv, pkg._metadata)) - if options.ignore_masked: + if include_arches is not None: + if arch not in include_arches: continue - # we are testing deps for a masked package; give it some lee-way - suffix = "masked" - matchmode = "minimum-all" - else: - suffix = "" - matchmode = "minimum-visible" - if not have_dev_keywords: - have_dev_keywords = \ - bool(dev_keywords.intersection(keywords)) + relevant_profiles.extend( + (keyword, groups, prof) for prof in profiles[arch]) - if prof.status == "dev": - suffix = suffix + "indev" + relevant_profiles.sort(key=sort_key) - for mytype in Package._dep_keys: + for keyword, groups, prof in relevant_profiles: - mykey = "dependency.bad" + suffix - myvalue = myaux[mytype] - if not myvalue: - continue + is_stable_profile = prof.status == "stable" + is_dev_profile = prof.status == "dev" and \ + options.include_dev + is_exp_profile = prof.status == "exp" and \ + options.include_exp_profiles == 'y' + if not (is_stable_profile or is_dev_profile or is_exp_profile): + continue - success, atoms = portage.dep_check( - myvalue, portdb, dep_settings, - use="all", mode=matchmode, trees=repo_settings.trees) - - if success: - if atoms: - - # Don't bother with dependency.unknown for - # cases in which *DEPEND.bad is triggered. - for atom in atoms: - # dep_check returns all blockers and they - # aren't counted for *DEPEND.bad, so we - # ignore them here. - if not atom.blocker: - unknown_pkgs.discard( - (mytype, atom.unevaluated_atom)) - - if not prof.sub_path: - # old-style virtuals currently aren't - # resolvable with empty profile, since - # 'virtuals' mappings are unavailable - # (it would be expensive to search - # for PROVIDE in all ebuilds) + dep_settings = arch_caches.get(prof.sub_path) + if dep_settings is None: + dep_settings = portage.config( + config_profile_path=prof.abs_path, + config_incrementals=repoman_incrementals, + config_root=config_root, + local_config=False, + _unmatched_removal=options.unmatched_removal, + env=env, repositories=repoman_settings.repositories) + dep_settings.categories = repoman_settings.categories + if options.without_mask: + dep_settings._mask_manager_obj = \ + copy.deepcopy(dep_settings._mask_manager) + dep_settings._mask_manager._pmaskdict.clear() + arch_caches[prof.sub_path] = dep_settings + + xmatch_cache_key = (prof.sub_path, tuple(groups)) + xcache = arch_xmatch_caches.get(xmatch_cache_key) + if xcache is None: + portdb.melt() + portdb.freeze() + xcache = portdb.xcache + xcache.update(shared_xmatch_caches) + arch_xmatch_caches[xmatch_cache_key] = xcache + + repo_settings.trees[repo_settings.root]["porttree"].settings = dep_settings + portdb.settings = dep_settings + portdb.xcache = xcache + + dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) + # just in case, prevent config.reset() from nuking these. + dep_settings.backup_changes("ACCEPT_KEYWORDS") + + # This attribute is used in dbapi._match_use() to apply + # use.stable.{mask,force} settings based on the stable + # status of the parent package. This is required in order + # for USE deps of unstable packages to be resolved correctly, + # since otherwise use.stable.{mask,force} settings of + # dependencies may conflict (see bug #456342). + dep_settings._parent_stable = dep_settings._isStable(pkg) + + # Handle package.use*.{force,mask) calculation, for use + # in dep_check. + dep_settings.useforce = dep_settings._use_manager.getUseForce( + pkg, stable=dep_settings._parent_stable) + dep_settings.usemask = dep_settings._use_manager.getUseMask( + pkg, stable=dep_settings._parent_stable) + + if not baddepsyntax: + ismasked = not ebuild_archs or \ + pkg.cpv not in portdb.xmatch("match-visible", pkg.cp) + if ismasked: + if not have_pmasked: + have_pmasked = bool(dep_settings._getMaskAtom( + pkg.cpv, pkg._metadata)) + if options.ignore_masked: + continue + # we are testing deps for a masked package; give it some lee-way + suffix = "masked" + matchmode = "minimum-all" + else: + suffix = "" + matchmode = "minimum-visible" + + if not have_dev_keywords: + have_dev_keywords = \ + bool(dev_keywords.intersection(keywords)) + + if prof.status == "dev": + suffix = suffix + "indev" + + for mytype in Package._dep_keys: + + mykey = "dependency.bad" + suffix + myvalue = myaux[mytype] + if not myvalue: + continue + + success, atoms = portage.dep_check( + myvalue, portdb, dep_settings, + use="all", mode=matchmode, trees=repo_settings.trees) + + if success: + if atoms: + + # Don't bother with dependency.unknown for + # cases in which *DEPEND.bad is triggered. + for atom in atoms: + # dep_check returns all blockers and they + # aren't counted for *DEPEND.bad, so we + # ignore them here. + if not atom.blocker: + unknown_pkgs.discard( + (mytype, atom.unevaluated_atom)) + + if not prof.sub_path: + # old-style virtuals currently aren't + # resolvable with empty profile, since + # 'virtuals' mappings are unavailable + # (it would be expensive to search + # for PROVIDE in all ebuilds) + atoms = [ + atom for atom in atoms if not ( + atom.cp.startswith('virtual/') + and not portdb.cp_list(atom.cp))] + + # we have some unsolvable deps + # remove ! deps, which always show up as unsatisfiable atoms = [ - atom for atom in atoms if not ( - atom.cp.startswith('virtual/') - and not portdb.cp_list(atom.cp))] - - # we have some unsolvable deps - # remove ! deps, which always show up as unsatisfiable - atoms = [ - str(atom.unevaluated_atom) - for atom in atoms if not atom.blocker] - - # if we emptied out our list, continue: - if not atoms: - continue + str(atom.unevaluated_atom) + for atom in atoms if not atom.blocker] + + # if we emptied out our list, continue: + if not atoms: + continue + qatracker.add_error( + mykey, "%s: %s: %s(%s) %s" % ( + ebuild.relative_path, mytype, keyword, prof, + repr(atoms))) + else: qatracker.add_error( mykey, "%s: %s: %s(%s) %s" % ( ebuild.relative_path, mytype, keyword, prof, repr(atoms))) - else: - qatracker.add_error( - mykey, "%s: %s: %s(%s) %s" % ( - ebuild.relative_path, mytype, keyword, prof, - repr(atoms))) - - if not baddepsyntax and unknown_pkgs: - type_map = {} - for mytype, atom in unknown_pkgs: - type_map.setdefault(mytype, set()).add(atom) - for mytype, atoms in type_map.items(): + + if not baddepsyntax and unknown_pkgs: + type_map = {} + for mytype, atom in unknown_pkgs: + type_map.setdefault(mytype, set()).add(atom) + for mytype, atoms in type_map.items(): + qatracker.add_error( + "dependency.unknown", "%s: %s: %s" + % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) + + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if allvalid: + for myflag in muselist.difference(used_useflags): qatracker.add_error( - "dependency.unknown", "%s: %s: %s" - % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) - - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if allvalid: - for myflag in muselist.difference(used_useflags): - qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) - -if options.if_modified == "y" and len(effective_scanlist) < 1: - logging.warn("--if-modified is enabled, but no modified packages were found!") - -if options.mode == "manifest": - sys.exit(dofail) - -# dofail will be true if we have failed in at least one non-warning category -dofail = 0 -# dowarn will be true if we tripped any warnings -dowarn = 0 -# dofull will be true if we should print a "repoman full" informational message -dofull = options.mode != 'full' - -for x in qacats: - if x not in qatracker.fails: - continue - dowarn = 1 - if x not in qawarnings: - dofail = 1 - -if dofail or \ - (dowarn and not (options.quiet or options.mode == "scan")): - dofull = 0 - -# Save QA output so that it can be conveniently displayed -# in $EDITOR while the user creates a commit message. -# Otherwise, the user would not be able to see this output -# once the editor has taken over the screen. -qa_output = io.StringIO() -style_file = ConsoleStyleFile(sys.stdout) -if options.mode == 'commit' and \ - (not commitmessage or not commitmessage.strip()): - style_file.write_listener = qa_output -console_writer = StyleWriter(file=style_file, maxcol=9999) -console_writer.style_listener = style_file.new_styles - -f = formatter.AbstractFormatter(console_writer) - -format_outputs = { - 'column': format_qa_output_column, - 'default': format_qa_output -} - -format_output = format_outputs.get( - options.output_style, format_outputs['default']) -format_output(f, qatracker.fails, dofull, dofail, options, qawarnings) - -style_file.flush() -del console_writer, f, style_file -qa_output = qa_output.getvalue() -qa_output = qa_output.splitlines(True) - -suggest_ignore_masked = False -suggest_include_dev = False - -if have_pmasked and not (options.without_mask or options.ignore_masked): - suggest_ignore_masked = True -if have_dev_keywords and not options.include_dev: - suggest_include_dev = True - -if suggest_ignore_masked or suggest_include_dev: - print() - if suggest_ignore_masked: - print(bold( - "Note: use --without-mask to check " - "KEYWORDS on dependencies of masked packages")) - - if suggest_include_dev: - print(bold( - "Note: use --include-dev (-d) to check " - "dependencies for 'dev' profiles")) - print() - -if options.mode != 'commit': - if dofull: - print(bold("Note: type \"repoman full\" for a complete listing.")) - if dowarn and not dofail: - utilities.repoman_sez( - "\"You're only giving me a partial QA payment?\n" - " I'll take it this time, but I'm not happy.\"") - elif not dofail: - utilities.repoman_sez( - "\"If everyone were like you, I'd be out of business!\"") - elif dofail: - print(bad("Please fix these important QA issues first.")) - utilities.repoman_sez( - "\"Make your QA payment on time" - " and you'll never see the likes of me.\"\n") - sys.exit(1) -else: - if dofail and can_force and options.force and not options.pretend: - utilities.repoman_sez( - " \"You want to commit even with these QA issues?\n" - " I'll take it this time, but I'm not happy.\"\n") - elif dofail: - if options.force and not can_force: - print(bad( - "The --force option has been disabled" - " due to extraordinary issues.")) - print(bad("Please fix these important QA issues first.")) - utilities.repoman_sez( - "\"Make your QA payment on time" - " and you'll never see the likes of me.\"\n") - sys.exit(1) + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) - if options.pretend: - utilities.repoman_sez( - "\"So, you want to play it safe. Good call.\"\n") + if options.if_modified == "y" and len(effective_scanlist) < 1: + logging.warn("--if-modified is enabled, but no modified packages were found!") - myunadded = [] - if vcs_settings.vcs == "cvs": - try: - myvcstree = portage.cvstree.getentries("./", recursive=1) - myunadded = portage.cvstree.findunadded( - myvcstree, recursive=1, basedir="./") - except SystemExit as e: - raise # TODO propagate this - except: - err("Error retrieving CVS tree; exiting.") - if vcs_settings.vcs == "svn": - try: - with repoman_popen("svn status --no-ignore") as f: - svnstatus = f.readlines() - myunadded = [ - "./" + elem.rstrip().split()[1] - for elem in svnstatus - if elem.startswith("?") or elem.startswith("I")] - except SystemExit as e: - raise # TODO propagate this - except: - err("Error retrieving SVN info; exiting.") - if vcs_settings.vcs == "git": - # get list of files not under version control or missing - myf = repoman_popen("git ls-files --others") - myunadded = ["./" + elem[:-1] for elem in myf] - myf.close() - if vcs_settings.vcs == "bzr": - try: - with repoman_popen("bzr status -S .") as f: - bzrstatus = f.readlines() - myunadded = [ - "./" + elem.rstrip().split()[1].split('/')[-1:][0] - for elem in bzrstatus - if elem.startswith("?") or elem[0:2] == " D"] - except SystemExit as e: - raise # TODO propagate this - except: - err("Error retrieving bzr info; exiting.") - if vcs_settings.vcs == "hg": - with repoman_popen("hg status --no-status --unknown .") as f: - myunadded = f.readlines() - myunadded = ["./" + elem.rstrip() for elem in myunadded] - - # Mercurial doesn't handle manually deleted files as removed from - # the repository, so the user need to remove them before commit, - # using "hg remove [FILES]" - with repoman_popen("hg status --no-status --deleted .") as f: - mydeleted = f.readlines() - mydeleted = ["./" + elem.rstrip() for elem in mydeleted] - - myautoadd = [] - if myunadded: - for x in range(len(myunadded) - 1, -1, -1): - xs = myunadded[x].split("/") - if xs[-1] == "files": - print("!!! files dir is not added! Please correct this.") - sys.exit(-1) - elif xs[-1] == "Manifest": - # It's a manifest... auto add - myautoadd += [myunadded[x]] - del myunadded[x] - - if myunadded: - print(red( - "!!! The following files are in your local tree" - " but are not added to the master")) - print(red( - "!!! tree. Please remove them from the local tree" - " or add them to the master tree.")) - for x in myunadded: - print(" ", x) - print() - print() - sys.exit(1) + if options.mode == "manifest": + sys.exit(dofail) + + # dofail will be true if we have failed in at least one non-warning category + dofail = 0 + # dowarn will be true if we tripped any warnings + dowarn = 0 + # dofull will be true if we should print a "repoman full" informational message + dofull = options.mode != 'full' - if vcs_settings.vcs == "hg" and mydeleted: - print(red( - "!!! The following files are removed manually" - " from your local tree but are not")) - print(red( - "!!! removed from the repository." - " Please remove them, using \"hg remove [FILES]\".")) - for x in mydeleted: - print(" ", x) + for x in qacats: + if x not in qatracker.fails: + continue + dowarn = 1 + if x not in qawarnings: + dofail = 1 + + if dofail or \ + (dowarn and not (options.quiet or options.mode == "scan")): + dofull = 0 + + # Save QA output so that it can be conveniently displayed + # in $EDITOR while the user creates a commit message. + # Otherwise, the user would not be able to see this output + # once the editor has taken over the screen. + qa_output = io.StringIO() + style_file = ConsoleStyleFile(sys.stdout) + if options.mode == 'commit' and \ + (not commitmessage or not commitmessage.strip()): + style_file.write_listener = qa_output + console_writer = StyleWriter(file=style_file, maxcol=9999) + console_writer.style_listener = style_file.new_styles + + f = formatter.AbstractFormatter(console_writer) + + format_outputs = { + 'column': format_qa_output_column, + 'default': format_qa_output + } + + format_output = format_outputs.get( + options.output_style, format_outputs['default']) + format_output(f, qatracker.fails, dofull, dofail, options, qawarnings) + + style_file.flush() + del console_writer, f, style_file + qa_output = qa_output.getvalue() + qa_output = qa_output.splitlines(True) + + suggest_ignore_masked = False + suggest_include_dev = False + + if have_pmasked and not (options.without_mask or options.ignore_masked): + suggest_ignore_masked = True + if have_dev_keywords and not options.include_dev: + suggest_include_dev = True + + if suggest_ignore_masked or suggest_include_dev: print() + if suggest_ignore_masked: + print(bold( + "Note: use --without-mask to check " + "KEYWORDS on dependencies of masked packages")) + + if suggest_include_dev: + print(bold( + "Note: use --include-dev (-d) to check " + "dependencies for 'dev' profiles")) print() - sys.exit(1) - if vcs_settings.vcs == "cvs": - mycvstree = cvstree.getentries("./", recursive=1) - mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./") - mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./") - myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./") - bin_blob_pattern = re.compile("^-kb$") - no_expansion = set(portage.cvstree.findoption( - mycvstree, bin_blob_pattern, recursive=1, basedir="./")) - - if vcs_settings.vcs == "svn": - with repoman_popen("svn status") as f: - svnstatus = f.readlines() - mychanged = [ - "./" + elem.split()[-1:][0] - for elem in svnstatus - if (elem[:1] in "MR" or elem[1:2] in "M")] - mynew = [ - "./" + elem.split()[-1:][0] - for elem in svnstatus - if elem.startswith("A")] - myremoved = [ - "./" + elem.split()[-1:][0] - for elem in svnstatus - if elem.startswith("D")] - - # Subversion expands keywords specified in svn:keywords properties. - with repoman_popen("svn propget -R svn:keywords") as f: - props = f.readlines() - expansion = dict( - ("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) - for prop in props if " - " in prop) - - elif vcs_settings.vcs == "git": - with repoman_popen( - "git diff-index --name-only " - "--relative --diff-filter=M HEAD") as f: - mychanged = f.readlines() - mychanged = ["./" + elem[:-1] for elem in mychanged] - - with repoman_popen( - "git diff-index --name-only " - "--relative --diff-filter=A HEAD") as f: - mynew = f.readlines() - mynew = ["./" + elem[:-1] for elem in mynew] - - with repoman_popen( - "git diff-index --name-only " - "--relative --diff-filter=D HEAD") as f: - myremoved = f.readlines() - myremoved = ["./" + elem[:-1] for elem in myremoved] - - if vcs_settings.vcs == "bzr": - with repoman_popen("bzr status -S .") as f: - bzrstatus = f.readlines() - mychanged = [ - "./" + elem.split()[-1:][0].split('/')[-1:][0] - for elem in bzrstatus - if elem and elem[1:2] == "M"] - mynew = [ - "./" + elem.split()[-1:][0].split('/')[-1:][0] - for elem in bzrstatus - if elem and (elem[1:2] in "NK" or elem[0:1] == "R")] - myremoved = [ - "./" + elem.split()[-1:][0].split('/')[-1:][0] - for elem in bzrstatus - if elem.startswith("-")] - myremoved = [ - "./" + elem.split()[-3:-2][0].split('/')[-1:][0] - for elem in bzrstatus - if elem and (elem[1:2] == "K" or elem[0:1] == "R")] - # Bazaar expands nothing. - - if vcs_settings.vcs == "hg": - with repoman_popen("hg status --no-status --modified .") as f: - mychanged = f.readlines() - mychanged = ["./" + elem.rstrip() for elem in mychanged] - - with repoman_popen("hg status --no-status --added .") as f: - mynew = f.readlines() - mynew = ["./" + elem.rstrip() for elem in mynew] - - with repoman_popen("hg status --no-status --removed .") as f: - myremoved = f.readlines() - myremoved = ["./" + elem.rstrip() for elem in myremoved] - - if vcs_settings.vcs: - a_file_is_changed = mychanged or mynew or myremoved - a_file_is_deleted_hg = vcs_settings.vcs == "hg" and mydeleted - - if not (a_file_is_changed or a_file_is_deleted_hg): + if options.mode != 'commit': + if dofull: + print(bold("Note: type \"repoman full\" for a complete listing.")) + if dowarn and not dofail: + utilities.repoman_sez( + "\"You're only giving me a partial QA payment?\n" + " I'll take it this time, but I'm not happy.\"") + elif not dofail: + utilities.repoman_sez( + "\"If everyone were like you, I'd be out of business!\"") + elif dofail: + print(bad("Please fix these important QA issues first.")) + utilities.repoman_sez( + "\"Make your QA payment on time" + " and you'll never see the likes of me.\"\n") + sys.exit(1) + else: + if dofail and can_force and options.force and not options.pretend: + utilities.repoman_sez( + " \"You want to commit even with these QA issues?\n" + " I'll take it this time, but I'm not happy.\"\n") + elif dofail: + if options.force and not can_force: + print(bad( + "The --force option has been disabled" + " due to extraordinary issues.")) + print(bad("Please fix these important QA issues first.")) + utilities.repoman_sez( + "\"Make your QA payment on time" + " and you'll never see the likes of me.\"\n") + sys.exit(1) + + if options.pretend: utilities.repoman_sez( - "\"Doing nothing is not always good for QA.\"") + "\"So, you want to play it safe. Good call.\"\n") + + myunadded = [] + if vcs_settings.vcs == "cvs": + try: + myvcstree = portage.cvstree.getentries("./", recursive=1) + myunadded = portage.cvstree.findunadded( + myvcstree, recursive=1, basedir="./") + except SystemExit as e: + raise # TODO propagate this + except: + err("Error retrieving CVS tree; exiting.") + if vcs_settings.vcs == "svn": + try: + with repoman_popen("svn status --no-ignore") as f: + svnstatus = f.readlines() + myunadded = [ + "./" + elem.rstrip().split()[1] + for elem in svnstatus + if elem.startswith("?") or elem.startswith("I")] + except SystemExit as e: + raise # TODO propagate this + except: + err("Error retrieving SVN info; exiting.") + if vcs_settings.vcs == "git": + # get list of files not under version control or missing + myf = repoman_popen("git ls-files --others") + myunadded = ["./" + elem[:-1] for elem in myf] + myf.close() + if vcs_settings.vcs == "bzr": + try: + with repoman_popen("bzr status -S .") as f: + bzrstatus = f.readlines() + myunadded = [ + "./" + elem.rstrip().split()[1].split('/')[-1:][0] + for elem in bzrstatus + if elem.startswith("?") or elem[0:2] == " D"] + except SystemExit as e: + raise # TODO propagate this + except: + err("Error retrieving bzr info; exiting.") + if vcs_settings.vcs == "hg": + with repoman_popen("hg status --no-status --unknown .") as f: + myunadded = f.readlines() + myunadded = ["./" + elem.rstrip() for elem in myunadded] + + # Mercurial doesn't handle manually deleted files as removed from + # the repository, so the user need to remove them before commit, + # using "hg remove [FILES]" + with repoman_popen("hg status --no-status --deleted .") as f: + mydeleted = f.readlines() + mydeleted = ["./" + elem.rstrip() for elem in mydeleted] + + myautoadd = [] + if myunadded: + for x in range(len(myunadded) - 1, -1, -1): + xs = myunadded[x].split("/") + if xs[-1] == "files": + print("!!! files dir is not added! Please correct this.") + sys.exit(-1) + elif xs[-1] == "Manifest": + # It's a manifest... auto add + myautoadd += [myunadded[x]] + del myunadded[x] + + if myunadded: + print(red( + "!!! The following files are in your local tree" + " but are not added to the master")) + print(red( + "!!! tree. Please remove them from the local tree" + " or add them to the master tree.")) + for x in myunadded: + print(" ", x) print() - print("(Didn't find any changed files...)") print() sys.exit(1) - # Manifests need to be regenerated after all other commits, so don't commit - # them now even if they have changed. - mymanifests = set() - myupdates = set() - for f in mychanged + mynew: - if "Manifest" == os.path.basename(f): - mymanifests.add(f) - else: - myupdates.add(f) - myupdates.difference_update(myremoved) - myupdates = list(myupdates) - mymanifests = list(mymanifests) - myheaders = [] - mydirty = [] - - commitmessage = options.commitmsg - if options.commitmsgfile: - try: - f = io.open( - _unicode_encode( - options.commitmsgfile, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - commitmessage = f.read() - f.close() - del f - except (IOError, OSError) as e: - if e.errno == errno.ENOENT: - portage.writemsg( - "!!! File Not Found:" - " --commitmsgfile='%s'\n" % options.commitmsgfile) - else: - raise - # We've read the content so the file is no longer needed. - commitmessagefile = None - if not commitmessage or not commitmessage.strip(): - try: - editor = os.environ.get("EDITOR") - if editor and utilities.editor_is_executable(editor): - commitmessage = utilities.get_commit_message_with_editor( - editor, message=qa_output) - else: - commitmessage = utilities.get_commit_message_with_stdin() - except KeyboardInterrupt: - logging.fatal("Interrupted; exiting...") - sys.exit(1) - if not commitmessage or not commitmessage.strip(): - print("* no commit message? aborting commit.") + if vcs_settings.vcs == "hg" and mydeleted: + print(red( + "!!! The following files are removed manually" + " from your local tree but are not")) + print(red( + "!!! removed from the repository." + " Please remove them, using \"hg remove [FILES]\".")) + for x in mydeleted: + print(" ", x) + print() + print() sys.exit(1) - commitmessage = commitmessage.rstrip() - changelog_msg = commitmessage - portage_version = getattr(portage, "VERSION", None) - gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "") - dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "") - if portage_version is None: - sys.stderr.write("Failed to insert portage version in message!\n") - sys.stderr.flush() - portage_version = "Unknown" - - report_options = [] - if options.force: - report_options.append("--force") - if options.ignore_arches: - report_options.append("--ignore-arches") - if include_arches is not None: - report_options.append( - "--include-arches=\"%s\"" % - " ".join(sorted(include_arches))) - - if vcs_settings.vcs == "git": - # Use new footer only for git (see bug #438364). - commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version - if report_options: - commit_footer += "\nRepoMan-Options: " + " ".join(report_options) - if repo_settings.sign_manifests: - commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, ) - if dco_sob: - commit_footer += "\nSigned-off-by: %s" % (dco_sob, ) - else: - unameout = platform.system() + " " - if platform.system() in ["Darwin", "SunOS"]: - unameout += platform.processor() - else: - unameout += platform.machine() - commit_footer = "\n\n" - if dco_sob: - commit_footer += "Signed-off-by: %s\n" % (dco_sob, ) - commit_footer += "(Portage version: %s/%s/%s" % \ - (portage_version, vcs_settings.vcs, unameout) - if report_options: - commit_footer += ", RepoMan options: " + " ".join(report_options) - if repo_settings.sign_manifests: - commit_footer += ", signed Manifest commit with key %s" % \ - (gpg_key, ) - else: - commit_footer += ", unsigned Manifest commit" - commit_footer += ")" - - commitmessage += commit_footer - - broken_changelog_manifests = [] - if options.echangelog in ('y', 'force'): - logging.info("checking for unmodified ChangeLog files") - committer_name = utilities.get_committer_name(env=repoman_settings) - for x in sorted(vcs_files_to_cps( - chain(myupdates, mymanifests, myremoved), - repolevel, reposplit, categories)): - catdir, pkgdir = x.split("/") - checkdir = repo_settings.repodir + "/" + x - checkdir_relative = "" - if repolevel < 3: - checkdir_relative = os.path.join(pkgdir, checkdir_relative) - if repolevel < 2: - checkdir_relative = os.path.join(catdir, checkdir_relative) - checkdir_relative = os.path.join(".", checkdir_relative) - - changelog_path = os.path.join(checkdir_relative, "ChangeLog") - changelog_modified = changelog_path in changed.changelogs - if changelog_modified and options.echangelog != 'force': - continue - # get changes for this package - cdrlen = len(checkdir_relative) - check_relative = lambda e: e.startswith(checkdir_relative) - split_relative = lambda e: e[cdrlen:] - clnew = list(map(split_relative, filter(check_relative, mynew))) - clremoved = list(map(split_relative, filter(check_relative, myremoved))) - clchanged = list(map(split_relative, filter(check_relative, mychanged))) - - # Skip ChangeLog generation if only the Manifest was modified, - # as discussed in bug #398009. - nontrivial_cl_files = set() - nontrivial_cl_files.update(clnew, clremoved, clchanged) - nontrivial_cl_files.difference_update(['Manifest']) - if not nontrivial_cl_files and options.echangelog != 'force': - continue + if vcs_settings.vcs == "cvs": + mycvstree = cvstree.getentries("./", recursive=1) + mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./") + mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./") + myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./") + bin_blob_pattern = re.compile("^-kb$") + no_expansion = set(portage.cvstree.findoption( + mycvstree, bin_blob_pattern, recursive=1, basedir="./")) + + if vcs_settings.vcs == "svn": + with repoman_popen("svn status") as f: + svnstatus = f.readlines() + mychanged = [ + "./" + elem.split()[-1:][0] + for elem in svnstatus + if (elem[:1] in "MR" or elem[1:2] in "M")] + mynew = [ + "./" + elem.split()[-1:][0] + for elem in svnstatus + if elem.startswith("A")] + myremoved = [ + "./" + elem.split()[-1:][0] + for elem in svnstatus + if elem.startswith("D")] + + # Subversion expands keywords specified in svn:keywords properties. + with repoman_popen("svn propget -R svn:keywords") as f: + props = f.readlines() + expansion = dict( + ("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) + for prop in props if " - " in prop) + + elif vcs_settings.vcs == "git": + with repoman_popen( + "git diff-index --name-only " + "--relative --diff-filter=M HEAD") as f: + mychanged = f.readlines() + mychanged = ["./" + elem[:-1] for elem in mychanged] + + with repoman_popen( + "git diff-index --name-only " + "--relative --diff-filter=A HEAD") as f: + mynew = f.readlines() + mynew = ["./" + elem[:-1] for elem in mynew] + + with repoman_popen( + "git diff-index --name-only " + "--relative --diff-filter=D HEAD") as f: + myremoved = f.readlines() + myremoved = ["./" + elem[:-1] for elem in myremoved] + + if vcs_settings.vcs == "bzr": + with repoman_popen("bzr status -S .") as f: + bzrstatus = f.readlines() + mychanged = [ + "./" + elem.split()[-1:][0].split('/')[-1:][0] + for elem in bzrstatus + if elem and elem[1:2] == "M"] + mynew = [ + "./" + elem.split()[-1:][0].split('/')[-1:][0] + for elem in bzrstatus + if elem and (elem[1:2] in "NK" or elem[0:1] == "R")] + myremoved = [ + "./" + elem.split()[-1:][0].split('/')[-1:][0] + for elem in bzrstatus + if elem.startswith("-")] + myremoved = [ + "./" + elem.split()[-3:-2][0].split('/')[-1:][0] + for elem in bzrstatus + if elem and (elem[1:2] == "K" or elem[0:1] == "R")] + # Bazaar expands nothing. - new_changelog = utilities.UpdateChangeLog( - checkdir_relative, committer_name, changelog_msg, - os.path.join(repo_settings.repodir, 'skel.ChangeLog'), - catdir, pkgdir, - new=clnew, removed=clremoved, changed=clchanged, - pretend=options.pretend) - if new_changelog is None: - writemsg_level( - "!!! Updating the ChangeLog failed\n", - level=logging.ERROR, noiselevel=-1) + if vcs_settings.vcs == "hg": + with repoman_popen("hg status --no-status --modified .") as f: + mychanged = f.readlines() + mychanged = ["./" + elem.rstrip() for elem in mychanged] + + with repoman_popen("hg status --no-status --added .") as f: + mynew = f.readlines() + mynew = ["./" + elem.rstrip() for elem in mynew] + + with repoman_popen("hg status --no-status --removed .") as f: + myremoved = f.readlines() + myremoved = ["./" + elem.rstrip() for elem in myremoved] + + if vcs_settings.vcs: + a_file_is_changed = mychanged or mynew or myremoved + a_file_is_deleted_hg = vcs_settings.vcs == "hg" and mydeleted + + if not (a_file_is_changed or a_file_is_deleted_hg): + utilities.repoman_sez( + "\"Doing nothing is not always good for QA.\"") + print() + print("(Didn't find any changed files...)") + print() sys.exit(1) - # if the ChangeLog was just created, add it to vcs - if new_changelog: - myautoadd.append(changelog_path) - # myautoadd is appended to myupdates below + # Manifests need to be regenerated after all other commits, so don't commit + # them now even if they have changed. + mymanifests = set() + myupdates = set() + for f in mychanged + mynew: + if "Manifest" == os.path.basename(f): + mymanifests.add(f) else: - myupdates.append(changelog_path) - - if options.ask and not options.pretend: - # regenerate Manifest for modified ChangeLog (bug #420735) - repoman_settings["O"] = checkdir - digestgen(mysettings=repoman_settings, myportdb=portdb) + myupdates.add(f) + myupdates.difference_update(myremoved) + myupdates = list(myupdates) + mymanifests = list(mymanifests) + myheaders = [] + mydirty = [] + + commitmessage = options.commitmsg + if options.commitmsgfile: + try: + f = io.open( + _unicode_encode( + options.commitmsgfile, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace') + commitmessage = f.read() + f.close() + del f + except (IOError, OSError) as e: + if e.errno == errno.ENOENT: + portage.writemsg( + "!!! File Not Found:" + " --commitmsgfile='%s'\n" % options.commitmsgfile) + else: + raise + # We've read the content so the file is no longer needed. + commitmessagefile = None + if not commitmessage or not commitmessage.strip(): + try: + editor = os.environ.get("EDITOR") + if editor and utilities.editor_is_executable(editor): + commitmessage = utilities.get_commit_message_with_editor( + editor, message=qa_output) + else: + commitmessage = utilities.get_commit_message_with_stdin() + except KeyboardInterrupt: + logging.fatal("Interrupted; exiting...") + sys.exit(1) + if not commitmessage or not commitmessage.strip(): + print("* no commit message? aborting commit.") + sys.exit(1) + commitmessage = commitmessage.rstrip() + changelog_msg = commitmessage + portage_version = getattr(portage, "VERSION", None) + gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "") + dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "") + if portage_version is None: + sys.stderr.write("Failed to insert portage version in message!\n") + sys.stderr.flush() + portage_version = "Unknown" + + report_options = [] + if options.force: + report_options.append("--force") + if options.ignore_arches: + report_options.append("--ignore-arches") + if include_arches is not None: + report_options.append( + "--include-arches=\"%s\"" % + " ".join(sorted(include_arches))) + + if vcs_settings.vcs == "git": + # Use new footer only for git (see bug #438364). + commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version + if report_options: + commit_footer += "\nRepoMan-Options: " + " ".join(report_options) + if repo_settings.sign_manifests: + commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, ) + if dco_sob: + commit_footer += "\nSigned-off-by: %s" % (dco_sob, ) + else: + unameout = platform.system() + " " + if platform.system() in ["Darwin", "SunOS"]: + unameout += platform.processor() else: - broken_changelog_manifests.append(x) + unameout += platform.machine() + commit_footer = "\n\n" + if dco_sob: + commit_footer += "Signed-off-by: %s\n" % (dco_sob, ) + commit_footer += "(Portage version: %s/%s/%s" % \ + (portage_version, vcs_settings.vcs, unameout) + if report_options: + commit_footer += ", RepoMan options: " + " ".join(report_options) + if repo_settings.sign_manifests: + commit_footer += ", signed Manifest commit with key %s" % \ + (gpg_key, ) + else: + commit_footer += ", unsigned Manifest commit" + commit_footer += ")" - if myautoadd: - print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...") - add_cmd = [vcs_settings.vcs, "add"] - add_cmd += myautoadd - if options.pretend: - portage.writemsg_stdout( - "(%s)\n" % " ".join(add_cmd), - noiselevel=-1) - else: + commitmessage += commit_footer - if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ - not os.path.isabs(add_cmd[0]): - # Python 3.1 _execvp throws TypeError for non-absolute executable - # path passed as bytes (see http://bugs.python.org/issue8513). - fullname = find_binary(add_cmd[0]) - if fullname is None: - raise portage.exception.CommandNotFound(add_cmd[0]) - add_cmd[0] = fullname - - add_cmd = [_unicode_encode(arg) for arg in add_cmd] - retcode = subprocess.call(add_cmd) - if retcode != os.EX_OK: - logging.error( - "Exiting on %s error code: %s\n" % (vcs_settings.vcs, retcode)) - sys.exit(retcode) - - myupdates += myautoadd - - print("* %s files being committed..." % green(str(len(myupdates))), end=' ') - - if vcs_settings.vcs not in ('cvs', 'svn'): - # With git, bzr and hg, there's never any keyword expansion, so - # there's no need to regenerate manifests and all files will be - # committed in one big commit at the end. - print() - elif not repo_settings.repo_config.thin_manifest: - if vcs_settings.vcs == 'cvs': - headerstring = "'\$(Header|Id).*\$'" - elif vcs_settings.vcs == "svn": - svn_keywords = dict((k.lower(), k) for k in [ - "Rev", - "Revision", - "LastChangedRevision", - "Date", - "LastChangedDate", - "Author", - "LastChangedBy", - "URL", - "HeadURL", - "Id", - "Header", - ]) - - for myfile in myupdates: - - # for CVS, no_expansion contains files that are excluded from expansion - if vcs_settings.vcs == "cvs": - if myfile in no_expansion: + broken_changelog_manifests = [] + if options.echangelog in ('y', 'force'): + logging.info("checking for unmodified ChangeLog files") + committer_name = utilities.get_committer_name(env=repoman_settings) + for x in sorted(vcs_files_to_cps( + chain(myupdates, mymanifests, myremoved), + repolevel, reposplit, categories)): + catdir, pkgdir = x.split("/") + checkdir = repo_settings.repodir + "/" + x + checkdir_relative = "" + if repolevel < 3: + checkdir_relative = os.path.join(pkgdir, checkdir_relative) + if repolevel < 2: + checkdir_relative = os.path.join(catdir, checkdir_relative) + checkdir_relative = os.path.join(".", checkdir_relative) + + changelog_path = os.path.join(checkdir_relative, "ChangeLog") + changelog_modified = changelog_path in changed.changelogs + if changelog_modified and options.echangelog != 'force': continue - # for SVN, expansion contains files that are included in expansion - elif vcs_settings.vcs == "svn": - if myfile not in expansion: + # get changes for this package + cdrlen = len(checkdir_relative) + check_relative = lambda e: e.startswith(checkdir_relative) + split_relative = lambda e: e[cdrlen:] + clnew = list(map(split_relative, filter(check_relative, mynew))) + clremoved = list(map(split_relative, filter(check_relative, myremoved))) + clchanged = list(map(split_relative, filter(check_relative, mychanged))) + + # Skip ChangeLog generation if only the Manifest was modified, + # as discussed in bug #398009. + nontrivial_cl_files = set() + nontrivial_cl_files.update(clnew, clremoved, clchanged) + nontrivial_cl_files.difference_update(['Manifest']) + if not nontrivial_cl_files and options.echangelog != 'force': continue - # Subversion keywords are case-insensitive - # in svn:keywords properties, - # but case-sensitive in contents of files. - enabled_keywords = [] - for k in expansion[myfile]: - keyword = svn_keywords.get(k.lower()) - if keyword is not None: - enabled_keywords.append(keyword) - - headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords) - - myout = repoman_getstatusoutput( - "egrep -q %s %s" % (headerstring, portage._shell_quote(myfile))) - if myout[0] == 0: - myheaders.append(myfile) - - print("%s have headers that will change." % green(str(len(myheaders)))) - print( - "* Files with headers will" - " cause the manifests to be changed and committed separately.") - - logging.info("myupdates: %s", myupdates) - logging.info("myheaders: %s", myheaders) - - uq = UserQuery(options) - if options.ask and uq.query('Commit changes?', True) != 'Yes': - print("* aborting commit.") - sys.exit(128 + signal.SIGINT) - - # Handle the case where committed files have keywords which - # will change and need a priming commit before the Manifest - # can be committed. - if (myupdates or myremoved) and myheaders: - myfiles = myupdates + myremoved - fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") - mymsg = os.fdopen(fd, "wb") - mymsg.write(_unicode_encode(commitmessage)) - mymsg.close() - - separator = '-' * 78 - - print() - print(green("Using commit message:")) - print(green(separator)) - print(commitmessage) - print(green(separator)) - print() + new_changelog = utilities.UpdateChangeLog( + checkdir_relative, committer_name, changelog_msg, + os.path.join(repo_settings.repodir, 'skel.ChangeLog'), + catdir, pkgdir, + new=clnew, removed=clremoved, changed=clchanged, + pretend=options.pretend) + if new_changelog is None: + writemsg_level( + "!!! Updating the ChangeLog failed\n", + level=logging.ERROR, noiselevel=-1) + sys.exit(1) - # Having a leading ./ prefix on file paths can trigger a bug in - # the cvs server when committing files to multiple directories, - # so strip the prefix. - myfiles = [f.lstrip("./") for f in myfiles] + # if the ChangeLog was just created, add it to vcs + if new_changelog: + myautoadd.append(changelog_path) + # myautoadd is appended to myupdates below + else: + myupdates.append(changelog_path) - commit_cmd = [vcs_settings.vcs] - commit_cmd.extend(vcs_settings.vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(vcs_settings.vcs_local_opts) - commit_cmd.extend(["-F", commitmessagefile]) - commit_cmd.extend(myfiles) + if options.ask and not options.pretend: + # regenerate Manifest for modified ChangeLog (bug #420735) + repoman_settings["O"] = checkdir + digestgen(mysettings=repoman_settings, myportdb=portdb) + else: + broken_changelog_manifests.append(x) - try: + if myautoadd: + print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...") + add_cmd = [vcs_settings.vcs, "add"] + add_cmd += myautoadd if options.pretend: - print("(%s)" % (" ".join(commit_cmd),)) + portage.writemsg_stdout( + "(%s)\n" % " ".join(add_cmd), + noiselevel=-1) else: - retval = spawn(commit_cmd, env=repo_settings.commit_env) - if retval != os.EX_OK: - writemsg_level( - "!!! Exiting on %s (shell) " - "error code: %s\n" % (vcs_settings.vcs, retval), - level=logging.ERROR, noiselevel=-1) - sys.exit(retval) - finally: - try: - os.unlink(commitmessagefile) - except OSError: - pass - # Setup the GPG commands - def gpgsign(filename): - gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND") - if gpgcmd is None: - raise MissingParameter( - "PORTAGE_GPG_SIGNING_COMMAND is unset!" - " Is make.globals missing?") - if "${PORTAGE_GPG_KEY}" in gpgcmd and \ - "PORTAGE_GPG_KEY" not in repoman_settings: - raise MissingParameter("PORTAGE_GPG_KEY is unset!") - if "${PORTAGE_GPG_DIR}" in gpgcmd: - if "PORTAGE_GPG_DIR" not in repoman_settings: - repoman_settings["PORTAGE_GPG_DIR"] = \ - os.path.expanduser("~/.gnupg") - logging.info( - "Automatically setting PORTAGE_GPG_DIR to '%s'" % - repoman_settings["PORTAGE_GPG_DIR"]) - else: - repoman_settings["PORTAGE_GPG_DIR"] = \ - os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"]) - if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK): - raise portage.exception.InvalidLocation( - "Unable to access directory: PORTAGE_GPG_DIR='%s'" % - repoman_settings["PORTAGE_GPG_DIR"]) - gpgvars = {"FILE": filename} - for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"): - v = repoman_settings.get(k) - if v is not None: - gpgvars[k] = v - gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars) - if options.pretend: - print("(" + gpgcmd + ")") - else: - # Encode unicode manually for bug #310789. - gpgcmd = portage.util.shlex_split(gpgcmd) - - if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ - not os.path.isabs(gpgcmd[0]): - # Python 3.1 _execvp throws TypeError for non-absolute executable - # path passed as bytes (see http://bugs.python.org/issue8513). - fullname = find_binary(gpgcmd[0]) - if fullname is None: - raise portage.exception.CommandNotFound(gpgcmd[0]) - gpgcmd[0] = fullname - - gpgcmd = [ - _unicode_encode(arg, encoding=_encodings['fs'], errors='strict') - for arg in gpgcmd] - rValue = subprocess.call(gpgcmd) - if rValue == os.EX_OK: - os.rename(filename + ".asc", filename) - else: - raise portage.exception.PortageException( - "!!! gpg exited with '" + str(rValue) + "' status") + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(add_cmd[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(add_cmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(add_cmd[0]) + add_cmd[0] = fullname + + add_cmd = [_unicode_encode(arg) for arg in add_cmd] + retcode = subprocess.call(add_cmd) + if retcode != os.EX_OK: + logging.error( + "Exiting on %s error code: %s\n" % (vcs_settings.vcs, retcode)) + sys.exit(retcode) + + myupdates += myautoadd + + print("* %s files being committed..." % green(str(len(myupdates))), end=' ') + + if vcs_settings.vcs not in ('cvs', 'svn'): + # With git, bzr and hg, there's never any keyword expansion, so + # there's no need to regenerate manifests and all files will be + # committed in one big commit at the end. + print() + elif not repo_settings.repo_config.thin_manifest: + if vcs_settings.vcs == 'cvs': + headerstring = "'\$(Header|Id).*\$'" + elif vcs_settings.vcs == "svn": + svn_keywords = dict((k.lower(), k) for k in [ + "Rev", + "Revision", + "LastChangedRevision", + "Date", + "LastChangedDate", + "Author", + "LastChangedBy", + "URL", + "HeadURL", + "Id", + "Header", + ]) + + for myfile in myupdates: + + # for CVS, no_expansion contains files that are excluded from expansion + if vcs_settings.vcs == "cvs": + if myfile in no_expansion: + continue - def need_signature(filename): - try: - with open( - _unicode_encode( - filename, encoding=_encodings['fs'], errors='strict'), - 'rb') as f: - return b"BEGIN PGP SIGNED MESSAGE" not in f.readline() - except IOError as e: - if e.errno in (errno.ENOENT, errno.ESTALE): - return False - raise + # for SVN, expansion contains files that are included in expansion + elif vcs_settings.vcs == "svn": + if myfile not in expansion: + continue - # When files are removed and re-added, the cvs server will put /Attic/ - # inside the $Header path. This code detects the problem and corrects it - # so that the Manifest will generate correctly. See bug #169500. - # Use binary mode in order to avoid potential character encoding issues. - cvs_header_re = re.compile(br'^#\s*\$Header.*\$$') - attic_str = b'/Attic/' - attic_replace = b'/' - for x in myheaders: - f = open( - _unicode_encode(x, encoding=_encodings['fs'], errors='strict'), - mode='rb') - mylines = f.readlines() - f.close() - modified = False - for i, line in enumerate(mylines): - if cvs_header_re.match(line) is not None and \ - attic_str in line: - mylines[i] = line.replace(attic_str, attic_replace) - modified = True - if modified: - portage.util.write_atomic(x, b''.join(mylines), mode='wb') + # Subversion keywords are case-insensitive + # in svn:keywords properties, + # but case-sensitive in contents of files. + enabled_keywords = [] + for k in expansion[myfile]: + keyword = svn_keywords.get(k.lower()) + if keyword is not None: + enabled_keywords.append(keyword) + + headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords) + + myout = repoman_getstatusoutput( + "egrep -q %s %s" % (headerstring, portage._shell_quote(myfile))) + if myout[0] == 0: + myheaders.append(myfile) + + print("%s have headers that will change." % green(str(len(myheaders)))) + print( + "* Files with headers will" + " cause the manifests to be changed and committed separately.") + + logging.info("myupdates: %s", myupdates) + logging.info("myheaders: %s", myheaders) + + uq = UserQuery(options) + if options.ask and uq.query('Commit changes?', True) != 'Yes': + print("* aborting commit.") + sys.exit(128 + signal.SIGINT) + + # Handle the case where committed files have keywords which + # will change and need a priming commit before the Manifest + # can be committed. + if (myupdates or myremoved) and myheaders: + myfiles = myupdates + myremoved + fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") + mymsg = os.fdopen(fd, "wb") + mymsg.write(_unicode_encode(commitmessage)) + mymsg.close() + + separator = '-' * 78 - if repolevel == 1: - utilities.repoman_sez( - "\"You're rather crazy... " - "doing the entire repository.\"\n") - - if vcs_settings.vcs in ('cvs', 'svn') and (myupdates or myremoved): - for x in sorted(vcs_files_to_cps( - chain(myupdates, myremoved, mymanifests), - repolevel, reposplit, categories)): - repoman_settings["O"] = os.path.join(repo_settings.repodir, x) - digestgen(mysettings=repoman_settings, myportdb=portdb) - - elif broken_changelog_manifests: - for x in broken_changelog_manifests: - repoman_settings["O"] = os.path.join(repo_settings.repodir, x) - digestgen(mysettings=repoman_settings, myportdb=portdb) - - signed = False - if repo_settings.sign_manifests: - signed = True - try: + print() + print(green("Using commit message:")) + print(green(separator)) + print(commitmessage) + print(green(separator)) + print() + + # Having a leading ./ prefix on file paths can trigger a bug in + # the cvs server when committing files to multiple directories, + # so strip the prefix. + myfiles = [f.lstrip("./") for f in myfiles] + + commit_cmd = [vcs_settings.vcs] + commit_cmd.extend(vcs_settings.vcs_global_opts) + commit_cmd.append("commit") + commit_cmd.extend(vcs_settings.vcs_local_opts) + commit_cmd.extend(["-F", commitmessagefile]) + commit_cmd.extend(myfiles) + + try: + if options.pretend: + print("(%s)" % (" ".join(commit_cmd),)) + else: + retval = spawn(commit_cmd, env=repo_settings.commit_env) + if retval != os.EX_OK: + writemsg_level( + "!!! Exiting on %s (shell) " + "error code: %s\n" % (vcs_settings.vcs, retval), + level=logging.ERROR, noiselevel=-1) + sys.exit(retval) + finally: + try: + os.unlink(commitmessagefile) + except OSError: + pass + + # When files are removed and re-added, the cvs server will put /Attic/ + # inside the $Header path. This code detects the problem and corrects it + # so that the Manifest will generate correctly. See bug #169500. + # Use binary mode in order to avoid potential character encoding issues. + cvs_header_re = re.compile(br'^#\s*\$Header.*\$$') + attic_str = b'/Attic/' + attic_replace = b'/' + for x in myheaders: + f = open( + _unicode_encode(x, encoding=_encodings['fs'], errors='strict'), + mode='rb') + mylines = f.readlines() + f.close() + modified = False + for i, line in enumerate(mylines): + if cvs_header_re.match(line) is not None and \ + attic_str in line: + mylines[i] = line.replace(attic_str, attic_replace) + modified = True + if modified: + portage.util.write_atomic(x, b''.join(mylines), mode='wb') + + if repolevel == 1: + utilities.repoman_sez( + "\"You're rather crazy... " + "doing the entire repository.\"\n") + + if vcs_settings.vcs in ('cvs', 'svn') and (myupdates or myremoved): for x in sorted(vcs_files_to_cps( chain(myupdates, myremoved, mymanifests), repolevel, reposplit, categories)): repoman_settings["O"] = os.path.join(repo_settings.repodir, x) - manifest_path = os.path.join(repoman_settings["O"], "Manifest") - if not need_signature(manifest_path): - continue - gpgsign(manifest_path) - except portage.exception.PortageException as e: - portage.writemsg("!!! %s\n" % str(e)) - portage.writemsg("!!! Disabled FEATURES='sign'\n") - signed = False - - if vcs_settings.vcs == 'git': - # It's not safe to use the git commit -a option since there might - # be some modified files elsewhere in the working tree that the - # user doesn't want to commit. Therefore, call git update-index - # in order to ensure that the index is updated with the latest - # versions of all new and modified files in the relevant portion - # of the working tree. - myfiles = mymanifests + myupdates - myfiles.sort() - update_index_cmd = ["git", "update-index"] - update_index_cmd.extend(f.lstrip("./") for f in myfiles) - if options.pretend: - print("(%s)" % (" ".join(update_index_cmd),)) - else: - retval = spawn(update_index_cmd, env=os.environ) - if retval != os.EX_OK: - writemsg_level( - "!!! Exiting on %s (shell) " - "error code: %s\n" % (vcs_settings.vcs, retval), - level=logging.ERROR, noiselevel=-1) - sys.exit(retval) - - if True: - myfiles = mymanifests[:] - # If there are no header (SVN/CVS keywords) changes in - # the files, this Manifest commit must include the - # other (yet uncommitted) files. - if not myheaders: - myfiles += myupdates - myfiles += myremoved - myfiles.sort() - - fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") - mymsg = os.fdopen(fd, "wb") - mymsg.write(_unicode_encode(commitmessage)) - mymsg.close() - - commit_cmd = [] - if options.pretend and vcs_settings.vcs is None: - # substitute a bogus value for pretend output - commit_cmd.append("cvs") - else: - commit_cmd.append(vcs_settings.vcs) - commit_cmd.extend(vcs_settings.vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(vcs_settings.vcs_local_opts) - if vcs_settings.vcs == "hg": - commit_cmd.extend(["--logfile", commitmessagefile]) - commit_cmd.extend(myfiles) - else: - commit_cmd.extend(["-F", commitmessagefile]) - commit_cmd.extend(f.lstrip("./") for f in myfiles) + digestgen(mysettings=repoman_settings, myportdb=portdb) - try: + elif broken_changelog_manifests: + for x in broken_changelog_manifests: + repoman_settings["O"] = os.path.join(repo_settings.repodir, x) + digestgen(mysettings=repoman_settings, myportdb=portdb) + + signed = False + if repo_settings.sign_manifests: + signed = True + try: + for x in sorted(vcs_files_to_cps( + chain(myupdates, myremoved, mymanifests), + repolevel, reposplit, categories)): + repoman_settings["O"] = os.path.join(repo_settings.repodir, x) + manifest_path = os.path.join(repoman_settings["O"], "Manifest") + if not need_signature(manifest_path): + continue + gpgsign(manifest_path) + except portage.exception.PortageException as e: + portage.writemsg("!!! %s\n" % str(e)) + portage.writemsg("!!! Disabled FEATURES='sign'\n") + signed = False + + if vcs_settings.vcs == 'git': + # It's not safe to use the git commit -a option since there might + # be some modified files elsewhere in the working tree that the + # user doesn't want to commit. Therefore, call git update-index + # in order to ensure that the index is updated with the latest + # versions of all new and modified files in the relevant portion + # of the working tree. + myfiles = mymanifests + myupdates + myfiles.sort() + update_index_cmd = ["git", "update-index"] + update_index_cmd.extend(f.lstrip("./") for f in myfiles) if options.pretend: - print("(%s)" % (" ".join(commit_cmd),)) + print("(%s)" % (" ".join(update_index_cmd),)) else: - retval = spawn(commit_cmd, env=repo_settings.commit_env) + retval = spawn(update_index_cmd, env=os.environ) if retval != os.EX_OK: - if repo_settings.repo_config.sign_commit and vcs_settings.vcs == 'git' and \ - not git_supports_gpg_sign(): - # Inform user that newer git is needed (bug #403323). - logging.error( - "Git >=1.7.9 is required for signed commits!") - writemsg_level( "!!! Exiting on %s (shell) " "error code: %s\n" % (vcs_settings.vcs, retval), level=logging.ERROR, noiselevel=-1) sys.exit(retval) - finally: + + if True: + myfiles = mymanifests[:] + # If there are no header (SVN/CVS keywords) changes in + # the files, this Manifest commit must include the + # other (yet uncommitted) files. + if not myheaders: + myfiles += myupdates + myfiles += myremoved + myfiles.sort() + + fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") + mymsg = os.fdopen(fd, "wb") + mymsg.write(_unicode_encode(commitmessage)) + mymsg.close() + + commit_cmd = [] + if options.pretend and vcs_settings.vcs is None: + # substitute a bogus value for pretend output + commit_cmd.append("cvs") + else: + commit_cmd.append(vcs_settings.vcs) + commit_cmd.extend(vcs_settings.vcs_global_opts) + commit_cmd.append("commit") + commit_cmd.extend(vcs_settings.vcs_local_opts) + if vcs_settings.vcs == "hg": + commit_cmd.extend(["--logfile", commitmessagefile]) + commit_cmd.extend(myfiles) + else: + commit_cmd.extend(["-F", commitmessagefile]) + commit_cmd.extend(f.lstrip("./") for f in myfiles) + try: - os.unlink(commitmessagefile) - except OSError: - pass + if options.pretend: + print("(%s)" % (" ".join(commit_cmd),)) + else: + retval = spawn(commit_cmd, env=repo_settings.commit_env) + if retval != os.EX_OK: + if repo_settings.repo_config.sign_commit and vcs_settings.vcs == 'git' and \ + not git_supports_gpg_sign(): + # Inform user that newer git is needed (bug #403323). + logging.error( + "Git >=1.7.9 is required for signed commits!") + + writemsg_level( + "!!! Exiting on %s (shell) " + "error code: %s\n" % (vcs_settings.vcs, retval), + level=logging.ERROR, noiselevel=-1) + sys.exit(retval) + finally: + try: + os.unlink(commitmessagefile) + except OSError: + pass - print() - if vcs_settings.vcs: - print("Commit complete.") - else: - print( - "repoman was too scared" - " by not seeing any familiar version control file" - " that he forgot to commit anything") - utilities.repoman_sez( - "\"If everyone were like you, I'd be out of business!\"\n") -sys.exit(0) + print() + if vcs_settings.vcs: + print("Commit complete.") + else: + print( + "repoman was too scared" + " by not seeing any familiar version control file" + " that he forgot to commit anything") + utilities.repoman_sez( + "\"If everyone were like you, I'd be out of business!\"\n") + sys.exit(0) diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py index fb96e13..b860f51 100644 --- a/tbc/pym/updatedb.py +++ b/tbc/pym/updatedb.py @@ -6,6 +6,7 @@ import sys import os import multiprocessing import time +import re import portage from portage.xml.metadata import MetaDataXML from sqlalchemy.orm import scoped_session, sessionmaker @@ -58,7 +59,7 @@ def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id): package = element[1] # update the categories table - repodir = self._myportdb.getRepositoryPath('gentoo') + repodir = myportdb.getRepositoryPath('gentoo') pkgdir = repodir + "/" + categories categories_metadataDict = get_categories_metadataDict(pkgdir) update_categories_db(session2, categories, categories_metadataDict)