* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, tbc/pym/repoman/
@ 2015-07-30 11:37 Magnus Granberg
0 siblings, 0 replies; 2+ messages in thread
From: Magnus Granberg @ 2015-07-30 11:37 UTC (permalink / raw
To: gentoo-commits
commit: 801894358cc5569b03844bcd043931d441fb8508
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 30 11:36:57 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jul 30 11:36:57 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=80189435
add repoman full scan and split repoman/main to more functions
tbc/pym/package.py | 9 +-
tbc/pym/qachecks.py | 79 ++++++------
tbc/pym/repoman/main.py | 278 +++++++++++++++++++++----------------------
tbc/pym/repoman/utilities.py | 4 +-
4 files changed, 191 insertions(+), 179 deletions(-)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index ba58e2e..0a19f15 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -8,7 +8,7 @@ from portage.xml.metadata import MetaDataXML
from tbc.flags import tbc_use_flags
from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman
+from tbc.qachecks import digestcheck, check_repoman, repoman_full
from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
@@ -282,6 +282,10 @@ class tbc_package(object):
manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
if manifest_checksum_tree is None:
return
+
+ # Check cp with repoman repoman full
+ repoman_full(self._session, pkgdir, self._config_id)
+
package_id = add_new_package_sql(self._session, cp, repo)
package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
@@ -336,6 +340,9 @@ class tbc_package(object):
log_msg = "U %s:%s" % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
+ # Check cp with repoman repoman full
+ repoman_full(self._session, pkgdir, self._config_id)
+
# Get the ebuild list for cp
old_ebuild_id_list = []
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py
index 8c522c6..2e25d86 100644
--- a/tbc/pym/qachecks.py
+++ b/tbc/pym/qachecks.py
@@ -15,6 +15,8 @@ from portage.exception import DigestException, FileNotFound, ParseError, Permiss
from _emerge.Package import Package
from _emerge.RootConfig import RootConfig
from tbc.repoman.checks.ebuilds.checks import run_checks
+from tbc.repoman.main import repoman_main
+from tbc.sqlquerys import get_configmetadata_info, get_config_info, get_setup_info
import portage
# Copy of portage.digestcheck() but without the writemsg() stuff
@@ -142,38 +144,47 @@ def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list
return
def check_repoman(mysettings, myportdb, cpv, repo):
- # We run repoman run_checks on the ebuild
- ebuild_version_tree = portage.versions.cpv_getversion(cpv)
- element = portage.versions.cpv_getkey(cpv).split('/')
- categories = element[0]
- package = element[1]
- pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
- full_path = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild"
- root = '/'
- trees = {
- root : {'porttree' : portage.portagetree(root, settings=mysettings)}
- }
- root_config = RootConfig(mysettings, trees[root], None)
- allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
- allvars.update(Package.metadata_keys)
- allvars = sorted(allvars)
- myaux = dict(zip(allvars, myportdb.aux_get(cpv, allvars, myrepo=repo)))
- pkg = Package(cpv=cpv, metadata=myaux, root_config=root_config, type_name='ebuild')
- fails = []
+ # We run repoman run_checks on the ebuild
+ ebuild_version_tree = portage.versions.cpv_getversion(cpv)
+ element = portage.versions.cpv_getkey(cpv).split('/')
+ categories = element[0]
+ package = element[1]
+ pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
+ full_path = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild"
+ root = '/'
+ trees = {
+ root : {'porttree' : portage.portagetree(root, settings=mysettings)}
+ }
+ root_config = RootConfig(mysettings, trees[root], None)
+ allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
+ allvars.update(Package.metadata_keys)
+ allvars = sorted(allvars)
+ myaux = dict(zip(allvars, myportdb.aux_get(cpv, allvars, myrepo=repo)))
+ pkg = Package(cpv=cpv, metadata=myaux, root_config=root_config, type_name='ebuild')
+ fails = []
+ try:
+ # All ebuilds should have utf_8 encoding.
+ f = codecs.open(_unicode_encode(full_path,
+ encoding = _encodings['fs'], errors = 'strict'),
+ mode = 'r', encoding = _encodings['repo.content'])
try:
- # All ebuilds should have utf_8 encoding.
- f = codecs.open(_unicode_encode(full_path,
- encoding = _encodings['fs'], errors = 'strict'),
- mode = 'r', encoding = _encodings['repo.content'])
- try:
- for check_name, e in run_checks(f, pkg):
- fails.append(check_name + ": " + e + "\n")
- finally:
- f.close()
- except UnicodeDecodeError:
- # A file.UTF8 failure will have already been recorded above.
- pass
- # fails will have a list with repoman errors
- if fails == []:
- return False
- return fails
\ No newline at end of file
+ for check_name, e in run_checks(f, pkg):
+ fails.append(check_name + ": " + e + "\n")
+ finally:
+ f.close()
+ except UnicodeDecodeError:
+ # A file.UTF8 failure will have already been recorded above.
+ pass
+ # fails will have a list with repoman errors
+ if fails == []:
+ return False
+ return fails
+
+def repoman_full(session, pkgdir, config_id):
+ ConfigsMetaData = get_configmetadata_info(session, config_id)
+ ConfigInfo = get_config_info(session, config_id)
+ SetupInfo = get_setup_info(session, config_id)
+ config_root = ConfigsMetaData.RepoPath + '/' + ConfigInfo.Hostname + "/" + SetupInfo.Setup
+ argscmd = []
+ argscmd.append('full')
+ repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir)
diff --git a/tbc/pym/repoman/main.py b/tbc/pym/repoman/main.py
index 2196009..5bdee0e 100755
--- a/tbc/pym/repoman/main.py
+++ b/tbc/pym/repoman/main.py
@@ -1,5 +1,4 @@
-#!/usr/bin/python -bO
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function, unicode_literals
@@ -165,113 +164,16 @@ def need_signature(filename):
return False
raise
-def repoman_main(argv):
- # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
- # behave incrementally.
- repoman_incrementals = tuple(
- x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
- config_root = os.environ.get("PORTAGE_CONFIGROOT")
- repoman_settings = portage.config(config_root=config_root, local_config=False)
-
- if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
- repoman_settings.get('TERM') == 'dumb' or \
- not sys.stdout.isatty():
- nocolor()
-
- options, arguments = parse_args(
- argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
-
- if options.version:
- print("Portage", portage.VERSION)
- sys.exit(0)
-
- if options.experimental_inherit == 'y':
- # This is experimental, so it's non-fatal.
- qawarnings.add("inherit.missing")
- checks_init(experimental_inherit=True)
-
- # Set this to False when an extraordinary issue (generally
- # something other than a QA issue) makes it impossible to
- # commit (like if Manifest generation fails).
- can_force = True
-
- portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
- if portdir is None:
- sys.exit(1)
-
- myreporoot = os.path.basename(portdir_overlay)
- myreporoot += mydir[len(portdir_overlay):]
- ##################
-
- vcs_settings = VCSSettings(options, repoman_settings)
-
- ##################
-
- repo_settings = RepoSettings(
- config_root, portdir, portdir_overlay,
- repoman_settings, vcs_settings, options, qawarnings)
-
- repoman_settings = repo_settings.repoman_settings
-
- portdb = repo_settings.portdb
- ##################
-
- if options.echangelog is None and repo_settings.repo_config.update_changelog:
- options.echangelog = 'y'
-
- if vcs_settings.vcs is None:
- options.echangelog = 'n'
-
- # The --echangelog option causes automatic ChangeLog generation,
- # which invalidates changelog.ebuildadded and changelog.missing
- # checks.
- # Note: Some don't use ChangeLogs in distributed SCMs.
- # It will be generated on server side from scm log,
- # before package moves to the rsync server.
- # This is needed because they try to avoid merge collisions.
- # Gentoo's Council decided to always use the ChangeLog file.
- # TODO: shouldn't this just be switched on the repo, iso the VCS?
- is_echangelog_enabled = options.echangelog in ('y', 'force')
- vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
- check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn
+def repoman_scan(repoman_settings, repo_settings, vcs_settings, options, repolevel):
- if 'digest' in repoman_settings.features and options.digest != 'n':
- options.digest = 'y'
-
- logging.debug("vcs: %s" % (vcs_settings.vcs,))
- logging.debug("repo config: %s" % (repo_settings.repo_config,))
- logging.debug("options: %s" % (options,))
-
- # It's confusing if these warnings are displayed without the user
- # being told which profile they come from, so disable them.
- env = os.environ.copy()
- env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
-
- categories = []
- for path in repo_settings.repo_config.eclass_db.porttrees:
- categories.extend(portage.util.grabfile(
- os.path.join(path, 'profiles', 'categories')))
- repoman_settings.categories = frozenset(
- portage.util.stack_lists([categories], incremental=1))
- categories = repoman_settings.categories
-
- portdb.settings = repoman_settings
- # We really only need to cache the metadata that's necessary for visibility
- # filtering. Anything else can be discarded to reduce memory consumption.
- portdb._aux_cache_keys.clear()
- portdb._aux_cache_keys.update(
- ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
-
- reposplit = myreporoot.split(os.path.sep)
- repolevel = len(reposplit)
-
- ###################
-
- if options.mode == 'commit':
- repochecks.commit_check(repolevel, reposplit)
- repochecks.conflict_check(vcs_settings, options)
+ if options.mode == "manifest":
+ pass
+ elif options.pretend:
+ print(green("\nRepoMan does a once-over of the neighborhood..."))
+ else:
+ print(green("\nRepoMan scours the neighborhood..."))
- ###################
+ qatracker = QATracker()
# Make startdir relative to the canonical repodir, so that we can pass
# it to digestgen and it won't have to be canonicalized again.
@@ -281,7 +183,9 @@ def repoman_main(argv):
startdir = normalize_path(mydir)
startdir = os.path.join(
repo_settings.repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
- ###################
+
+ have_pmasked = False
+ have_dev_keywords = False
# get lists of valid keywords, licenses, and use
new_data = repo_metadata(repo_settings.portdb, repoman_settings)
@@ -305,28 +209,12 @@ def repoman_main(argv):
####################
- dev_keywords = dev_keywords(profiles)
-
- qatracker = QATracker()
-
-
- if options.mode == "manifest":
- pass
- elif options.pretend:
- print(green("\nRepoMan does a once-over of the neighborhood..."))
- else:
- print(green("\nRepoMan scours the neighborhood..."))
-
- #####################
-
- changed = Changes(options)
- changed.scan(vcs_settings)
-
- ######################
+ dev_keyword = dev_keywords(profiles)
- have_pmasked = False
- have_dev_keywords = False
- dofail = 0
+ # Disable the "ebuild.notadded" check when not in commit mode and
+ # running `svn status` in every package dir will be too expensive.
+ check_ebuild_notadded = not \
+ (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode != "commit")
# NOTE: match-all caches are not shared due to potential
# differences between profiles in _get_implicit_iuse.
@@ -339,19 +227,6 @@ def repoman_main(argv):
include_arches = set()
include_arches.update(*[x.split() for x in options.include_arches])
- # Disable the "ebuild.notadded" check when not in commit mode and
- # running `svn status` in every package dir will be too expensive.
-
- check_ebuild_notadded = not \
- (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode != "commit")
-
- effective_scanlist = scanlist
- if options.if_modified == "y":
- effective_scanlist = sorted(vcs_files_to_cps(
- chain(changed.changed, changed.new, changed.removed),
- repolevel, reposplit, categories))
-
- ######################
# initialize our checks classes here before the big xpkg loop
manifester = Manifests(options, qatracker, repoman_settings)
is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker)
@@ -814,7 +689,7 @@ def repoman_main(argv):
if not have_dev_keywords:
have_dev_keywords = \
- bool(dev_keywords.intersection(keywords))
+ bool(dev_keyword.intersection(keywords))
if prof.status == "dev":
suffix = suffix + "indev"
@@ -890,6 +765,123 @@ def repoman_main(argv):
"metadata.warning",
"%s/metadata.xml: unused local USE-description: '%s'"
% (xpkg, myflag))
+ return qatracker
+
+def repoman_main(argv, config_root=None, pkgdir=None):
+ # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
+ # behave incrementally.
+ repoman_incrementals = tuple(
+ x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
+ if config_root is None:
+ config_root = os.environ.get("PORTAGE_CONFIGROOT")
+ repoman_settings = portage.config(config_root=config_root, local_config=False)
+
+ if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
+ repoman_settings.get('TERM') == 'dumb' or \
+ not sys.stdout.isatty():
+ nocolor()
+
+ options, arguments = parse_args(
+ argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
+
+ if options.version:
+ print("Portage", portage.VERSION)
+ sys.exit(0)
+
+ if options.experimental_inherit == 'y':
+ # This is experimental, so it's non-fatal.
+ qawarnings.add("inherit.missing")
+ checks_init(experimental_inherit=True)
+
+ # Set this to False when an extraordinary issue (generally
+ # something other than a QA issue) makes it impossible to
+ # commit (like if Manifest generation fails).
+ can_force = True
+
+ portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings, pkgdir)
+ if portdir is None:
+ sys.exit(1)
+
+ myreporoot = os.path.basename(portdir_overlay)
+ myreporoot += mydir[len(portdir_overlay):]
+ ##################
+
+ vcs_settings = VCSSettings(options, repoman_settings)
+
+ ##################
+
+ repo_settings = RepoSettings(
+ config_root, portdir, portdir_overlay,
+ repoman_settings, vcs_settings, options, qawarnings)
+
+ repoman_settings = repo_settings.repoman_settings
+
+ portdb = repo_settings.portdb
+ ##################
+
+ if options.echangelog is None and repo_settings.repo_config.update_changelog:
+ options.echangelog = 'y'
+
+ if vcs_settings.vcs is None:
+ options.echangelog = 'n'
+
+ # The --echangelog option causes automatic ChangeLog generation,
+ # which invalidates changelog.ebuildadded and changelog.missing
+ # checks.
+ # Note: Some don't use ChangeLogs in distributed SCMs.
+ # It will be generated on server side from scm log,
+ # before package moves to the rsync server.
+ # This is needed because they try to avoid merge collisions.
+ # Gentoo's Council decided to always use the ChangeLog file.
+ # TODO: shouldn't this just be switched on the repo, iso the VCS?
+ is_echangelog_enabled = options.echangelog in ('y', 'force')
+ vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
+ check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn
+
+ if 'digest' in repoman_settings.features and options.digest != 'n':
+ options.digest = 'y'
+
+ logging.debug("vcs: %s" % (vcs_settings.vcs,))
+ logging.debug("repo config: %s" % (repo_settings.repo_config,))
+ logging.debug("options: %s" % (options,))
+
+ # It's confusing if these warnings are displayed without the user
+ # being told which profile they come from, so disable them.
+ env = os.environ.copy()
+ env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
+
+ categories = []
+ for path in repo_settings.repo_config.eclass_db.porttrees:
+ categories.extend(portage.util.grabfile(
+ os.path.join(path, 'profiles', 'categories')))
+ repoman_settings.categories = frozenset(
+ portage.util.stack_lists([categories], incremental=1))
+ categories = repoman_settings.categories
+
+ portdb.settings = repoman_settings
+ # We really only need to cache the metadata that's necessary for visibility
+ # filtering. Anything else can be discarded to reduce memory consumption.
+ portdb._aux_cache_keys.clear()
+ portdb._aux_cache_keys.update(
+ ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
+
+ reposplit = myreporoot.split(os.path.sep)
+ repolevel = len(reposplit)
+
+ if options.mode == 'commit':
+ repochecks.commit_check(repolevel, reposplit)
+ repochecks.conflict_check(vcs_settings, options)
+
+ changed = Changes(options)
+ changed.scan(vcs_settings)
+
+ effective_scanlist = scanlist
+ if options.if_modified == "y":
+ effective_scanlist = sorted(vcs_files_to_cps(
+ chain(changed.changed, changed.new, changed.removed),
+ repolevel, reposplit, categories))
+
+ qatracker = repoman_scan(repoman_settings, repo_settings, vcs_settings, options, repolevel)
if options.if_modified == "y" and len(effective_scanlist) < 1:
logging.warn("--if-modified is enabled, but no modified packages were found!")
diff --git a/tbc/pym/repoman/utilities.py b/tbc/pym/repoman/utilities.py
index 5e43851..b1871db 100644
--- a/tbc/pym/repoman/utilities.py
+++ b/tbc/pym/repoman/utilities.py
@@ -240,7 +240,7 @@ def get_commit_message_with_stdin():
return commitmessage
-def FindPortdir(settings):
+def FindPortdir(settings, pkdir):
""" Try to figure out what repo we are in and whether we are in a regular
tree or an overlay.
@@ -276,6 +276,8 @@ def FindPortdir(settings):
# the current working directory (from the shell).
location = pwd
+ if not pkdir is None:
+ location = pkgdir
location = normalize_path(location)
path_ids = {}
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, tbc/pym/repoman/
@ 2015-08-01 20:49 Magnus Granberg
0 siblings, 0 replies; 2+ messages in thread
From: Magnus Granberg @ 2015-08-01 20:49 UTC (permalink / raw
To: gentoo-commits
commit: 2410346473e1d00de202eac126d995a148b9609f
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 1 20:49:08 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Aug 1 20:49:08 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=24103464
add repoman full to host and guest
tbc/pym/build_log.py | 41 ++++++++++++++++++++++++++++++-----------
tbc/pym/db_mapping.py | 8 ++++++++
tbc/pym/package.py | 32 ++++++++++++++------------------
tbc/pym/qachecks.py | 13 ++++++++++++-
tbc/pym/repoman/main.py | 10 ++++++----
tbc/pym/sqlquerys.py | 15 ++++++++++++++-
6 files changed, 84 insertions(+), 35 deletions(-)
diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py
index 9c8bbfa..71c14b5 100644
--- a/tbc/pym/build_log.py
+++ b/tbc/pym/build_log.py
@@ -20,18 +20,38 @@ portage.proxy.lazyimport.lazyimport(globals(),
'tbc.actions:action_info,load_emerge_config',
)
-from tbc.qachecks import check_repoman
+from tbc.qachecks import check_repoman, repoman_full
from tbc.text import get_log_text_dict
-from tbc.package import tbc_package
from tbc.readconf import read_config_settings
from tbc.flags import tbc_use_flags
from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \
get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
- update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info
+ update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
+ add_repoman_log
from sqlalchemy.orm import sessionmaker
+def repoman_check_full(session, pkgdir, package_id, config_id):
+ # Check cp with repoman repoman full
+ status = repoman_full(session, pkgdir, config_id)
+ repoman_hash = hashlib.sha256()
+ repoman_log = ""
+ if status:
+ for k, v in status.items():
+ repoman_line = k + '/n'
+ repoman_hash.update(repoman_line.encode('utf-8'))
+ repoman_log = repoman_log + repoman_line
+ for line in v:
+ repoman_line = line + '/n'
+ repoman_hash.update(repoman_line.encode('utf-8'))
+ repoman_log = repoman_log + repoman_line
+ add_repoman_log(session, package_id, repoman_log, repoman_hash.hexdigest())
+ return repoman_log
+ else:
+ return status
+
+
def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
myportdb = portage.portdbapi(mysettings=settings)
cpvr_list = catpkgsplit(pkg.cpv, silent=1)
@@ -48,6 +68,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
build_dict['cpv'] = pkg.cpv
build_dict['categories'] = categories
build_dict['package'] = package
+ build_dict['repo'] = repo
build_dict['config_id'] = config_id
init_useflags = tbc_use_flags(settings, myportdb, pkg.cpv)
iuse_flags_list, final_use_list = init_useflags.get_flags_pkg(pkg, settings)
@@ -77,12 +98,6 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
add_logs(session, log_msg, "info", config_id)
update_manifest_sql(session, build_dict['package_id'], "0")
- init_package = tbc_package(session, settings, myportdb, config_id)
- init_package.update_package_db(build_dict['package_id'])
- ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'])
- if status and ebuild_id_list is None:
- log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
- add_logs(session, log_msg, "error", config_id)
else:
old_ebuild_id_list = []
for ebuild_id in ebuild_id_list:
@@ -204,8 +219,12 @@ def get_buildlog_info(session, settings, pkg, build_dict):
qa_error_list.append(logfile_text_dict[i])
i = i +1
- # Run repoman check_repoman()
- repoman_error_list = check_repoman(settings, myportdb, build_dict['cpv'], pkg.repo)
+ # Run repoman full
+ element = portage.versions.cpv_getkey(build_dict['cpv']).split('/')
+ categories = element[0]
+ package = element[1]
+ pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package
+ repoman_error_list = repoman_check_full(session, pkgdir, build_dict['package_id'], config_id)
build_log_dict = {}
build_log_dict['fail'] = False
if repoman_error_list:
diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index a296c2e..dbae5a8 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -230,6 +230,14 @@ class BuildLogsRepomanQa(Base):
SummeryText = Column('summery_text', Text)
__tablename__ = 'build_logs_repoman_qa'
+class PackagesRepoman(Base):
+ Id = Column('id', Integer, primary_key=True)
+ PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
+ RepomanText = Column('repoman_text', Text)
+ RepomanHash = Column('repoman_hash', String(100))
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'packages_repoman'
+
class ErrorsInfo(Base):
ErrorId = Column('error_id', Integer, primary_key=True)
ErrorName = Column('error_name', String)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index 0a19f15..657c90d 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -3,12 +3,14 @@
from __future__ import print_function
import re
+import hashlib
import portage
from portage.xml.metadata import MetaDataXML
from tbc.flags import tbc_use_flags
from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman, repoman_full
+from tbc.qachecks import digestcheck, check_repoman
+from tbc.build_log import check_repoman_full
from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
@@ -283,11 +285,13 @@ class tbc_package(object):
if manifest_checksum_tree is None:
return
- # Check cp with repoman repoman full
- repoman_full(self._session, pkgdir, self._config_id)
-
package_id = add_new_package_sql(self._session, cp, repo)
+ # Check cp with repoman full
+ status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
+ if status:
+ log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
+ add_logs(self._session, log_msg, "error", self._config_id)
package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
# Get the ebuild list for cp
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
@@ -308,11 +312,7 @@ class tbc_package(object):
# take package descriptions from the ebuilds
if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
- if packageDict[cpv]['checksum'] != "0":
- repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo)
- if repoman_fail:
- log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
+
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
@@ -340,8 +340,11 @@ class tbc_package(object):
log_msg = "U %s:%s" % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
- # Check cp with repoman repoman full
- repoman_full(self._session, pkgdir, self._config_id)
+ # Check cp with repoman full
+ status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
+ if status:
+ log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
+ add_logs(self._session, log_msg, "error", self._config_id)
# Get the ebuild list for cp
old_ebuild_id_list = []
@@ -399,13 +402,6 @@ class tbc_package(object):
else:
ebuild_version_manifest_checksum_db = checksums_db
- # Check with repoman
- if (ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db) and ebuild_version_checksum_tree != "0":
- repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo)
- if repoman_fail:
- log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
-
# Check if the checksum have change
if ebuild_version_manifest_checksum_db is None:
# N = New ebuild
diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py
index 2e25d86..766b572 100644
--- a/tbc/pym/qachecks.py
+++ b/tbc/pym/qachecks.py
@@ -186,5 +186,16 @@ def repoman_full(session, pkgdir, config_id):
SetupInfo = get_setup_info(session, config_id)
config_root = ConfigsMetaData.RepoPath + '/' + ConfigInfo.Hostname + "/" + SetupInfo.Setup
argscmd = []
+ argscmd.append('--xmlparse')
argscmd.append('full')
- repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir)
+ qatracker, qawarnings = repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir)
+ adict = {}
+ for key in qatracker.fails.items():
+ alist = []
+ for foo in key[1]:
+ alist.append(foo)
+ adict[key[0]] = alist
+ if adict == {}:
+ return False
+ return adict
+
diff --git a/tbc/pym/repoman/main.py b/tbc/pym/repoman/main.py
index 2353455..a30704a 100644
--- a/tbc/pym/repoman/main.py
+++ b/tbc/pym/repoman/main.py
@@ -789,7 +789,9 @@ def repoman_scan(repoman_settings, repo_settings, vcs_settings, portdb, options,
"metadata.warning",
"%s/metadata.xml: unused local USE-description: '%s'"
% (xpkg, myflag))
-
+
+ suggest_ignore_masked = False
+ suggest_include_dev = False
if have_pmasked and not (options.without_mask or options.ignore_masked):
suggest_ignore_masked = True
if have_dev_keywords and not options.include_dev:
@@ -973,7 +975,7 @@ def repoman_main(argv, config_root=None, pkgdir=None):
utilities.repoman_sez(
"\"Make your QA payment on time"
" and you'll never see the likes of me.\"\n")
- sys.exit(1)
+ return qatracker, qawarnings
else:
if dofail and can_force and options.force and not options.pretend:
utilities.repoman_sez(
@@ -988,7 +990,7 @@ def repoman_main(argv, config_root=None, pkgdir=None):
utilities.repoman_sez(
"\"Make your QA payment on time"
" and you'll never see the likes of me.\"\n")
- sys.exit(1)
+ return qatracker, qawarnings
if options.pretend:
utilities.repoman_sez(
@@ -1627,4 +1629,4 @@ def repoman_main(argv, config_root=None, pkgdir=None):
" that he forgot to commit anything")
utilities.repoman_sez(
"\"If everyone were like you, I'd be out of business!\"\n")
- sys.exit(0)
+ return qatracker, qawarnings
diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index 8fdf901..4d2a5e5 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -8,7 +8,8 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack
Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \
HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \
BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
- Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata
+ Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata, \
+ PackagesRepoman
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy import and_, or_
@@ -566,3 +567,15 @@ def get_ebuild_restrictions(session, ebuild_id):
restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
return restrictions
return restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
+
+def add_repoman_log(session, package_id, repoman_log, repoman_hash):
+ try:
+ PackagesRepomanInfo = session.query(PackagesRepoman).filter_by(PackageId = package_id).one()
+ except NoResultFound as e:
+ session.add(PackagesRepoman(PackageId = package_id, RepomanText = repoman_log, RepomanHash = repoman_hash))
+ session.commit()
+ else:
+ if PackagesRepomanInfo.RepomanHash != repoman_hash:
+ PackagesRepomanInfo.RepomanHash = repoman_hash
+ PackagesRepomanInfo.RepomanText = repoman_log
+ session.commit()
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2015-08-01 20:50 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2015-08-01 20:49 [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, tbc/pym/repoman/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2015-07-30 11:37 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox