From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id D69021395E4 for ; Sat, 1 Aug 2015 20:50:02 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 7127514026; Sat, 1 Aug 2015 20:50:00 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 11E2C14026 for ; Sat, 1 Aug 2015 20:49:59 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id E86D33406CE for ; Sat, 1 Aug 2015 20:49:58 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id F27AAB8 for ; Sat, 1 Aug 2015 20:49:55 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1438462148.2410346473e1d00de202eac126d995a148b9609f.zorry@gentoo> Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, tbc/pym/repoman/ X-VCS-Repository: proj/tinderbox-cluster X-VCS-Files: tbc/pym/build_log.py tbc/pym/db_mapping.py tbc/pym/package.py tbc/pym/qachecks.py tbc/pym/repoman/main.py tbc/pym/sqlquerys.py X-VCS-Directories: tbc/pym/ tbc/pym/repoman/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 2410346473e1d00de202eac126d995a148b9609f X-VCS-Branch: master Date: Sat, 1 Aug 2015 20:49:55 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 627d9aa5-61b0-4099-ab16-fe3e284c60f3 X-Archives-Hash: 3136afda80c850bb818583d2a66cad41 commit: 2410346473e1d00de202eac126d995a148b9609f Author: Magnus Granberg gentoo org> AuthorDate: Sat Aug 1 20:49:08 2015 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Sat Aug 1 20:49:08 2015 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=24103464 add repoman full to host and guest tbc/pym/build_log.py | 41 ++++++++++++++++++++++++++++++----------- tbc/pym/db_mapping.py | 8 ++++++++ tbc/pym/package.py | 32 ++++++++++++++------------------ tbc/pym/qachecks.py | 13 ++++++++++++- tbc/pym/repoman/main.py | 10 ++++++---- tbc/pym/sqlquerys.py | 15 ++++++++++++++- 6 files changed, 84 insertions(+), 35 deletions(-) diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py index 9c8bbfa..71c14b5 100644 --- a/tbc/pym/build_log.py +++ b/tbc/pym/build_log.py @@ -20,18 +20,38 @@ portage.proxy.lazyimport.lazyimport(globals(), 'tbc.actions:action_info,load_emerge_config', ) -from tbc.qachecks import check_repoman +from tbc.qachecks import check_repoman, repoman_full from tbc.text import get_log_text_dict -from tbc.package import tbc_package from tbc.readconf import read_config_settings from tbc.flags import tbc_use_flags from tbc.ConnectionManager import NewConnection from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \ get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \ add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \ - update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info + update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \ + add_repoman_log from sqlalchemy.orm import sessionmaker +def repoman_check_full(session, pkgdir, package_id, config_id): + # Check cp with repoman repoman full + status = repoman_full(session, pkgdir, config_id) + repoman_hash = hashlib.sha256() + repoman_log = "" + if status: + for k, v in status.items(): + repoman_line = k + '/n' + repoman_hash.update(repoman_line.encode('utf-8')) + repoman_log = repoman_log + repoman_line + for line in v: + repoman_line = line + '/n' + repoman_hash.update(repoman_line.encode('utf-8')) + repoman_log = repoman_log + repoman_line + add_repoman_log(session, package_id, repoman_log, repoman_hash.hexdigest()) + return repoman_log + else: + return status + + def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg): myportdb = portage.portdbapi(mysettings=settings) cpvr_list = catpkgsplit(pkg.cpv, silent=1) @@ -48,6 +68,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg): build_dict['cpv'] = pkg.cpv build_dict['categories'] = categories build_dict['package'] = package + build_dict['repo'] = repo build_dict['config_id'] = config_id init_useflags = tbc_use_flags(settings, myportdb, pkg.cpv) iuse_flags_list, final_use_list = init_useflags.get_flags_pkg(pkg, settings) @@ -77,12 +98,6 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg): log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,) add_logs(session, log_msg, "info", config_id) update_manifest_sql(session, build_dict['package_id'], "0") - init_package = tbc_package(session, settings, myportdb, config_id) - init_package.update_package_db(build_dict['package_id']) - ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id']) - if status and ebuild_id_list is None: - log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,) - add_logs(session, log_msg, "error", config_id) else: old_ebuild_id_list = [] for ebuild_id in ebuild_id_list: @@ -204,8 +219,12 @@ def get_buildlog_info(session, settings, pkg, build_dict): qa_error_list.append(logfile_text_dict[i]) i = i +1 - # Run repoman check_repoman() - repoman_error_list = check_repoman(settings, myportdb, build_dict['cpv'], pkg.repo) + # Run repoman full + element = portage.versions.cpv_getkey(build_dict['cpv']).split('/') + categories = element[0] + package = element[1] + pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package + repoman_error_list = repoman_check_full(session, pkgdir, build_dict['package_id'], config_id) build_log_dict = {} build_log_dict['fail'] = False if repoman_error_list: diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py index a296c2e..dbae5a8 100644 --- a/tbc/pym/db_mapping.py +++ b/tbc/pym/db_mapping.py @@ -230,6 +230,14 @@ class BuildLogsRepomanQa(Base): SummeryText = Column('summery_text', Text) __tablename__ = 'build_logs_repoman_qa' +class PackagesRepoman(Base): + Id = Column('id', Integer, primary_key=True) + PackageId = Column('package_id', Integer, ForeignKey('packages.package_id')) + RepomanText = Column('repoman_text', Text) + RepomanHash = Column('repoman_hash', String(100)) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'packages_repoman' + class ErrorsInfo(Base): ErrorId = Column('error_id', Integer, primary_key=True) ErrorName = Column('error_name', String) diff --git a/tbc/pym/package.py b/tbc/pym/package.py index 0a19f15..657c90d 100644 --- a/tbc/pym/package.py +++ b/tbc/pym/package.py @@ -3,12 +3,14 @@ from __future__ import print_function import re +import hashlib import portage from portage.xml.metadata import MetaDataXML from tbc.flags import tbc_use_flags from tbc.text import get_ebuild_cvs_revision, get_log_text_dict from tbc.flags import tbc_use_flags -from tbc.qachecks import digestcheck, check_repoman, repoman_full +from tbc.qachecks import digestcheck, check_repoman +from tbc.build_log import check_repoman_full from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \ add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \ get_package_metadata_sql, update_package_metadata, update_manifest_sql, \ @@ -283,11 +285,13 @@ class tbc_package(object): if manifest_checksum_tree is None: return - # Check cp with repoman repoman full - repoman_full(self._session, pkgdir, self._config_id) - package_id = add_new_package_sql(self._session, cp, repo) + # Check cp with repoman full + status = check_repoman_full(self._session, pkgdir, package_id, self._config_id) + if status: + log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) + add_logs(self._session, log_msg, "error", self._config_id) package_metadataDict = self.get_package_metadataDict(pkgdir, package_id) # Get the ebuild list for cp ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree) @@ -308,11 +312,7 @@ class tbc_package(object): # take package descriptions from the ebuilds if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']: package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree'] - if packageDict[cpv]['checksum'] != "0": - repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo) - if repoman_fail: - log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) - add_logs(self._session, log_msg, "error", self._config_id) + self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree) log_msg = "C %s:%s ... Done." % (cp, repo) add_logs(self._session, log_msg, "info", self._config_id) @@ -340,8 +340,11 @@ class tbc_package(object): log_msg = "U %s:%s" % (cp, repo) add_logs(self._session, log_msg, "info", self._config_id) - # Check cp with repoman repoman full - repoman_full(self._session, pkgdir, self._config_id) + # Check cp with repoman full + status = check_repoman_full(self._session, pkgdir, package_id, self._config_id) + if status: + log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) + add_logs(self._session, log_msg, "error", self._config_id) # Get the ebuild list for cp old_ebuild_id_list = [] @@ -399,13 +402,6 @@ class tbc_package(object): else: ebuild_version_manifest_checksum_db = checksums_db - # Check with repoman - if (ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db) and ebuild_version_checksum_tree != "0": - repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo) - if repoman_fail: - log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) - add_logs(self._session, log_msg, "error", self._config_id) - # Check if the checksum have change if ebuild_version_manifest_checksum_db is None: # N = New ebuild diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py index 2e25d86..766b572 100644 --- a/tbc/pym/qachecks.py +++ b/tbc/pym/qachecks.py @@ -186,5 +186,16 @@ def repoman_full(session, pkgdir, config_id): SetupInfo = get_setup_info(session, config_id) config_root = ConfigsMetaData.RepoPath + '/' + ConfigInfo.Hostname + "/" + SetupInfo.Setup argscmd = [] + argscmd.append('--xmlparse') argscmd.append('full') - repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir) + qatracker, qawarnings = repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir) + adict = {} + for key in qatracker.fails.items(): + alist = [] + for foo in key[1]: + alist.append(foo) + adict[key[0]] = alist + if adict == {}: + return False + return adict + diff --git a/tbc/pym/repoman/main.py b/tbc/pym/repoman/main.py index 2353455..a30704a 100644 --- a/tbc/pym/repoman/main.py +++ b/tbc/pym/repoman/main.py @@ -789,7 +789,9 @@ def repoman_scan(repoman_settings, repo_settings, vcs_settings, portdb, options, "metadata.warning", "%s/metadata.xml: unused local USE-description: '%s'" % (xpkg, myflag)) - + + suggest_ignore_masked = False + suggest_include_dev = False if have_pmasked and not (options.without_mask or options.ignore_masked): suggest_ignore_masked = True if have_dev_keywords and not options.include_dev: @@ -973,7 +975,7 @@ def repoman_main(argv, config_root=None, pkgdir=None): utilities.repoman_sez( "\"Make your QA payment on time" " and you'll never see the likes of me.\"\n") - sys.exit(1) + return qatracker, qawarnings else: if dofail and can_force and options.force and not options.pretend: utilities.repoman_sez( @@ -988,7 +990,7 @@ def repoman_main(argv, config_root=None, pkgdir=None): utilities.repoman_sez( "\"Make your QA payment on time" " and you'll never see the likes of me.\"\n") - sys.exit(1) + return qatracker, qawarnings if options.pretend: utilities.repoman_sez( @@ -1627,4 +1629,4 @@ def repoman_main(argv, config_root=None, pkgdir=None): " that he forgot to commit anything") utilities.repoman_sez( "\"If everyone were like you, I'd be out of business!\"\n") - sys.exit(0) + return qatracker, qawarnings diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py index 8fdf901..4d2a5e5 100644 --- a/tbc/pym/sqlquerys.py +++ b/tbc/pym/sqlquerys.py @@ -8,7 +8,8 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \ HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \ BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \ - Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata + Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata, \ + PackagesRepoman from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound from sqlalchemy import and_, or_ @@ -566,3 +567,15 @@ def get_ebuild_restrictions(session, ebuild_id): restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one()) return restrictions return restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one()) + +def add_repoman_log(session, package_id, repoman_log, repoman_hash): + try: + PackagesRepomanInfo = session.query(PackagesRepoman).filter_by(PackageId = package_id).one() + except NoResultFound as e: + session.add(PackagesRepoman(PackageId = package_id, RepomanText = repoman_log, RepomanHash = repoman_hash)) + session.commit() + else: + if PackagesRepomanInfo.RepomanHash != repoman_hash: + PackagesRepomanInfo.RepomanHash = repoman_hash + PackagesRepomanInfo.RepomanText = repoman_log + session.commit()