From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 35FEE13835A for ; Fri, 2 Apr 2021 22:12:31 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 3CA62E087E; Fri, 2 Apr 2021 22:12:30 +0000 (UTC) Received: from smtp.gentoo.org (mail.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 18FE2E087E for ; Fri, 2 Apr 2021 22:12:30 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id D55BE340BEF for ; Fri, 2 Apr 2021 22:12:28 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 3EF0763E for ; Fri, 2 Apr 2021 22:12:27 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1617401509.8bdcac12a5c3a41d8321a55b8fef36355dc0f6e7.zorry@gentoo> Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/ X-VCS-Repository: proj/tinderbox-cluster X-VCS-Files: buildbot_gentoo_ci/steps/portage.py buildbot_gentoo_ci/steps/update_db.py buildbot_gentoo_ci/steps/version.py X-VCS-Directories: buildbot_gentoo_ci/steps/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 8bdcac12a5c3a41d8321a55b8fef36355dc0f6e7 X-VCS-Branch: master Date: Fri, 2 Apr 2021 22:12:27 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 6b34161b-7e44-421c-9340-da2c967e8dd2 X-Archives-Hash: e3bd214a8fcdf12f9157e2ce05be08c8 commit: 8bdcac12a5c3a41d8321a55b8fef36355dc0f6e7 Author: Magnus Granberg gentoo org> AuthorDate: Fri Apr 2 22:11:49 2021 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Fri Apr 2 22:11:49 2021 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8bdcac12 Add GetAuxMetadata and use that to get aux metadata Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/portage.py | 63 ++++++++++++++++++++++++++++++++--- buildbot_gentoo_ci/steps/update_db.py | 17 +++++++++- buildbot_gentoo_ci/steps/version.py | 12 ++++--- 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py index 293b862..d66c952 100644 --- a/buildbot_gentoo_ci/steps/portage.py +++ b/buildbot_gentoo_ci/steps/portage.py @@ -10,6 +10,7 @@ from portage import _encodings from portage import _unicode_encode from portage import _parse_eapi_ebuild_head, eapi_is_supported from portage.versions import cpv_getversion, pkgsplit, catpkgsplit +from portage import portdbapi from twisted.internet import defer from twisted.python import log @@ -249,7 +250,7 @@ class SetMakeConf(BuildStep): makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"') makeconf_list.append('PORTAGE_ELOG_CLASSES="*"') # We need echo:info to get the logfile name - makeconf_list.append('PORTAGE_ELOG_SYSTEM="save echo:info"') + makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"') # add ACCEPT_KEYWORDS from the project_data info keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id']) if project_data['status'] == 'unstable': @@ -395,7 +396,8 @@ class SetReposConfLocal(BuildStep): repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf') repos_conf_default_path = yield os.path.join(repos_conf_path, 'default.conf') self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] - self.repository_basedir = self.gentooci.config.project['repository_basedir'] + # the path should be set in the confg + self.repository_basedir2 = '/home/repos2/' if not os.path.isfile(repos_conf_default_path): # setup the default.conf repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'repos.conf') @@ -409,7 +411,7 @@ class SetReposConfLocal(BuildStep): yield WriteTextToFile(repos_conf_default_path, default_conf) repos_conf_repository_path = yield os.path.join(repos_conf_path, self.getProperty("repository_data")['name'] + '.conf') if not os.path.isfile(repos_conf_repository_path): - repository_path = yield os.path.join(self.getProperty("builddir"), self.repository_basedir, self.getProperty("repository_data")['name']) + repository_path = yield os.path.join(self.repository_basedir2, self.getProperty("repository_data")['name']) repository_conf = [] repository_conf.append('[' + self.getProperty("repository_data")['name'] + ']') repository_conf.append('location = ' + repository_path) @@ -444,7 +446,7 @@ class SetMakeConfLocal(BuildStep): makeconf_list.append('ACCEPT_KEYWORDS="~amd64 amd64"') makeconf_list.append('EMERGE_DEFAULT_OPTS=""') makeconf_list.append('ABI_X86="32 64"') - makeconf_list.append('FEATURES=""') + makeconf_list.append('FEATURES="sandbox"') yield WriteTextToFile(make_conf_path, makeconf_list) return SUCCESS @@ -530,3 +532,56 @@ class SetEnvForEbuildSH(BuildStep): )) yield self.build.addStepsAfterCurrentStep(addStepEbuildSH) return SUCCESS + +class GetAuxMetadata(BuildStep): + + name = 'GetAuxMetadata' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + #@defer.inlineCallbacks + def thd_getAuxDbKeys(self): + auxdbs = self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys, myrepo=self.getProperty("repository_data")['name']) + return auxdbs + + @defer.inlineCallbacks + def run(self): + config_root = yield os.path.join(self.getProperty("builddir"), '') + # setup mysettings and myportdb + mysettings = yield portage_config(config_root = config_root) + self.myportdb = yield portdbapi(mysettings=mysettings) + auxdbs = yield self.thd_getAuxDbKeys() + metadata = None + NoSplit = [] + NoSplit.append('DESCRIPTION') + # should have 22 lines + if len(auxdbkeys) != len(auxdbs) or not isinstance(auxdbs, list): + # number of lines is incorrect or not a list. + print("Lines don't match or not a list") + yield self.myportdb.close_caches() + yield portdbapi.portdbapi_instances.remove(self.myportdb) + #self.setProperty('auxdb', metadata, 'auxdb') + return FAILURE + # split all keys to list instead of speces + metadata = {} + i = 0 + for key in auxdbkeys: + if auxdbs[i] == '': + metadata[key] = None + else: + if ' ' in auxdbs[i] and key not in NoSplit: + metadata[key] = auxdbs[i].split(' ') + else: + metadata[key] = [] + metadata[key].append(auxdbs[i]) + i = i + 1 + self.setProperty('auxdb', metadata, 'auxdb') + yield self.myportdb.close_caches() + yield portdbapi.portdbapi_instances.remove(self.myportdb) + return SUCCESS diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py index b84e256..3594c1b 100644 --- a/buildbot_gentoo_ci/steps/update_db.py +++ b/buildbot_gentoo_ci/steps/update_db.py @@ -140,11 +140,20 @@ class UpdateRepos(BuildStep): else: raise AssertionError('Unknown merge analysis result') + @defer.inlineCallbacks + def setchmod(self, path): + for root, dirs, files in os.walk(path): + for d in dirs: + yield os.chmod(os.path.join(root, d), 0o0755) + for f in files: + yield os.chmod(os.path.join(root, f), 0o0644) + @defer.inlineCallbacks def run(self): #FIXME check HEAD agenst local and worker local tree so we don't gitpull evrytime self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] - self.repository_basedir = self.gentooci.config.project['repository_basedir'] + #self.repository_basedir = self.gentooci.config.project['repository_basedir'] + self.repository_basedir = yield os.path.join('/home', 'repos2') self.profile_repository_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name']) repo_path = yield pygit2.discover_repository(self.profile_repository_path) print(repo_path) @@ -153,6 +162,9 @@ class UpdateRepos(BuildStep): else: repo = yield pygit2.Repository(repo_path) yield self.gitPull(repo) + # chmod and chown + yield self.setchmod(self.profile_repository_path) + #yield os.chown(self.profile_repository_path, 'buildbot', 'portage') if self.getProperty("profile_repository_data")['name'] != self.getProperty("repository_data")['name']: self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) repo_path = yield pygit2.discover_repository(self.repository_path) @@ -161,6 +173,9 @@ class UpdateRepos(BuildStep): else: repo = yield pygit2.Repository(repo_path) yield self.gitPull(repo) + # chmod and chown + yield self.setchmod(self.profile_repository_path) + #yield os.chown(self.repository_path, 'buildbot', 'portage') return SUCCESS class TriggerCheckForCPV(BuildStep): diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py index 4c35914..426ddd4 100644 --- a/buildbot_gentoo_ci/steps/version.py +++ b/buildbot_gentoo_ci/steps/version.py @@ -131,7 +131,6 @@ class AddVersionKeyword(BuildStep): return SUCCESS print(auxdb) for keyword in auxdb: - print(keyword) status = 'stable' if keyword[0] in ["~"]: keyword = keyword[1:] @@ -170,7 +169,8 @@ class CheckPathHash(BuildStep): def run(self): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] self.repository_basedir = self.gentooci.config.project['repository_basedir'] - self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) + self.repository_path = yield os.path.join('/home', 'repos2', self.getProperty("repository_data")['name']) + #self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) self.cp_path = yield pkgsplit(self.getProperty("cpv"))[0] self.file_name = yield self.getProperty("package_data")['name'] + '-' + self.getProperty("version") + '.ebuild' self.ebuild_file = yield os.path.join(self.repository_path, self.cp_path, self.file_name) @@ -256,7 +256,9 @@ class CheckV(BuildStep): if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None: if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']: addStepVData.append(GetCommitdata()) - addStepVData.append(portage_steps.SetEnvForEbuildSH()) + #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH + #addStepVData.append(portage_steps.SetEnvForEbuildSH()) + addStepVData.append(portage_steps.GetAuxMetadata()) addStepVData.append(AddVersion()) addStepVData.append(AddVersionKeyword()) addStepVData.append(TriggerBuildCheck()) @@ -265,7 +267,9 @@ class CheckV(BuildStep): return SUCCESS if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None: addStepVData.append(GetCommitdata()) - addStepVData.append(portage_steps.SetEnvForEbuildSH()) + #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH + #addStepVData.append(portage_steps.SetEnvForEbuildSH()) + addStepVData.append(portage_steps.GetAuxMetadata()) addStepVData.append(AddVersion()) addStepVData.append(AddVersionKeyword()) addStepVData.append(TriggerBuildCheck())