From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id D13101382C5 for ; Sat, 17 Apr 2021 17:25:21 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 1A88FE0848; Sat, 17 Apr 2021 17:25:21 +0000 (UTC) Received: from smtp.gentoo.org (dev.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id E7038E0848 for ; Sat, 17 Apr 2021 17:25:20 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 95B2A335CEF for ; Sat, 17 Apr 2021 17:25:19 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 03F1F4A5 for ; Sat, 17 Apr 2021 17:25:18 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1618680288.bfa939daa5d6aad8f4a7b129bced7e9dba796828.zorry@gentoo> Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/ X-VCS-Repository: proj/tinderbox-cluster X-VCS-Files: buildbot_gentoo_ci/config/builders.py buildbot_gentoo_ci/config/buildfactorys.py buildbot_gentoo_ci/config/schedulers.py buildbot_gentoo_ci/config/workers.py buildbot_gentoo_ci/steps/category.py buildbot_gentoo_ci/steps/package.py buildbot_gentoo_ci/steps/portage.py buildbot_gentoo_ci/steps/update_db.py buildbot_gentoo_ci/steps/version.py X-VCS-Directories: buildbot_gentoo_ci/config/ buildbot_gentoo_ci/steps/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: bfa939daa5d6aad8f4a7b129bced7e9dba796828 X-VCS-Branch: master Date: Sat, 17 Apr 2021 17:25:18 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: b1505c85-018a-4e1c-aa5f-b62dfc37d69b X-Archives-Hash: c2fb8f2becffdd8655be568ace02ce17 commit: bfa939daa5d6aad8f4a7b129bced7e9dba796828 Author: Magnus Granberg gentoo org> AuthorDate: Sat Apr 17 17:24:48 2021 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Sat Apr 17 17:24:48 2021 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=bfa939da Add support for more LocalWorkers Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/builders.py | 28 +++++++------- buildbot_gentoo_ci/config/buildfactorys.py | 8 ---- buildbot_gentoo_ci/config/schedulers.py | 10 ++--- buildbot_gentoo_ci/config/workers.py | 3 ++ buildbot_gentoo_ci/steps/category.py | 2 +- buildbot_gentoo_ci/steps/package.py | 32 +++++++-------- buildbot_gentoo_ci/steps/portage.py | 3 +- buildbot_gentoo_ci/steps/update_db.py | 62 ++++++++++-------------------- buildbot_gentoo_ci/steps/version.py | 46 +++++++++++----------- 9 files changed, 85 insertions(+), 109 deletions(-) diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py index d022ac7..56cdde1 100644 --- a/buildbot_gentoo_ci/config/builders.py +++ b/buildbot_gentoo_ci/config/builders.py @@ -4,8 +4,14 @@ from buildbot.plugins import util from buildbot_gentoo_ci.config import buildfactorys +# FIXME: get workers from db or file +LocalWorkers = [] +LocalWorkers.append('updatedb_1') +LocalWorkers.append('updatedb_2') +LocalWorkers.append('updatedb_3') +LocalWorkers.append('updatedb_4') + def gentoo_builders(b=[]): - # FIXME: get workers from db b.append(util.BuilderConfig( name='update_db_check', workername='updatedb_1', @@ -14,47 +20,44 @@ def gentoo_builders(b=[]): factory=buildfactorys.update_db_check() ) ) - # FIXME: get workers from db b.append(util.BuilderConfig( name='update_repo_check', - workername='updatedb_1', + workername='updatedb_2', workerbuilddir='builds', collapseRequests=True, factory=buildfactorys.update_repo_check() ) ) - # FIXME: get workers from db - # Use multiplay workers depend on Property(cpv) + # Use multiplay workers depend on Property(cp) # if cp do not match next one, use diffrent worker then # or first cp have done its buildsteps. + # first LocalWorker need to be done before we can use mulitplay workers (git pull) b.append(util.BuilderConfig( name='update_cpv_data', - workername='updatedb_1', + workernames=LocalWorkers, workerbuilddir='builds', collapseRequests=False, factory=buildfactorys.update_db_cp() ) ) - # FIXME: get workers from db # Use multiplay workers b.append(util.BuilderConfig( name='update_v_data', - workername='updatedb_1', + workernames=LocalWorkers, workerbuilddir='builds', collapseRequests=False, factory=buildfactorys.update_db_v() ) ) - # FIXME: get workers from db # Use multiplay workers b.append(util.BuilderConfig( name='build_request_data', - workername='updatedb_1', + workernames=LocalWorkers, collapseRequests=False, factory=buildfactorys.build_request_check() ) ) - # FIXME: get workers from db + # FIXME: get workers from db or file # Use multiplay workers b.append(util.BuilderConfig( name='run_build_request', @@ -63,11 +66,10 @@ def gentoo_builders(b=[]): factory=buildfactorys.run_build_request() ) ) - # FIXME: get workers from db # Use multiplay workers b.append(util.BuilderConfig( name='parse_build_log', - workername='updatedb_1', + workernames=LocalWorkers, collapseRequests=False, factory=buildfactorys.parse_build_log() ) diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index 962eeb0..e08127b 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -21,14 +21,6 @@ def update_db_check(): f.addStep(update_db.GetDataGentooCiProject()) # update the repos f.addStep(update_db.TriggerUpdateRepositorys()) - # Check if needed path is there - f.addStep(portage.CheckPathLocal()) - # setup the profile - f.addStep(portage.SetMakeProfileLocal()) - # setup repos.conf dir - f.addStep(portage.SetReposConfLocal()) - # setup make.conf - f.addStep(portage.SetMakeConfLocal()) # Make a for loop and trigger new builders with cpv from git_changes # return cpv, repository, project_data f.addStep(update_db.TriggerCheckForCPV()) diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py index 107bb3c..e520076 100644 --- a/buildbot_gentoo_ci/config/schedulers.py +++ b/buildbot_gentoo_ci/config/schedulers.py @@ -12,7 +12,7 @@ def builderUpdateDbNames(props): return list(builders) @util.renderer -def gitUpdateDb(props): +def getGitChanges(props): k = props.changes[0] change_data = {} print(k) @@ -23,7 +23,6 @@ def gitUpdateDb(props): p = v.split('/')[1] pv = v.split('/')[2][:-7] cpv = c + '/' + pv - print(cpv) change_data['cp'] = c + '/' + p change_data['cpvs'].append(cpv) if k['repository'].endswith('.git'): @@ -35,16 +34,15 @@ def gitUpdateDb(props): change_data['comments'] = k['comments'] change_data['revision'] = k['revision'] change_data['timestamp'] =k['when_timestamp'] - print(change_data) return change_data def gentoo_schedulers(): scheduler_update_db = schedulers.SingleBranchScheduler( name='scheduler_update_db', treeStableTimer=0, - properties = { - 'git_change' : gitUpdateDb, - }, + properties = dict( + change_data = getGitChanges + ), builderNames = builderUpdateDbNames, change_filter=util.ChangeFilter(branch='master'), ) diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py index a566b79..6141d89 100644 --- a/buildbot_gentoo_ci/config/workers.py +++ b/buildbot_gentoo_ci/config/workers.py @@ -6,5 +6,8 @@ from buildbot.plugins import worker def gentoo_workers(w=[]): # FIXME: Get workers from db w.append(worker.LocalWorker('updatedb_1')) + w.append(worker.LocalWorker('updatedb_2')) + w.append(worker.LocalWorker('updatedb_3')) + w.append(worker.LocalWorker('updatedb_4')) w.append(worker.Worker('bot-test', 'test1234')) return w diff --git a/buildbot_gentoo_ci/steps/category.py b/buildbot_gentoo_ci/steps/category.py index 81061e6..946dc63 100644 --- a/buildbot_gentoo_ci/steps/category.py +++ b/buildbot_gentoo_ci/steps/category.py @@ -49,7 +49,7 @@ class CheckC(BuildStep): @defer.inlineCallbacks def run(self): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] - self.category = yield self.getProperty("cpv").split('/')[0] + self.category = self.getProperty("change_data")['cp'].split('/')[0] print(self.category) self.category_data = yield self.gentooci.db.categorys.getCategoryByName(self.category) print(self.category_data) diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py index b4e06b7..76c90cc 100644 --- a/buildbot_gentoo_ci/steps/package.py +++ b/buildbot_gentoo_ci/steps/package.py @@ -57,7 +57,7 @@ class CheckP(BuildStep): @defer.inlineCallbacks def run(self): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] - self.package = yield catpkgsplit(self.getProperty("cpv"))[1] + self.package = self.getProperty("change_data")['cp'].split('/')[1] print(self.package) self.package_data = yield self.gentooci.db.packages.getPackageByName(self.package, self.getProperty("category_data")['uuid'], @@ -86,20 +86,22 @@ class TriggerCheckForV(BuildStep): @defer.inlineCallbacks def run(self): addStepUpdateVData = [] - addStepUpdateVData.append( - steps.Trigger( - schedulerNames=['update_v_data'], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - 'cpv' : self.getProperty("cpv"), - 'package_data' : self.getProperty("package_data"), - 'repository_data' : self.getProperty("repository_data"), - 'category_data' : self.getProperty("category_data"), - 'revision_data' : self.getProperty("revision_data"), - 'project_data' : self.getProperty("project_data"), - } - ) + for cpv in self.getProperty("change_data")['cpvs']: + print(cpv) + addStepUpdateVData.append( + steps.Trigger( + schedulerNames=['update_v_data'], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + 'cpv' : cpv, + 'package_data' : self.getProperty("package_data"), + 'repository_data' : self.getProperty("repository_data"), + 'category_data' : self.getProperty("category_data"), + 'change_data' : self.getProperty("change_data"), + 'project_data' : self.getProperty("project_data"), + } ) + ) yield self.build.addStepsAfterCurrentStep(addStepUpdateVData) return SUCCESS diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py index c7e82ce..37f7342 100644 --- a/buildbot_gentoo_ci/steps/portage.py +++ b/buildbot_gentoo_ci/steps/portage.py @@ -407,11 +407,12 @@ class SetMakeProfileLocal(BuildStep): if os.path.isfile(parent_path): return SKIPPED self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty('project_data')['profile_repository_uuid']) self.repository_basedir = self.gentooci.config.project['repository_basedir'] makeprofiles_paths = [] makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'make.profile') for makeprofile in makeprofiles_data: - makeprofile_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name'], 'profiles', makeprofile['value'], '') + makeprofile_path = yield os.path.join(self.repository_basedir, self.profile_repository_data['name'], 'profiles', makeprofile['value'], '') makeprofiles_paths.append('../../../' + makeprofile_path) yield WriteTextToFile(parent_path, makeprofiles_paths) return SUCCESS diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py index b19d0e7..8e7f8ee 100644 --- a/buildbot_gentoo_ci/steps/update_db.py +++ b/buildbot_gentoo_ci/steps/update_db.py @@ -25,11 +25,13 @@ class GetDataGentooCiProject(BuildStep): return FAILURE self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.project_data['profile_repository_uuid']) if self.profile_repository_data is None: - log.err('No data for repository in the database') + log.err('No data for profile repository in the database') + return FAILURE + if not isinstance(self.getProperty("change_data"), dict): return FAILURE print(self.project_data) print(self.profile_repository_data) - print(self.getProperty("git_change")) + print(self.getProperty("change_data")) print(self.getProperty("repository")) repository = False self.repository_data = False @@ -39,6 +41,9 @@ class GetDataGentooCiProject(BuildStep): repository = v[:-4] if repository: self.repository_data = yield self.gentooci.db.repositorys.getRepositoryByName(repository) + if self.getProperty("change_data")['repository'] != self.repository_data['name']: + log.msg("%s don't match" % self.getProperty("change_data")['repository']) + return FAILURE self.setProperty("project_data", self.project_data, 'project_data') self.setProperty("profile_repository_data", self.profile_repository_data, 'profile_repository_data') self.setProperty("repository_data", self.repository_data, 'repository_data') @@ -65,7 +70,7 @@ class TriggerUpdateRepositorys(BuildStep): set_properties={ 'profile_repository_uuid' : self.getProperty("profile_repository_data")['uuid'], 'repository_uuid' : self.getProperty("repository_data")['uuid'], - 'commit_time' : self.getProperty("git_change")['timestamp'], + 'commit_time' : self.getProperty("change_data")['timestamp'], } ) ]) @@ -84,43 +89,16 @@ class TriggerCheckForCPV(BuildStep): @defer.inlineCallbacks def run(self): - change_data = self.getProperty("git_change") - # check if git_change is a dict - if not isinstance(change_data, dict): - return FAILURE - addStepUpdateCPVData = [] - # make a trigger for all cpv in the list - for cpv in change_data['cpvs']: - self.success = True - if change_data['repository'] != self.getProperty("repository_data")['name']: - log.msg("%s don't match" % change_data['repository']) - self.success = False - # Trigger cpv builds and update db if we are working with ebuilds - # check that cpv is valied - if catpkgsplit(cpv) is None: - log.msg("%s is not vaild package name" % cpv) - self.success = False - if self.success: - revision_data = {} - revision_data['author'] = change_data['author'] - revision_data['committer'] = change_data['committer'] - revision_data['comments'] = change_data['comments'] - revision_data['revision'] = change_data['revision'] - addStepUpdateCPVData.append( - steps.Trigger( - schedulerNames=['update_cpv_data'], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - 'cpv' : cpv, - 'project_data' : self.getProperty("project_data"), - 'repository_data' : self.getProperty("repository_data"), - 'revision_data' : revision_data, - } - ) - ) - print(addStepUpdateCPVData) - yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData) - if self.success is False: - return FAILURE + yield self.build.addStepsAfterCurrentStep([ + steps.Trigger( + schedulerNames=['update_cpv_data'], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + 'project_data' : self.getProperty("project_data"), + 'repository_data' : self.getProperty("repository_data"), + 'change_data' : self.getProperty("change_data"), + } + ) + ]) return SUCCESS diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py index e6a927a..df97536 100644 --- a/buildbot_gentoo_ci/steps/version.py +++ b/buildbot_gentoo_ci/steps/version.py @@ -15,6 +15,7 @@ from twisted.python import log from buildbot.process.buildstep import BuildStep from buildbot.process.results import SUCCESS from buildbot.process.results import FAILURE +from buildbot.process.results import WARNINGS from buildbot.plugins import steps from buildbot_gentoo_ci.steps import portage as portage_steps @@ -36,6 +37,7 @@ class GetVData(BuildStep): # set cwd to builddir yield os.chdir(self.getProperty("builddir")) self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + print(self.getProperty("cpv")) self.version = yield cpv_getversion(self.getProperty("cpv")) print(self.version) self.old_version_data = yield self.gentooci.db.versions.getVersionByName(self.version, self.getProperty("package_data")['uuid']) @@ -90,8 +92,8 @@ class GetCommitdata(BuildStep): #@defer.inlineCallbacks def run(self): - print(self.getProperty("revision_data")) - self.setProperty('commit_id', self.getProperty("revision_data")['revision'], 'commit_id') + print(self.getProperty("change_data")) + self.setProperty('commit_id', self.getProperty("change_data")['revision'], 'commit_id') return SUCCESS class AddVersionKeyword(BuildStep): @@ -243,32 +245,30 @@ class CheckV(BuildStep): @defer.inlineCallbacks def run(self): - self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] - self.old_version_data = self.getProperty("old_version_data") - self.ebuild_file = self.getProperty("ebuild_file") addStepVData = [] - print(self.ebuild_file) - print(self.old_version_data) + print(self.getProperty("ebuild_file")) + print(self.getProperty("old_version_data")) print(self.getProperty("ebuild_file_hash")) - if self.getProperty("ebuild_file") is None and self.getProperty("old_version_data") is not None: - addStepVData.append(TriggerBuildCheck()) - addStepVData.append(DeleteOldVersion()) - if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None: - if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']: - addStepVData.append(GetCommitdata()) - #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH - #addStepVData.append(portage_steps.SetEnvForEbuildSH()) - addStepVData.append(portage_steps.GetAuxMetadata()) - addStepVData.append(AddVersion()) - addStepVData.append(AddVersionKeyword()) + if self.getProperty("ebuild_file") is None: + if self.getProperty("old_version_data") is None: + return WARNINGS + else: addStepVData.append(TriggerBuildCheck()) addStepVData.append(DeleteOldVersion()) - else: - return SUCCESS - if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None: + else: + if self.getProperty("old_version_data") is not None: + if self.getProperty("ebuild_file_hash") == self.getProperty("old_version_data")['file_hash']: + return WARNINGS + else: + addStepVData.append(DeleteOldVersion()) + # setup /etc/portage + addStepVData.append(portage_steps.CheckPathLocal()) + addStepVData.append(portage_steps.SetMakeProfileLocal()) + addStepVData.append(portage_steps.SetReposConfLocal()) + addStepVData.append(portage_steps.SetMakeConfLocal()) + # get commit data addStepVData.append(GetCommitdata()) - #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH - #addStepVData.append(portage_steps.SetEnvForEbuildSH()) + # get ebuild aux metadata addStepVData.append(portage_steps.GetAuxMetadata()) addStepVData.append(AddVersion()) addStepVData.append(AddVersionKeyword())