* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
@ 2021-01-28 1:09 Magnus Granberg
0 siblings, 0 replies; 2+ messages in thread
From: Magnus Granberg @ 2021-01-28 1:09 UTC (permalink / raw
To: gentoo-commits
commit: 7c12da2ec9470fcf82b37943e2a1cc523351aefe
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 28 01:09:38 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jan 28 01:09:38 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7c12da2e
Add SetReposConf and UpdateRepos
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 13 ++--
buildbot_gentoo_ci/db/model.py | 3 +-
buildbot_gentoo_ci/db/projects.py | 29 ++++++++-
buildbot_gentoo_ci/steps/builders.py | 98 ++++++++++++++++++++++++++----
4 files changed, 123 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index a6bafb5..3fad219 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,8 +88,8 @@ def build_request_check():
def run_build_request():
f = util.BuildFactory()
# FIXME: 5
- # update repo for the profile
- f.addStep(builders.UpdateProfileRepo())
+ # set needed Propertys
+ f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
workdir='/etc/'))
@@ -99,9 +99,10 @@ def run_build_request():
f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
workdir='/etc/portage/'))
f.addStep(builders.SetMakeProfile())
- # setup repo.conf dir
- #f.addStep(buildbot_steps.MakeDirectory(dir="repo.conf",
+ # setup repos.conf dir
+ f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
workdir='/etc/portage/'))
- # check if we have all repository's in repos.conf listed in project_repository's
- # update all repos listed in project_repository's
+ f.addStep(builders.SetReposConf())
+ # update the repositorys listed in project_repository
+ f.addStep(builders.UpdateRepos())
return f
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0defb0c..596d04e 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -138,7 +138,8 @@ class Model(base.DBConnectorComponent):
sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
- sa.Column('directorys', sa.Enum('make.profile'), nullable=False),
+ # FIXME: directorys should be moved to own table
+ sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'), nullable=False),
sa.Column('value', sa.String(255), nullable=False),
)
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 5c3406a..00e1569 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -80,7 +80,19 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
return res
@defer.inlineCallbacks
- def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+ def getRepositorysByProjectUuid(self, uuid, auto=True):
+ def thd(conn):
+ tbl = self.db.model.projects_repositorys
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.auto == auto)
+ return [self._row2dict_projects_repositorys(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getAllProjectPortageByUuidAndDirectory(self, uuid, directory):
def thd(conn):
tbl = self.db.model.projects_portage
q = tbl.select()
@@ -91,6 +103,21 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+ def thd(conn):
+ tbl = self.db.model.projects_portage
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.directorys == directory)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict_projects_portage(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 1c8cbb0..d3b3607 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -80,9 +80,9 @@ class GetProjectRepositoryData(BuildStep):
yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
return SUCCESS
-class UpdateProfileRepo(BuildStep):
+class SetupPropertys(BuildStep):
- name = 'UpdateProfileRepo'
+ name = 'SetupPropertys'
description = 'Running'
descriptionDone = 'Ran'
descriptionSuffix = None
@@ -101,17 +101,8 @@ class UpdateProfileRepo(BuildStep):
self.setProperty('portage_repos_path', self.portage_repos_path, 'portage_repos_path')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
- repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
self.setProperty('project_data', project_data, 'project_data')
- self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
- profile_repository_path = yield os.path.join(self.portage_repos_path, self.profile_repository_data['name'])
- yield self.build.addStepsAfterCurrentStep([
- steps.Git(repourl=self.profile_repository_data['mirror_url'],
- mode='incremental',
- submodules=True,
- workdir=os.path.join(profile_repository_path, ''))
- ])
return SUCCESS
class SetMakeProfile(BuildStep):
@@ -133,7 +124,7 @@ class SetMakeProfile(BuildStep):
project_data = self.getProperty('project_data')
profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
makeprofiles_paths = []
- makeprofiles_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
+ makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
for makeprofile in makeprofiles_data:
makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', makeprofile['value'], '')
makeprofiles_paths.append('../../..' + makeprofile_path)
@@ -145,3 +136,86 @@ class SetMakeProfile(BuildStep):
workdir='/etc/portage/')
])
return SUCCESS
+
+class SetReposConf(BuildStep):
+
+ name = 'SetReposConf'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ # setup the default.conf
+ repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'repos.conf')
+ if repos_conf_data is None:
+ print('Default repo is not set in repos.conf')
+ return FAILURE
+ # check if repos_conf_data['value'] is vaild repo name
+ separator = '\n'
+ default_conf = []
+ default_conf.append('[DEFAULT]')
+ default_conf.append('main-repo = ' + repos_conf_data['value'])
+ default_conf.append('auto-sync = no')
+ default_conf_string = separator.join(default_conf)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(default_conf_string + separator,
+ workerdest="repos.conf/default.conf",
+ workdir='/etc/portage/')
+ ])
+ # add all repos that project have in projects_repositorys to repos.conf/reponame.conf
+ projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+ for project_repository_data in projects_repositorys_data:
+ repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+ repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
+ repository_conf = []
+ repository_conf.append('[' + repository_data['name'] + ']')
+ repository_conf.append('location = ' + repository_path)
+ repository_conf.append('sync-uri = ' + repository_data['mirror_url'])
+ repository_conf.append('sync-type = git')
+ repository_conf.append('auto-sync = no')
+ repository_conf_string = separator.join(repository_conf)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(repository_conf_string + separator,
+ workerdest='repos.conf/' + repository_data['name'] + '.conf',
+ workdir='/etc/portage/')
+ ])
+ return SUCCESS
+
+class UpdateRepos(BuildStep):
+
+ name = 'UpdateRepos'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ # update/add all repos that in project_repository for the project
+ projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+ for project_repository_data in projects_repositorys_data:
+ repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+ repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Git(repourl=repository_data['mirror_url'],
+ mode='incremental',
+ submodules=True,
+ workdir=os.path.join(repository_path, ''))
+ ])
+ return SUCCESS
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
@ 2021-09-08 0:20 Magnus Granberg
0 siblings, 0 replies; 2+ messages in thread
From: Magnus Granberg @ 2021-09-08 0:20 UTC (permalink / raw
To: gentoo-commits
commit: 396cbf6ec3c530541c278155828677a341fda248
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Sep 8 00:21:03 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Sep 8 00:21:03 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=396cbf6e
Support dynamically projects and worker in run_build_request
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 24 +++++++++++++++++++++---
buildbot_gentoo_ci/db/model.py | 20 ++++++++++++++++++++
buildbot_gentoo_ci/db/projects.py | 18 ++++++++++++++++++
buildbot_gentoo_ci/steps/builders.py | 28 +++++++++++++++++++---------
4 files changed, 78 insertions(+), 12 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index 56cdde1..c28d016 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,16 +1,34 @@
# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from twisted.internet import defer
+
from buildbot.plugins import util
from buildbot_gentoo_ci.config import buildfactorys
-# FIXME: get workers from db or file
+# FIXME: get LocalWorkers and BuildWorkers from db or file
LocalWorkers = []
LocalWorkers.append('updatedb_1')
LocalWorkers.append('updatedb_2')
LocalWorkers.append('updatedb_3')
LocalWorkers.append('updatedb_4')
+BuildWorkers = []
+BuildWorkers.append('a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd')
+
+@defer.inlineCallbacks
+def CanWorkerBuildProject(builder, wfb, request):
+ gentooci = builder.master.namedServices['services'].namedServices['gentooci']
+ project_build_data = request.properties['project_build_data']
+ project_workers = yield gentooci.db.projects.getWorkersByProjectUuid(project_build_data['project_uuid'])
+ print(project_workers)
+ print(wfb)
+ for worker in project_workers:
+ if wfb.worker.workername == worker['worker_uuid']:
+ return True
+ print('no worker')
+ return False
+
def gentoo_builders(b=[]):
b.append(util.BuilderConfig(
name='update_db_check',
@@ -57,11 +75,11 @@ def gentoo_builders(b=[]):
factory=buildfactorys.build_request_check()
)
)
- # FIXME: get workers from db or file
# Use multiplay workers
b.append(util.BuilderConfig(
name='run_build_request',
- workername='bot-test',
+ workernames=BuildWorkers,
+ canStartBuild=CanWorkerBuildProject,
collapseRequests=False,
factory=buildfactorys.run_build_request()
)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index ca9932a..d9a3972 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -232,6 +232,17 @@ class Model(base.DBConnectorComponent):
sa.Column('search_type', sa.Enum('in', 'startswith', 'endswith', 'search'), default='in'),
)
+ projects_workers = sautils.Table(
+ "projects_workers", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('worker_uuid', sa.String(36),
+ sa.ForeignKey('workers.uuid', ondelete='CASCADE'),
+ nullable=False),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
@@ -289,6 +300,15 @@ class Model(base.DBConnectorComponent):
sa.Column('status', sa.Enum('stable','unstable','negative','all'), nullable=False),
)
+ workers = sautils.Table(
+ "workers", metadata,
+ # unique id per project
+ sa.Column('uuid', sa.String(36), primary_key=True,
+ default=lambda: str(uuid.uuid4())),
+ sa.Column('type', sa.Enum('local','default','latent'), nullable=False),
+ sa.Column('enabled', sa.Boolean, default=False),
+ )
+
# Tables related to users
# -----------------------
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 176be92..fbef435 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -189,6 +189,17 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getWorkersByProjectUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_workers
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ return [self._row2dict_projects_workers(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -217,6 +228,13 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
build=row.build
)
+ def _row2dict_projects_workers(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ worker_uuid=row.worker_uuid,
+ )
+
def _row2dict_projects_portage(self, conn, row):
return dict(
id=row.id,
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 5b53018..9918938 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -186,7 +186,10 @@ class TriggerRunBuildRequest(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, **kwargs):
+ def __init__(self, projectrepository_data, use_data, project_data, **kwargs):
+ self.projectrepository_data = projectrepository_data
+ self.use_data = use_data
+ self.project_data = project_data
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -194,7 +197,7 @@ class TriggerRunBuildRequest(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
if self.getProperty('project_build_data') is None:
project_build_data = {}
- project_build_data['project_uuid'] = self.getProperty('project_data')['uuid']
+ project_build_data['project_uuid'] = self.project_data['uuid']
project_build_data['version_uuid'] = self.getProperty("version_data")['uuid']
project_build_data['status'] = 'waiting'
project_build_data['requested'] = False
@@ -210,8 +213,8 @@ class TriggerRunBuildRequest(BuildStep):
set_properties={
'cpv' : self.getProperty("cpv"),
'version_data' : self.getProperty("version_data"),
- 'projectrepository_data' : self.getProperty('projectrepository_data'),
- 'use_data' : self.getProperty("use_data"),
+ 'projectrepository_data' : self.projectrepository_data,
+ 'use_data' : self.use_data,
'fullcheck' : self.getProperty("fullcheck"),
'project_build_data' : project_build_data
}
@@ -239,14 +242,17 @@ class GetProjectRepositoryData(BuildStep):
return SUCCESS
# for loop to get all the projects that have the repository
for projectrepository_data in self.projectrepositorys_data:
+ print(projectrepository_data)
# get project data
project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
+ #FIXME: check if we have working workers
+ project_workers = yield self.gentooci.db.projects.getWorkersByProjectUuid(project_data['uuid'])
+ if project_workers == []:
+ print('No Workers on this profile')
+ continue
# check if auto, enabled and not in config.project['project']
- if project_data['auto'] is True and project_data['enabled'] is True and project_data['name'] != self.gentooci.config.project['project']:
+ if project_data['auto'] is True and project_data['enabled'] is True and project_data['name'] != self.gentooci.config.project['project']['update_db']:
# set Property projectrepository_data so we can use it in the trigger
- self.setProperty('projectrepository_data', projectrepository_data, 'projectrepository_data')
- self.setProperty('use_data', None, 'use_data')
- self.setProperty('project_data', project_data, 'project_data')
# get name o project keyword
project_keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
# if not * (all keywords)
@@ -258,7 +264,11 @@ class GetProjectRepositoryData(BuildStep):
version_keywords_data = self.getProperty("version_keyword_dict")[project_keyword_data['name']]
# if match trigger BuildRequest on cpv
if project_data['status'] == version_keywords_data['status']:
- yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
+ yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest(
+ projectrepository_data = projectrepository_data,
+ use_data = None,
+ project_data = project_data
+ )])
return SUCCESS
class SetupPropertys(BuildStep):
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2021-09-08 0:20 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-01-28 1:09 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2021-09-08 0:20 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox