* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
@ 2021-02-27 22:46 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2021-02-27 22:46 UTC (permalink / raw
To: gentoo-commits
commit: 22ad0178cca549026ab51f9f26e3fb9ad28f6374
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 27 22:45:35 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 27 22:45:35 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=22ad0178
Add SetPackageDefault and SetEnvDefault to steps
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 6 +-
buildbot_gentoo_ci/db/model.py | 27 ++++++++-
buildbot_gentoo_ci/db/portages.py | 14 +++++
buildbot_gentoo_ci/db/projects.py | 44 ++++++++++++++
buildbot_gentoo_ci/steps/portage.py | 97 ++++++++++++++++++++++++++++++
5 files changed, 185 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index ac8bac1..acafed8 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -9,6 +9,7 @@ from buildbot_gentoo_ci.steps import category
from buildbot_gentoo_ci.steps import package
from buildbot_gentoo_ci.steps import version
from buildbot_gentoo_ci.steps import builders
+from buildbot_gentoo_ci.steps import portage
def update_db_check():
f = util.BuildFactory()
@@ -114,9 +115,10 @@ def run_build_request():
f.addStep(builders.UpdateRepos())
# setup make.conf
f.addStep(builders.SetMakeConf())
- # setup package.*
- #f.addStep(portages.SetPackageUse())
# setup env
+ f.addStep(portage.SetEnvDefault())
+ # setup package.*
+ #f.addStep(portage.SetPackageDefault())
# setup files in /etc if needed
# run --regen if needed on repo
# update packages before any tests
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 2153b79..b91f158 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -139,7 +139,6 @@ class Model(base.DBConnectorComponent):
sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
- # FIXME: directorys should be moved to own table
sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'), nullable=False),
sa.Column('value', sa.String(255), nullable=False),
)
@@ -162,6 +161,32 @@ class Model(base.DBConnectorComponent):
sa.Column('value', sa.String(255), nullable=False),
)
+ # projects etc/portage/env settings
+ projects_portages_env = sautils.Table(
+ "projects_portages_env", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('makeconf_id', sa.String(255),
+ sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('name', sa.String(255), nullable=False),
+ sa.Column('value', sa.String(255), nullable=False),
+ )
+
+ # projects etc/portage/package.* settings
+ projects_portage_package = sautils.Table(
+ "projects_portage_package", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('directorys', sa.Enum('use', 'accept_keywords', 'env'), nullable=False),
+ sa.Column('value1', sa.String(255), nullable=False),
+ sa.Column('value2', sa.String(255), nullable=True),
+ )
+
projects_emerge_options = sautils.Table(
"projects_emerge_options", metadata,
sa.Column('id', sa.Integer, primary_key=True),
diff --git a/buildbot_gentoo_ci/db/portages.py b/buildbot_gentoo_ci/db/portages.py
index 428fb84..86321e3 100644
--- a/buildbot_gentoo_ci/db/portages.py
+++ b/buildbot_gentoo_ci/db/portages.py
@@ -36,6 +36,20 @@ class PortagesConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getVariableById(self, id):
+ def thd(conn):
+ tbl = self.db.model.portages_makeconf
+ q = tbl.select()
+ q = q.where(tbl.c.id == id)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
id=row.id,
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 37d401c..37ae2f1 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -130,6 +130,28 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectPortageEnvByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_portages_env
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ return [self._row2dict_projects_portages_env(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getProjectPortagePackageByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_portages_package
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ return [self._row2dict_projects_portages_package(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
@defer.inlineCallbacks
def getProjectEmergeOptionsByUuid(self, uuid):
def thd(conn):
@@ -190,6 +212,28 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
build_id=0
)
+ def _row2dict_projects_portages_env(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ makeconf_id=row.makeconf_id,
+ name=row.name,
+ value=row.value
+ )
+
+ def _row2dict_projects_portages_package(self, conn, row):
+ if row.value2 == '':
+ value2 = None
+ else:
+ value2 = row.value2
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ directorys=row.directorys,
+ value1=row.value1,
+ value2=value2
+ )
+
def _row2dict_projects_emerge_options(self, conn, row):
return dict(
id=row.id,
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
new file mode 100644
index 0000000..52daf83
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -0,0 +1,97 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.plugins import steps
+
+class SetPackageDefault(BuildStep):
+
+ name = 'SetPackageDefault'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ aftersteps_list = []
+ packagedir_list = []
+ packagedir_list.append('env')
+ #FIXME:
+ # get list what dir we need to make from db
+ # create the dirs
+ for packagedir in packagedir_list:
+ aftersteps_list.append(steps.MakeDirectory(dir='package.' + packagedir,
+ workdir='/etc/portage/'))
+ #FIXME:
+ # add the needed package.* settings from db
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
+
+class SetEnvDefault(BuildStep):
+
+ name = 'SetEnvDefault'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ def getPortageEnv(self, portage_env_data, portage_env_dict = {}):
+ for project_portage_env in portage_env_data:
+ if not project_portage_env['name'] in portage_env_dict:
+ portage_env_dict[project_portage_env['name']] = {
+ project_portage_env['makeconf_id'] : []
+ }
+ if not project_portage_env['makeconf_id'] in portage_env_dict[project_portage_env['name']]:
+ portage_env_dict[project_portage_env['name']][project_portage_env['makeconf_id']] = []
+ portage_env_dict[project_portage_env['name']][project_portage_env['makeconf_id']].append(project_portage_env['value'])
+ return portage_env_dict
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+ aftersteps_list = []
+ separator1 = '\n'
+ separator2 = ' '
+ # create the dir
+ aftersteps_list.append(steps.MakeDirectory(dir='env',
+ workdir='/etc/portage/'))
+ #FIXME:
+ # add env settings from the db
+ default_project_portage_env_data = yield self.gentooci.db.projects.getProjectPortageEnvByUuid(default_project_data['uuid'])
+ project_portage_env_data = yield self.gentooci.db.projects.getProjectPortageEnvByUuid(project_data['uuid'])
+ project_portage_env_dict = yield self.getPortageEnv(default_project_portage_env_data, portage_env_dict = {})
+ project_portage_env_dict = yield self.getPortageEnv(project_portage_env_data, portage_env_dict = project_portage_env_dict)
+ print(project_portage_env_dict)
+ for k, v in project_portage_env_dict.items():
+ env_strings = []
+ for a, b in v.items():
+ variable_data = yield self.gentooci.db.portages.getVariableById(a)
+ env_variable_string = variable_data['variable'] + '="' + separator2.join(b) + '"'
+ env_strings.append(env_variable_string)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(separator1.join(env_strings) + separator1,
+ workerdest=k + '.conf',
+ workdir='/etc/portage/env/')
+ ])
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
@ 2021-03-11 1:14 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2021-03-11 1:14 UTC (permalink / raw
To: gentoo-commits
commit: 5176761ad56557e45359729d7b2989dadeace7b5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 11 01:14:02 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Mar 11 01:14:02 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5176761a
Add GitPuller to change_source and change cpv_changes to git_changes
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/change_source.py | 18 ++++++
buildbot_gentoo_ci/config/schedulers.py | 40 +++++++++----
buildbot_gentoo_ci/db/model.py | 10 +++-
buildbot_gentoo_ci/db/repositorys.py | 48 ++++++++++-----
buildbot_gentoo_ci/steps/update_db.py | 95 +++++++++++++++---------------
5 files changed, 134 insertions(+), 77 deletions(-)
diff --git a/buildbot_gentoo_ci/config/change_source.py b/buildbot_gentoo_ci/config/change_source.py
new file mode 100644
index 0000000..454ed4a
--- /dev/null
+++ b/buildbot_gentoo_ci/config/change_source.py
@@ -0,0 +1,18 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from buildbot.plugins import changes, util
+
+#FIXME:
+# Get the repositorys info from the gentoo-ci db
+def gentoo_change_source(cs=[]):
+ cs.append(changes.GitPoller(
+ repourl='https://github.com/gentoo/gentoo.git',
+ branches=True,
+ workdir= 'repositorys' + '/gentoo.git/',
+ pollInterval=240,
+ pollRandomDelayMin=20,
+ pollRandomDelayMax=60,
+ project='gentoo'
+ ))
+ return cs
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index 7cfb122..0cbec96 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -4,7 +4,7 @@
from buildbot.plugins import schedulers, util
@util.renderer
-def builderUpdateDbNames(self, props):
+def builderUpdateDbNames(props):
builders = set()
for f in props.files:
if f.endswith('.ebuild'):
@@ -12,22 +12,38 @@ def builderUpdateDbNames(self, props):
return list(builders)
@util.renderer
-def cpvUpdateDb(props):
- cpv_changes = []
- for f in props.files:
- if f.endswith('.ebuild'):
- cppv = f.split('.eb', 0)
- cpv = cppv.split('/', 0) + '/' + cppv.split('/', 2)
- if not cpv in cpv_changes:
- cpv_changes.append(cpv)
- return cpv_changes
+def gitUpdateDb(props):
+ git_changes = []
+ print(props.changes)
+ for k in props.changes:
+ change_data = {}
+ print(k)
+ change_data['cpvs'] = []
+ for v in k['files']:
+ if v.endswith('.ebuild'):
+ c = v.split('/')[0]
+ pv = v.split('/')[2][:-7]
+ cpv = c + '/' + pv
+ print(cpv)
+ change_data['cpvs'].append(cpv)
+ if k['repository'].endswith('.git'):
+ for v in k['repository'].split('/'):
+ if v.endswith('.git'):
+ change_data['repository'] = v[:-4]
+ change_data['author'] = k['author']
+ change_data['committer'] = k['committer']
+ change_data['comments'] = k['comments']
+ change_data['revision'] = k['revision']
+ git_changes.append(change_data)
+ print(git_changes)
+ return git_changes
def gentoo_schedulers():
scheduler_update_db = schedulers.SingleBranchScheduler(
name='scheduler_update_db',
treeStableTimer=60,
properties = {
- 'cpv_changes' : cpvUpdateDb,
+ 'git_changes' : gitUpdateDb,
},
builderNames = builderUpdateDbNames,
change_filter=util.ChangeFilter(branch='master'),
@@ -60,7 +76,7 @@ def gentoo_schedulers():
builderNames=["run_build_request"])
s = []
s.append(test_updatedb)
- #s.append(scheduler_update_db)
+ s.append(scheduler_update_db)
s.append(update_cpv_data)
s.append(update_v_data)
s.append(build_request_data)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 2221ba4..939059c 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -71,14 +71,15 @@ class Model(base.DBConnectorComponent):
# description of the repository
sa.Column('description', sa.Text, nullable=True),
sa.Column('mirror_url', sa.String(255), nullable=True),
+ sa.Column('type', sa.Enum('gitpuller'), nullable=False, default='gitpuller'),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
sa.Column('ebuild', sa.Boolean, default=False),
)
# Use by GitPoller
- repository_gitpuller = sautils.Table(
- "repository_gitpuller", metadata,
+ repositorys_gitpullers = sautils.Table(
+ "repositorys_gitpullers", metadata,
# unique id per repository
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('repository_uuid', sa.String(36),
@@ -86,7 +87,10 @@ class Model(base.DBConnectorComponent):
nullable=False),
sa.Column('project', sa.String(255), nullable=False, default='gentoo'),
sa.Column('url', sa.String(255), nullable=False),
- sa.Column('branche', sa.String(255), nullable=False, default='master'),
+ sa.Column('branches', sa.String(255), nullable=False, default='all'),
+ sa.Column('poll_interval', sa.Integer, nullable=False, default=600),
+ sa.Column('poll_random_delay_min', sa.Integer, nullable=False, default=600),
+ sa.Column('poll_random_delay_max', sa.Integer, nullable=False, default=600),
)
projects = sautils.Table(
diff --git a/buildbot_gentoo_ci/db/repositorys.py b/buildbot_gentoo_ci/db/repositorys.py
index 4660530..6a4ef83 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -25,16 +25,18 @@ from buildbot.db import base
class RepositorysConnectorComponent(base.DBConnectorComponent):
- def getRepositorys(self):
+ @defer.inlineCallbacks
+ def getAllRepositorysByEnableAuto(self):
def thd(conn):
tbl = self.db.model.repositorys
q = tbl.select()
q = q.where(tbl.c.enable == 1,
tbl.c.auto == 1
)
- r = conn.execute(q)
- return [dict(row) for row in r.fetchall()]
- return self.db.pool.do(thd)
+ return [self._row2dict(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
@defer.inlineCallbacks
def getRepositoryByName(self, name):
@@ -64,18 +66,19 @@ class RepositorysConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
- def getRepositorysGitPoller(self):
+ @defer.inlineCallbacks
+ def getGitPollerByUuid(self, uuid):
def thd(conn):
- tblr = self.db.model.repositorys
- tblrg = self.db.model.repository_gitpuller
- from_clause = tblr.join(tblg, tblr.c.uuid == tblrg.c.repository_uuid)
- q = sa.select([tblrg]).select_from(
- from_clause).where(tblr.c.enabled == 1,
- tblr.c.auto == 1
- )
- r = conn.execute(q)
- return [dict(row) for row in r.fetchall()]
- return self.db.pool.do(thd)
+ tbl = self.db.model.repositorys_gitpullers
+ q = tbl.select()
+ q = q.where(tbl.c.repository_uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict_gitpuller(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
def _row2dict(self, conn, row):
return dict(
@@ -85,5 +88,18 @@ class RepositorysConnectorComponent(base.DBConnectorComponent):
mirror_url=row.mirror_url,
auto=row.auto,
enabled=row.enabled,
- ebuild=row.ebuild
+ ebuild=row.ebuild,
+ type=row.type
+ )
+
+ def _row2dict_gitpuller(self, conn, row):
+ return dict(
+ id=row.id,
+ repository_uuid=row.repository_uuid,
+ project=row.project,
+ url=row.url,
+ branches=row.branches,
+ poll_interval=row.poll_interval,
+ poll_random_delay_min=row.poll_random_delay_min,
+ poll_random_delay_max=row.poll_random_delay_max
)
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index 3f20cb9..94d0b4f 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -37,18 +37,22 @@ class GetDataGentooCiProject(BuildStep):
if self.profile_repository_data is None:
log.err('No data for repository in the database')
return FAILURE
- #self.repository = self.getProperty("repository")
- self.repository = 'gentoo'
- self.repository_data = yield self.gentooci.db.repositorys.getRepositoryByName(self.repository)
print(self.project_data)
print(self.project_repository_data)
print(self.profile_repository_data)
- print(self.getProperty("cpv_changes"))
- print(self.repository_data)
+ print(self.getProperty("git_changes"))
+ print(self.getProperty("repository"))
+ repository = False
+ self.repository_data = False
+ if self.getProperty("repository").endswith('.git'):
+ for v in self.getProperty("repository").split('/'):
+ if v.endswith('.git'):
+ repository = v[:-4]
+ if repository:
+ self.repository_data = yield self.gentooci.db.repositorys.getRepositoryByName(repository)
self.setProperty("project_data", self.project_data, 'project_data')
self.setProperty("project_repository_data", self.project_repository_data, 'project_repository_data')
self.setProperty("profile_repository_data", self.profile_repository_data, 'profile_repository_data')
- self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
self.setProperty("repository_data", self.repository_data, 'repository_data')
return SUCCESS
@@ -82,12 +86,9 @@ class CheckPathGentooCiProject(BuildStep):
is_dir = False
success = False
print("isdir(%s): %s" %(x, is_dir))
+ print(self.getProperty("builddir"))
if not success:
return FAILURE
- self.setProperty("project_data", self.project_data, 'project_data')
- self.setProperty("project_repository_data", self.project_repository_data, 'project_repository_data')
- self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
- self.setProperty("repository_data", self.repository_data, 'repository_data')
return SUCCESS
class CheckProjectGentooCiProject(BuildStep):
@@ -111,9 +112,6 @@ class CheckProjectGentooCiProject(BuildStep):
print("project portage conf has error %s" %(str(e)))
return FAILURE
self.setProperty("config_root", self.config_root, 'config_root')
- self.setProperty("project_data", self.project_data, 'project_data')
- self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
- self.setProperty("repository_data", self.getProperty("repository_data"), 'repository')
return SUCCESS
class CheckCPVGentooCiProject(BuildStep):
@@ -122,41 +120,46 @@ class CheckCPVGentooCiProject(BuildStep):
@defer.inlineCallbacks
def run(self):
- #self.cpv_changes = self.getProperty("cpv_changes")
- self.cpv_changes = []
- self.cpv_changes.append('dev-python/django-3.1.7')
- self.cpv_changes.append('dev-python/scrypt-0.8.16')
- print(self.cpv_changes)
- print(self.getProperty("repository_data"))
- # check if cpv_change is a string or a list
- if isinstance(self.cpv_changes, list):
- self.cpv_list = self.cpv_changes
- else:
- self.cpv_list = []
- self.cpv_list.append(self.cpv_changes)
+ self.git_changes = self.getProperty("git_changes")
+ print(self.git_changes)
+ # check if git_change is a string or a list
+ if not isinstance(self.git_changes, list):
+ return FAILURE
self.success = True
addStepUpdateCPVData = []
- for cpv in sorted(self.cpv_list):
- # check that cpv is valied
- if catpkgsplit(cpv) is None:
- log.msg("%s is not vaild package name" % cpv)
- self.success = False
- else:
- # call update_cpv_data
- addStepUpdateCPVData.append(
- steps.Trigger(
- schedulerNames=['update_cpv_data'],
- waitForFinish=False,
- updateSourceStamp=False,
- set_properties={
- 'cpv' : cpv,
- 'config_root' : self.getProperty("config_root"),
- 'project_data' : self.getProperty("project_data"),
- 'repository_data' : self.getProperty("repository_data"),
- }
- )
- )
- yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
+ for change_data in self.git_changes:
+ # make a trigger for all cpv in the list
+ for cpv in change_data['cpvs']:
+ # check that cpv is valied
+ if catpkgsplit(cpv) is None:
+ log.msg("%s is not vaild package name" % cpv)
+ self.success = False
+ else:
+ if change_data['repository'] != self.getProperty("repository_data")['name']:
+ log.msg("%s don't match" % change_data['repository'])
+ self.success = False
+ else:
+ revision_data = {}
+ revision_data['author'] = change_data['author']
+ revision_data['committer'] = change_data['committer']
+ revision_data['comments'] = change_data['comments']
+ revision_data['revision'] = change_data['revision']
+ # call update_cpv_data
+ addStepUpdateCPVData.append(
+ steps.Trigger(
+ schedulerNames=['update_cpv_data'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : cpv,
+ 'config_root' : self.getProperty("config_root"),
+ 'project_data' : self.getProperty("project_data"),
+ 'repository_data' : self.getProperty("repository_data"),
+ 'revision_data' : revision_data,
+ }
+ )
+ )
+ yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
if self.success is False:
return FAILURE
return SUCCESS
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/
@ 2022-07-27 19:32 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2022-07-27 19:32 UTC (permalink / raw
To: gentoo-commits
commit: aba0962b8f901301302a2fefacac14f8cd34a51b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 27 19:32:09 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jul 27 19:32:09 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aba0962b
Move UpdateRepos to repos.py and support gitlab stage
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 4 +-
buildbot_gentoo_ci/db/model.py | 12 ++++-
buildbot_gentoo_ci/db/repositorys.py | 19 +++++++-
buildbot_gentoo_ci/steps/builders.py | 40 ----------------
buildbot_gentoo_ci/steps/repos.py | 73 +++++++++++++++++++++++++++++-
5 files changed, 102 insertions(+), 46 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 67447aa..7f2dff9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -46,7 +46,7 @@ def update_db_cpv():
# set needed Propertys
f.addStep(package.SetupPropertys())
# update the repositorys listed in project_repository
- f.addStep(builders.UpdateRepos())
+ f.addStep(repos.UpdateRepos())
# add repo.conf
#f.addStep(portage.SetReposConf())
# if categorys in db
@@ -99,7 +99,7 @@ def run_build_request():
# set needed Propertys
f.addStep(builders.SetupPropertys())
# update the repositorys listed in project_repository
- f.addStep(builders.UpdateRepos())
+ f.addStep(repos.UpdateRepos())
# Clean and add new /etc/portage
#NOTE: remove the symlink befor the dir
#f.addStep(buildbot_steps.ShellCommand(
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 908ebb5..d6e9860 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -71,10 +71,18 @@ class Model(base.DBConnectorComponent):
# description of the repository
sa.Column('description', sa.Text, nullable=True),
sa.Column('url', sa.String(255), nullable=True),
- sa.Column('type', sa.Enum('gitpuller'), nullable=False, default='gitpuller'),
+ sa.Column('branch', sa.String(255), nullable=True),
+ sa.Column('type', sa.Enum('git', 'gitlab'), nullable=False, default='git'),
+ sa.Column('mode', sa.Enum('full', 'incremental'), nullable=False, default='incremental'),
+ sa.Column('method', sa.Enum('clobber', 'fresh', 'clean', 'copy'), nullable=False, default='fresh'),
+ sa.Column('alwaysuselatest', sa.Boolean, default=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
- sa.Column('ebuild', sa.Boolean, default=False),
+ sa.Column('ebuild', sa.Boolean, default=True),
+ sa.Column('merge', sa.Boolean, default=False),
+ # filenames for SecretInAFile
+ sa.Column('sshprivatekey', sa.String(50), nullable=True),
+ sa.Column('sshhostkey', sa.String(50), nullable=True),
)
# Use by GitPoller
diff --git a/buildbot_gentoo_ci/db/repositorys.py b/buildbot_gentoo_ci/db/repositorys.py
index 8bac1a1..4d5d048 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -91,15 +91,32 @@ class RepositorysConnectorComponent(base.DBConnectorComponent):
yield self.db.pool.do(thd)
def _row2dict(self, conn, row):
+ if row.branch == 'none':
+ branch = False
+ else:
+ branch = row.branch
+ if row.sshprivatekey == 'none':
+ sshprivatekey = False
+ sshhostkey = False
+ else:
+ sshprivatekey = row.sshprivatekey
+ sshhostkey = row.sshhostkey
return dict(
uuid=row.uuid,
name=row.name,
description=row.description,
url=row.url,
+ type=row.type,
+ branch=branch,
+ mode=row.mode,
+ method=row.method,
+ alwaysuselatest=row.alwaysuselatest,
auto=row.auto,
enabled=row.enabled,
ebuild=row.ebuild,
- type=row.type
+ merge=row.merge,
+ sshprivatekey=sshprivatekey,
+ sshhostkey=sshhostkey
)
def _row2dict_gitpuller(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 383ae60..a54c68c 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -343,46 +343,6 @@ class SetupPropertys(BuildStep):
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for project', self.getProperty('project_data')['name']])
return SUCCESS
-# Should be moved to repos.py
-class UpdateRepos(BuildStep):
-
- name = 'UpdateRepos'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- @defer.inlineCallbacks
- def run(self):
- self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- portage_repos_path = self.getProperty('portage_repos_path')
- project_data = self.getProperty('project_data')
- # update/add all repos that in project_repository for the project
- projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
- for project_repository_data in projects_repositorys_data:
- repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
- if self.getProperty('rootworkdir'):
- repository_path = os.path.join(self.getProperty('rootworkdir'), portage_repos_path[1:], repository_data['name'])
- else:
- repository_path = os.path.join(portage_repos_path, repository_data['name'], '')
- yield self.build.addStepsAfterCurrentStep([
- steps.Git(repourl=repository_data['url'],
- name = 'Git pull ' + repository_data['name'],
- mode='full',
- submodules=True,
- alwaysUseLatest=True,
- workdir=repository_path,
- #FIXME: set filenames in repositorys db
- sshPrivateKey = util.Secret("gitlab.gentoo.org_gentoo-ci.priv"),
- sshHostKey = util.Secret("gitlab.gentoo.org.host")
- )
- ])
- return SUCCESS
-
class RunEmerge(BuildStep):
description = 'Running'
diff --git a/buildbot_gentoo_ci/steps/repos.py b/buildbot_gentoo_ci/steps/repos.py
index ae63c78..5b4b623 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -11,7 +11,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
-from buildbot.plugins import steps
+from buildbot.plugins import steps, util
from buildbot.config import error as config_error
class CheckPathRepositoryLocal(BuildStep):
@@ -168,3 +168,74 @@ class CheckRepository(BuildStep):
return FAILURE
#yield self.gentooci.db.repositorys.updateGitPollerTime(repository_uuid)
return SUCCESS
+
+class UpdateRepos(BuildStep):
+
+ name = 'UpdateRepos'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ # update/add all repos that in project_repository for the project
+ projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+ for project_repository_data in projects_repositorys_data:
+ repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+ print(repository_data)
+ if repository_data['auto'] and repository_data['enabled']:
+ if self.getProperty('rootworkdir'):
+ repository_path = os.path.join(self.getProperty('rootworkdir'), portage_repos_path[1:], repository_data['name'])
+ else:
+ repository_path = os.path.join(portage_repos_path, repository_data['name'], '')
+ if repository_data['branch']:
+ branch = repository_data['branch']
+ else:
+ branch = 'HEAD'
+ # filenames to use with Secret
+ if repository_data['sshprivatekey']:
+ sshprivatekey = util.Secret(repository_data['sshprivatekey'])
+ sshhostkey = util.Secret(repository_data['sshhostkey'])
+ else:
+ sshprivatekey = None
+ sshhostkey = None
+ GitdescriptionDone = ' '.join([repository_data['type'], 'pull', repository_data['name'], branch])
+ if repository_data['type'] == 'git':
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Git(repourl=repository_data['url'],
+ name = 'RunGit',
+ descriptionDone=GitdescriptionDone,
+ branch = branch,
+ mode=repository_data['mode'],
+ method=repository_data['method'],
+ submodules=True,
+ alwaysUseLatest=repository_data['alwaysuselatest'],
+ workdir=repository_path,
+ sshPrivateKey = sshprivatekey,
+ sshHostKey = sshhostkey
+ )
+ ])
+ if repository_data['type'] =='gitlab':
+ yield self.build.addStepsAfterCurrentStep([
+ steps.GitLab(repourl=repository_data['url'],
+ name = 'RunGit',
+ descriptionDone=GitdescriptionDone,
+ branch = branch,
+ mode=repository_data['mode'],
+ method=repository_data['method'],
+ submodules=True,
+ alwaysUseLatest=repository_data['alwaysuselatest'],
+ workdir=repository_path,
+ sshPrivateKey = sshprivatekey,
+ sshHostKey = sshhostkey
+ )
+ ])
+ return SUCCESS
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2022-07-27 19:32 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-03-11 1:14 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2022-07-27 19:32 Magnus Granberg
2021-02-27 22:46 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox