* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2021-01-17 21:20 Magnus Granberg
0 siblings, 0 replies; 4+ messages in thread
From: Magnus Granberg @ 2021-01-17 21:20 UTC (permalink / raw
To: gentoo-commits
commit: 500c23afb393cc5330b82ba9d86a966e0abe6289
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 17 21:20:23 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 17 21:20:23 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=500c23af
Add BuildFactory build_request_check and start of run_build_request
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 8 +++
buildbot_gentoo_ci/config/buildfactorys.py | 12 ++++
buildbot_gentoo_ci/config/schedulers.py | 3 +
buildbot_gentoo_ci/db/model.py | 6 +-
buildbot_gentoo_ci/db/projects.py | 37 ++++++++++-
buildbot_gentoo_ci/steps/builders.py | 98 ++++++++++++++++++++++++++++++
buildbot_gentoo_ci/steps/version.py | 4 +-
7 files changed, 163 insertions(+), 5 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index 9fef46a..69dd840 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -38,4 +38,12 @@ def gentoo_builders(b=[]):
factory=buildfactorys.build_request_check()
)
)
+ # FIXME: get workers from db
+ # Use multiplay workers
+ b.append(util.BuilderConfig(
+ name='run_build_request',
+ workername='updatedb_1',
+ factory=buildfactorys.run_build_request()
+ )
+ )
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index ba96e5a..e0a6634 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -8,6 +8,7 @@ from buildbot_gentoo_ci.steps import update_db
from buildbot_gentoo_ci.steps import category
from buildbot_gentoo_ci.steps import package
from buildbot_gentoo_ci.steps import version
+from buildbot_gentoo_ci.steps import builders
def update_db_check():
f = util.BuildFactory()
@@ -78,4 +79,15 @@ def update_db_v():
def build_request_check():
f = util.BuildFactory()
# FIXME: 4
+ # get project_data
+ # check what tests to do
+ # triggger build request
+ f.addStep(builders.GetProjectRepositoryData())
+ return f
+
+def run_build_request():
+ f = util.BuildFactory()
+ # FIXME: 5
+ # Check if all is setup on the worker
+ f.addStep(builders.SetupBuildWorker())
return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index 687b52f..7cfb122 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -56,10 +56,13 @@ def gentoo_schedulers():
builderNames=["update_v_data"])
build_request_data = schedulers.Triggerable(name="build_request_data",
builderNames=["build_request_data"])
+ run_build_request = schedulers.Triggerable(name="run_build_request",
+ builderNames=["run_build_request"])
s = []
s.append(test_updatedb)
#s.append(scheduler_update_db)
s.append(update_cpv_data)
s.append(update_v_data)
s.append(build_request_data)
+ s.append(run_build_request)
return s
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 9ad1331..b051b23 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -109,7 +109,7 @@ class Model(base.DBConnectorComponent):
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE'),
nullable=False),
- sa.Column('unstable', sa.Boolean, default=False),
+ sa.Column('status', sa.Enum('stable','unstable','all'), nullable=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
sa.Column('created_by', sa.Integer,
@@ -121,12 +121,14 @@ class Model(base.DBConnectorComponent):
projects_repositorys = sautils.Table(
"projects_repositorys", metadata,
sa.Column('id', sa.Integer, primary_key=True),
- sa.Column('projects_uuid', sa.String(36),
+ sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('repository_uuid', sa.String(36),
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
+ sa.Column('auto', sa.Boolean, default=False),
+ sa.Column('pkgcheck', sa.Boolean, default=False),
)
keywords = sautils.Table(
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 4a14bac..065c9a0 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -53,6 +53,32 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects
+ q = tbl.select()
+ q = q.where(tbl.c.uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getProjectRepositorysByUuid(self, uuid, auto=True):
+ def thd(conn):
+ tbl = self.db.model.projects_repositorys
+ q = tbl.select()
+ q = q.where(tbl.c.repository_uuid == uuid)
+ q = q.where(tbl.c.auto == auto)
+ return [self._row2dict_projects_repositorys(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -62,8 +88,17 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
profile=row.profile,
profile_repository_uuid=row.profile_repository_uuid,
keyword_id=row.keyword_id,
- unstable=row.unstable,
+ status=row.status,
auto=row.auto,
enabled=row.enabled,
created_by=row.created_by
)
+
+ def _row2dict_projects_repositorys(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ repository_uuid=row.repository_uuid,
+ auto=row.auto,
+ pkgcheck=row.pkgcheck
+ )
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
new file mode 100644
index 0000000..29389a1
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -0,0 +1,98 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.plugins import steps
+
+class TriggerRunBuildRequest(BuildStep):
+
+ name = 'TriggerRunBuildRequest'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Trigger(
+ schedulerNames=['run_build_request'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : self.getProperty("cpv"),
+ 'version_data' : self.getProperty("version_data"),
+ 'projectrepository_data' : self.getProperty('projectrepository_data'),
+ 'repository_data' : self.getProperty("repository_data"),
+ 'use_data' : self.getProperty("use_data"),
+ 'fullcheck' : self.getProperty("fullcheck"),
+ }
+ )])
+ return SUCCESS
+
+class GetProjectRepositoryData(BuildStep):
+
+ name = 'GetProjectRepositoryData'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.projectrepositorys_data = yield self.gentooci.db.projects.getProjectRepositorysByUuid(self.getProperty("repository_data")['uuid'])
+ if self.projectrepositorys_data is None:
+ print('No Projects have this %s repository for testing' % self.getProperty("repository_data")['name'])
+ return SUCCESS
+ # for loop to get all the projects that have the repository
+ for projectrepository_data in self.projectrepositorys_data:
+ # get project data
+ project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
+ # check if auto, enabled and not in config.project['project']
+ if project_data['auto'] is True and project_data['enabled'] is True and project_data['name'] != self.gentooci.config.project['project']:
+ # set Property projectrepository_data so we can use it in the trigger
+ self.setProperty('projectrepository_data', projectrepository_data, 'projectrepository_data')
+ self.setProperty('use_data', None, 'use_data')
+ # get name o project keyword
+ project_keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
+ # if not * (all keywords)
+ if project_keyword_data['name'] != '*' or project_data['status'] == 'all':
+ self.setProperty('fullcheck', False, 'fullcheck')
+ # get status of the keyword on cpv
+ version_keywords_data = self.getProperty("version_keyword_dict")[project_keyword_data['name']]
+ # if unstable trigger BuildRequest on cpv
+ if project_data['status'] == version_keywords_data['status']:
+ yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
+ return SUCCESS
+
+class SetupBuildWorker(BuildStep):
+
+ name = 'SetupBuildWorker'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ print('build this %s' % self.getProperty("cpv"))
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index e274e65..5430e6e 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -194,7 +194,7 @@ class TriggerBuildCheck(BuildStep):
@defer.inlineCallbacks
def run(self):
- yield steps.Trigger(
+ yield self.build.addStepsAfterCurrentStep([steps.Trigger(
schedulerNames=['build_request_data'],
waitForFinish=False,
updateSourceStamp=False,
@@ -204,7 +204,7 @@ class TriggerBuildCheck(BuildStep):
'version_keyword_dict' : self.getProperty('version_keyword_dict'),
'repository_data' : self.getProperty("repository_data"),
}
- )
+ )])
return SUCCESS
class DeleteOldVersion(BuildStep):
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2021-01-31 17:42 Magnus Granberg
0 siblings, 0 replies; 4+ messages in thread
From: Magnus Granberg @ 2021-01-31 17:42 UTC (permalink / raw
To: gentoo-commits
commit: c667f7b39ec20d30678e758dc74d669f6634797a
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 31 17:43:03 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 31 17:43:03 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c667f7b3
Add SetMakeConf
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 3 +
buildbot_gentoo_ci/db/connector.py | 2 +
buildbot_gentoo_ci/db/model.py | 18 ++++++
buildbot_gentoo_ci/db/portages.py | 43 ++++++++++++++
buildbot_gentoo_ci/db/projects.py | 21 +++++++
buildbot_gentoo_ci/steps/builders.py | 95 ++++++++++++++++++++++++++++++
6 files changed, 182 insertions(+)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 3fad219..861a4e1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,6 +88,7 @@ def build_request_check():
def run_build_request():
f = util.BuildFactory()
# FIXME: 5
+ # Move the etc/portage stuff to is own file
# set needed Propertys
f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
@@ -105,4 +106,6 @@ def run_build_request():
f.addStep(builders.SetReposConf())
# update the repositorys listed in project_repository
f.addStep(builders.UpdateRepos())
+ # setup make.conf
+ f.addStep(builders.SetMakeConf())
return f
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index 35febe6..7d218ab 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -36,6 +36,7 @@ from buildbot_gentoo_ci.db import categorys
from buildbot_gentoo_ci.db import packages
from buildbot_gentoo_ci.db import versions
from buildbot_gentoo_ci.db import keywords
+from buildbot_gentoo_ci.db import portages
upgrade_message = textwrap.dedent("""\
@@ -81,6 +82,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.packages = packages.PackagesConnectorComponent(self)
self.versions = versions.VersionsConnectorComponent(self)
self.keywords = keywords.KeywordsConnectorComponent(self)
+ self.portages = portages.PortagesConnectorComponent(self)
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 596d04e..bd111be 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -143,6 +143,24 @@ class Model(base.DBConnectorComponent):
sa.Column('value', sa.String(255), nullable=False),
)
+ portages_makeconf = sautils.Table(
+ "portages_makeconf", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('variable', sa.String(255), nullable=False),
+ )
+
+ projects_portages_makeconf = sautils.Table(
+ "projects_portages_makeconf", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('makeconf_id', sa.String(255),
+ sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('value', sa.String(255), nullable=False),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
diff --git a/buildbot_gentoo_ci/db/portages.py b/buildbot_gentoo_ci/db/portages.py
new file mode 100644
index 0000000..428fb84
--- /dev/null
+++ b/buildbot_gentoo_ci/db/portages.py
@@ -0,0 +1,43 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class PortagesConnectorComponent(base.DBConnectorComponent):
+
+ @defer.inlineCallbacks
+ def getVariables(self):
+ def thd(conn):
+ tbl = self.db.model.portages_makeconf
+ q = tbl.select()
+ return [self._row2dict(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def _row2dict(self, conn, row):
+ return dict(
+ id=row.id,
+ variable=row.variable
+ )
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 00e1569..1f19a00 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -118,6 +118,18 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectMakeConfById(self, uuid, id):
+ def thd(conn):
+ tbl = self.db.model.projects_portages_makeconf
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.makeconf_id == id)
+ return [self._row2dict_projects_portages_makeconf(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -149,3 +161,12 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
directorys=row.directorys,
value=row.value
)
+
+ def _row2dict_projects_portages_makeconf(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ makeconf_id=row.makeconf_id,
+ value=row.value,
+ build_id=0
+ )
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index d3b3607..085d096 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -219,3 +219,98 @@ class UpdateRepos(BuildStep):
workdir=os.path.join(repository_path, ''))
])
return SUCCESS
+
+class SetMakeConf(BuildStep):
+
+ name = 'SetMakeConf'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ #FIXME: Make a dict before we pass it to the make.conf
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ makeconf_variables_data = yield self.gentooci.db.portages.getVariables()
+ separator1 = '\n'
+ separator2 = ' '
+ makeconf_list = []
+ for k in makeconf_variables_data:
+ makeconf_variables_values_data = yield self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
+ makeconf_variable_list = []
+ # we add some default values
+ #FIXME:
+ # we could set them in a config variables
+ # FEATURES
+ if k['variable'] == 'FEATURES':
+ makeconf_variable_list.append('xattr')
+ makeconf_variable_list.append('cgroup')
+ makeconf_variable_list.append('-news')
+ makeconf_variable_list.append('-collision-protect')
+ # EMERGE_DEFAULT_OPTS
+ if k['variable'] == 'EMERGE_DEFAULT_OPTS':
+ makeconf_variable_list.append('--buildpkg=y')
+ makeconf_variable_list.append('--rebuild-if-new-rev=y')
+ makeconf_variable_list.append('--rebuilt-binaries=y')
+ makeconf_variable_list.append('--usepkg=y')
+ makeconf_variable_list.append('--nospinner')
+ makeconf_variable_list.append('--color=n')
+ makeconf_variable_list.append('--ask=n')
+ # CFLAGS
+ if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
+ makeconf_variable_list.append('-O2')
+ makeconf_variable_list.append('-pipe')
+ makeconf_variable_list.append('-march=native')
+ makeconf_variable_list.append('-fno-diagnostics-color')
+ #FIXME:
+ # Depend on worker we may have to add a diffrent march
+ if k['variable'] == 'CXXFLAGS':
+ makeconf_variable_list.append('${CFLAGS}')
+ if k['variable'] == 'FFLAGS':
+ makeconf_variable_list.append('${FCFLAGS}')
+ if k['variable'] == 'ACCEPT_PROPERTIES':
+ makeconf_variable_list.append('-interactive')
+ if k['variable'] == 'ACCEPT_RESTRICT':
+ makeconf_variable_list.append('-fetch')
+ for v in makeconf_variables_values_data:
+ if v['build_id'] is 0:
+ makeconf_variable_list.append(v['value'])
+ if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list != []:
+ makeconf_variable_list.append('ACCEPT_LICENSE="*"')
+ if makeconf_variable_list != []:
+ makeconf_variable_string = k['variable'] + '="' + separator2.join(makeconf_variable_list) + '"'
+ makeconf_list.append(makeconf_variable_string)
+ # add hardcoded variables and values
+ #FIXME:
+ # we could set them in a config variables
+ makeconf_list.append('LC_MESSAGES=C')
+ makeconf_list.append('NOCOLOR="true"')
+ makeconf_list.append('GCC_COLORS=""')
+ makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
+ makeconf_list.append('CLEAN_DELAY=0')
+ makeconf_list.append('NOCOLOR=true')
+ makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
+ makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
+ makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
+ makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
+ # add ACCEPT_KEYWORDS from the project_data info
+ keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
+ if project_data['status'] == 'unstable':
+ makeconf_keyword = '~' + keyword_data['name']
+ else:
+ makeconf_keyword = keyword_data['name']
+ makeconf_list.append('ACCEPT_KEYWORDS="' + makeconf_keyword + '"')
+ makeconf_string = separator1.join(makeconf_list)
+ print(makeconf_string)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(makeconf_string + separator1,
+ workerdest="make.conf",
+ workdir='/etc/portage/')
+ ])
+ return SUCCESS
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2021-02-20 18:40 Magnus Granberg
0 siblings, 0 replies; 4+ messages in thread
From: Magnus Granberg @ 2021-02-20 18:40 UTC (permalink / raw
To: gentoo-commits
commit: 2dc7f260e99203e183d352601c6a479ae482697b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 20 18:39:19 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 20 18:39:19 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2dc7f260
Add RunEmerge and CheckEmergeLogs
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 21 +---
buildbot_gentoo_ci/db/model.py | 11 ++
buildbot_gentoo_ci/db/projects.py | 23 +++++
buildbot_gentoo_ci/steps/builders.py | 155 ++++++++++++++++++++++++++++-
buildbot_gentoo_ci/steps/update_db.py | 2 +-
5 files changed, 192 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 2916847..3c77762 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -115,31 +115,20 @@ def run_build_request():
# run --regen if needed on repo
# update packages before any tests
# run pretend on packages update on worker
- shell_commad_list = [
- 'emerge',
- '-uDNv',
- '--changed-deps',
- '--changed-use',
- '--pretend',
- '@world'
- ]
- f.addStep(buildbot_steps.SetPropertyFromCommandNewStyle(
- command=shell_commad_list,
- strip=True,
- extract_fn=builders.PersOutputOfEmerge,
- workdir='/'
- ))
+ f.addStep(builders.RunEmerge(step='pre-update'))
# look at the log to see if we need to do stuff
# run update package on worker
+ f.addStep(builders.RunEmerge(step='update'))
# check log
# run pretend @preserved-rebuild if needed
- # look at the log to see if we need to do stuff
- # run @preserved-rebuild
+ f.addStep(builders.RunEmerge(step='preserved-libs'))
# check log
# run depclean if set
# depclean pretend
+ f.addStep(builders.RunEmerge(step='pre-depclean'))
# look at the log to see if we need to do stuff
# depclean
+ f.addStep(builders.RunEmerge(step='depclean'))
# check log
# setup make.conf if build id has changes make.conf as dict from SetMakeConf
# setup package.* env if build id has changes
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 4c5e04c..0a2c145 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -161,6 +161,17 @@ class Model(base.DBConnectorComponent):
sa.Column('value', sa.String(255), nullable=False),
)
+ projects_emerge_options = sautils.Table(
+ "projects_emerge_options", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('oneshot', sa.Boolean, default=True),
+ sa.Column('depclean', sa.Boolean, default=True),
+ sa.Column('preserved_libs', sa.Boolean, default=True),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 1f19a00..eb8f727 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -130,6 +130,20 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectEmergeOptionsByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_emerge_options
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict_projects_emerge_options(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -170,3 +184,12 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
value=row.value,
build_id=0
)
+
+ def _row2dict_projects_emerge_options(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ oneshot=row.oneshot,
+ depclean=row.depclean,
+ preserved_libs=row.preserved_libs
+ )
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 862f0d4..c0567c1 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -15,13 +15,15 @@ from buildbot.plugins import steps
def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output = {}
emerge_output['rc'] = rc
+ emerge_output['preserved_libs'] = False
+ emerge_output['depclean'] = False
package_dict = {}
print(stderr)
+ # split the lines
for line in stdout.split('\n'):
# package list
subdict = {}
- if line.startswith('[ebuild'):
- # split the line
+ if line.startswith('[ebuild') or line.startswith('[binary'):
# if binaries
if line.startswith('[ebuild'):
subdict['binary'] = False
@@ -57,7 +59,18 @@ def PersOutputOfEmerge(rc, stdout, stderr):
subdict['python_targets'] = None
# CPU_FLAGS_X86 list
package_dict[cpv] = subdict
- #FIXME: Handling of stderr output
+ if line.startswith('>>>'):
+ #FIXME: Handling of >>> output
+ pass
+ if line.startswith('!!!'):
+ #FIXME: Handling of !!! output
+ if line.startswith('!!! existing preserved libs'):
+ pass
+ #FIXME: Handling of depclean output dict of packages that get removed or saved
+ # split the lines
+ #FIXME: Handling of stderr output
+ for line in stderr.split('\n'):
+ pass
return {
'emerge_output' : emerge_output
}
@@ -154,6 +167,8 @@ class SetupPropertys(BuildStep):
print(projectrepository_data)
project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
self.setProperty('project_data', project_data, 'project_data')
+ self.setProperty('preserved_libs', False, 'preserved-libs')
+ self.setProperty('depclean', False, 'depclean')
return SUCCESS
class SetMakeProfile(BuildStep):
@@ -311,6 +326,7 @@ class SetMakeConf(BuildStep):
makeconf_variable_list.append('--rebuilt-binaries=y')
makeconf_variable_list.append('--usepkg=y')
makeconf_variable_list.append('--binpkg-respect-use=y')
+ makeconf_variable_list.append('--binpkg-changed-deps=y')
makeconf_variable_list.append('--nospinner')
makeconf_variable_list.append('--color=n')
makeconf_variable_list.append('--ask=n')
@@ -366,3 +382,136 @@ class SetMakeConf(BuildStep):
workdir='/etc/portage/')
])
return SUCCESS
+
+class RunEmerge(BuildStep):
+
+ name = 'RunEmerge'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, step=None,**kwargs):
+ self.step = step
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+ shell_commad_list = [
+ 'emerge',
+ '-v'
+ ]
+ aftersteps_list = []
+ if self.step == 'pre-update':
+ shell_commad_list.append('-uDN')
+ shell_commad_list.append('--changed-deps')
+ shell_commad_list.append('--changed-use')
+ shell_commad_list.append('--pretend')
+ shell_commad_list.append('@world')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/'
+ ))
+ aftersteps_list.append(CheckEmergeLogs('pre-update'))
+
+ if self.step == 'update':
+ shell_commad_list.append('-uDNq')
+ shell_commad_list.append('--changed-deps')
+ shell_commad_list.append('--changed-use')
+ shell_commad_list.append('@world')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ timeout=None
+ ))
+ aftersteps_list.append(CheckEmergeLogs('update'))
+
+ if self.step == 'preserved-libs' and self.getProperty('preserved_libs'):
+ shell_commad_list.append('-q')
+ shell_commad_list.append('@preserved-rebuild')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ timeout=None
+ ))
+ aftersteps_list.append(CheckEmergeLogs('preserved-libs'))
+ self.setProperty('preserved_libs', False, 'preserved-libs')
+
+ if self.step == 'pre-depclean' and projects_emerge_options['depclean']:
+ shell_commad_list.append('--pretend')
+ shell_commad_list.append('--depclean')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/'
+ ))
+ aftersteps_list.append(CheckEmergeLogs('depclean'))
+ self.setProperty('depclean', False, 'depclean')
+
+ if self.step == 'depclean' and self.getProperty('depclean') and projects_emerge_options['depclean']:
+ shell_commad_list.append('-q')
+ shell_commad_list.append('--depclean')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/'
+ ))
+ aftersteps_list.append(CheckEmergeLogs('depclean'))
+ if not self.step is None:
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
+
+class CheckEmergeLogs(BuildStep):
+
+ name = 'CheckLogs'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, step=None,**kwargs):
+ self.step = step
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+ emerge_output = self.getProperty('emerge_output')
+ shell_commad_list = [
+ 'emerge',
+ '-v'
+ ]
+ aftersteps_list = []
+
+ #FIXME: Prosees the logs and do stuff
+ # preserved-libs
+ if emerge_output['preserved_libs'] and projects_emerge_options['preserved_libs']:
+ self.setProperty('preserved_libs', True, 'preserved-libs')
+ # depclean
+ # FIXME: check if don't remove needed stuff.
+ if emerge_output['depclean'] and projects_emerge_options['depclean']:
+ self.setProperty('depclean', True, 'depclean')
+
+ if not self.step is None:
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index ee2ba88..3f20cb9 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -124,7 +124,7 @@ class CheckCPVGentooCiProject(BuildStep):
def run(self):
#self.cpv_changes = self.getProperty("cpv_changes")
self.cpv_changes = []
- self.cpv_changes.append('dev-python/django-3.1.5')
+ self.cpv_changes.append('dev-python/django-3.1.7')
self.cpv_changes.append('dev-python/scrypt-0.8.16')
print(self.cpv_changes)
print(self.getProperty("repository_data"))
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2021-02-23 22:27 Magnus Granberg
0 siblings, 0 replies; 4+ messages in thread
From: Magnus Granberg @ 2021-02-23 22:27 UTC (permalink / raw
To: gentoo-commits
commit: c138ea64173bbc73930199f07f9eb6814ebc43f6
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 23 22:26:57 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Feb 23 22:26:57 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c138ea64
Add RunBuild
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 11 ++++
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/projects.py | 3 +-
buildbot_gentoo_ci/steps/builders.py | 84 +++++++++++++++++++++++++++++-
4 files changed, 96 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index cfe4131..b22e1dc 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -139,4 +139,15 @@ def run_build_request():
# run pkgcheck if wanted
# check log
f.addStep(builders.RunPkgCheck())
+ # check cpv match
+ f.addStep(builders.RunEmerge(step='match'))
+ # Add the needed steps for build
+ f.addStep(builders.RunBuild())
+ # clean up the worker
+ f.addStep(builders.RunEmerge(step='pre-depclean'))
+ # look at the log to see if we need to do stuff
+ # depclean
+ f.addStep(builders.RunEmerge(step='depclean'))
+ f.addStep(builders.RunEmerge(step='preserved-libs'))
+ f.addStep(builders.RunEmerge(step='depclean'))
return f
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 4d691ea..2153b79 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -129,6 +129,7 @@ class Model(base.DBConnectorComponent):
nullable=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('pkgcheck', sa.Enum('package','full','none'), default='none'),
+ sa.Column('build', sa.Boolean, default=False),
)
# projects etc/portage settings
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 9aeca50..37d401c 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -169,7 +169,8 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
project_uuid=row.project_uuid,
repository_uuid=row.repository_uuid,
auto=row.auto,
- pkgcheck=pkgcheck
+ pkgcheck=pkgcheck,
+ build=row.build
)
def _row2dict_projects_portage(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 6d46ae6..3ecb78b 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -21,6 +21,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['depclean'] = False
package_dict = {}
print(stderr)
+ emerge_output['stderr'] = stderr
# split the lines
for line in stdout.split('\n'):
# package list
@@ -69,6 +70,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
if line.startswith('!!! existing preserved libs'):
pass
#FIXME: Handling of depclean output dict of packages that get removed or saved
+ emerge_output['package'] = package_dict
# split the lines
#FIXME: Handling of stderr output
for line in stderr.split('\n'):
@@ -191,6 +193,7 @@ class SetupPropertys(BuildStep):
self.setProperty('project_data', project_data, 'project_data')
self.setProperty('preserved_libs', False, 'preserved-libs')
self.setProperty('depclean', False, 'depclean')
+ self.setProperty('cpv_build', False, 'cpv_build')
return SUCCESS
class SetMakeProfile(BuildStep):
@@ -511,7 +514,51 @@ class RunEmerge(BuildStep):
workdir='/'
))
aftersteps_list.append(CheckEmergeLogs('depclean'))
- if not self.step is None:
+
+ if self.step == 'match':
+ cpv = self.getProperty("cpv")
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ shell_commad_list.append('-pO')
+ shell_commad_list.append(c + '/' + p)
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ timeout=None
+ ))
+ aftersteps_list.append(CheckEmergeLogs('match'))
+
+ if self.step == 'pre-build':
+ shell_commad_list.append('-p')
+ shell_commad_list.append('=' + self.getProperty('cpv'))
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ timeout=None
+ ))
+ aftersteps_list.append(CheckEmergeLogs('pre-build'))
+
+ if self.step == 'build':
+ shell_commad_list.append('-q')
+ shell_commad_list.append('-1')
+ shell_commad_list.append('=' + self.getProperty('cpv'))
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ timeout=None
+ ))
+ aftersteps_list.append(CheckEmergeLogs('build'))
+
+ if not self.step is None and aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
@@ -549,7 +596,18 @@ class CheckEmergeLogs(BuildStep):
if emerge_output['depclean'] and projects_emerge_options['depclean']:
self.setProperty('depclean', True, 'depclean')
- if not self.step is None:
+ # FIXME: check if cpv match
+ if self.step == 'match'and self.getProperty('projectrepository_data')['build']:
+ if emerge_output['package'][self.getProperty('cpv')]:
+ self.setProperty('cpv_build', True, 'cpv_build')
+ print(self.getProperty('cpv_build'))
+
+ #FIXME:
+ # update package.* if needed and rerun pre-build max X times
+ if self.step == 'pre-build':
+ print(emerge_output)
+
+ if not self.step is None and aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
@@ -623,3 +681,25 @@ class CheckPkgCheckLogs(BuildStep):
# Perse the logs
# tripp irc request with pkgcheck info
return SUCCESS
+
+class RunBuild(BuildStep):
+
+ name = 'RunBuild'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ if not self.getProperty('cpv_build'):
+ return SUCCESS
+ aftersteps_list = []
+ aftersteps_list.append(RunEmerge(step='pre-build'))
+ aftersteps_list.append(RunEmerge(step='build'))
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2021-02-23 22:27 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-02-20 18:40 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2021-02-23 22:27 Magnus Granberg
2021-01-31 17:42 Magnus Granberg
2021-01-17 21:20 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox