* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, /, buildbot_gentoo_ci/config/
@ 2021-01-10 15:34 Magnus Granberg
0 siblings, 0 replies; only message in thread
From: Magnus Granberg @ 2021-01-10 15:34 UTC (permalink / raw
To: gentoo-commits
commit: d9dcf85c92b58ca427123ef2c38b8374ecdae293
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 10 15:24:41 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 15:24:41 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d9dcf85c
Add BuildFactory update_db_check
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 18 +++-
buildbot_gentoo_ci/config/buildfactorys.py | 66 ++++++++----
buildbot_gentoo_ci/config/config.py | 2 +-
buildbot_gentoo_ci/config/schedulers.py | 9 +-
buildbot_gentoo_ci/config/service.py | 6 +-
buildbot_gentoo_ci/db/connector.py | 7 +-
buildbot_gentoo_ci/db/model.py | 6 +-
buildbot_gentoo_ci/db/projects.py | 69 ++++++++++++
buildbot_gentoo_ci/db/repositorys.py | 73 +++++++++++++
buildbot_gentoo_ci/steps/__init__.py | 0
buildbot_gentoo_ci/steps/update_db.py | 162 +++++++++++++++++++++++++++++
gentooci.cfg | 2 +-
12 files changed, 383 insertions(+), 37 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index 114bb83..c069841 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from buildbot.plugins import util
@@ -7,9 +7,19 @@ from buildbot_gentoo_ci.config import buildfactorys
def gentoo_builders(b=[]):
# FIXME: get workers from db
b.append(util.BuilderConfig(
- name='update_db_packages',
- workername=[ 'updatedb_1', ],
- factory=buildfactorys.f_update_db_packages()
+ name='update_db_check',
+ workername='updatedb_1',
+ factory=buildfactorys.update_db_check()
+ )
+ )
+ # FIXME: get workers from db
+ # Use multiplay workers depend on Property(cpv)
+ # if cp do not match next one, use diffrent worker then
+ # or first cp have done its buildsteps.
+ b.append(util.BuilderConfig(
+ name='update_cpv_data',
+ workername='updatedb_1',
+ factory=buildfactorys.update_db_cp()
)
)
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index ef479a2..3fae376 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -1,28 +1,50 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-from buildbot.plugins import util, steps
-
-def f_update_db_packages():
+from buildbot.plugins import steps as buildbot_steps
+from buildbot.plugins import util
+
+from buildbot_gentoo_ci.steps import update_db
+#from buildbot_gentoo_ci.steps import category
+#from buildbot_gentoo_ci.steps import package
+
+def update_db_check():
f = util.BuildFactory()
- # FIXME:
- # Get base project from db
- # Check if base project repo for is cloned
- # Check if gentoo repo is cloned
+ # FIXME: 1
+ # Get base project data from db
+ # return project_repository, profile_repository,
+ # project
+ f.addStep(update_db.GetDataGentooCiProject())
+ # Check if base project repo is cloned
+ # Check if profile repo is cloned
# check if the profile link is there
+ # check if the repository is cloned
+ f.addStep(update_db.CheckPathGentooCiProject())
# check if etc/portage has no error
- # Get the repository some way
- repository=util.Property('repository')
- # For loop
- # for cpv in util.Property('cpv_changes')
- # check if not categorys in db
- # run update categorys db step
- # check if not package in db
- # run update package db step
- # and get foo{cpv] with metadata back
- # if new or updated package
- # else
- # run update package db step in a pool of workers
- # and get foo{cpv] with metadata back
- # if new or updated package
+ # return config_root
+ f.addStep(update_db.CheckProjectGentooCiProject())
+ # Make a for loop and trigger new builders with cpv from cpv_changes
+ # return cpv, repository, project_data, config_root
+ f.addStep(update_db.CheckCPVGentooCiProject())
+ return f
+
+def update_db_cp():
+ f = util.BuildFactory()
+ # FIXME: 2
+ # if categorys in db
+ # return category_data
+ # add check category path step at end
+ # else
+ # add category to db step
+ # return category_data
+ #f.addStep(category.CheckCGentooCiProject())
+ # if package in db
+ # return package_data
+ # add check package path step at end
+ # else
+ # add package to db step
+ # return package_data
+ #f.addStep(package.CheckPGentooCiProject())
+ # Make a for loop and trigger new builders with v from cpv
+ # return package_data, cpv, repository, project_data, config_root
return f
diff --git a/buildbot_gentoo_ci/config/config.py b/buildbot_gentoo_ci/config/config.py
index e58e67c..3fed5d2 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -49,7 +49,7 @@ from buildbot.config import ConfigErrors, error, loadConfigDict
_errors = None
DEFAULT_DB_URL = 'sqlite:///gentoo.sqlite'
-DEFAULT_REPOSITORY_BASEDIR = 'repository'
+DEFAULT_REPOSITORY_BASEDIR = '/srv/repository'
#Use GentooCiConfig.loadFromDict
@implementer(interfaces.IConfigLoader)
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index f5b19da..88d0e49 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from buildbot.plugins import schedulers, util
@@ -8,7 +8,7 @@ def builderUpdateDbNames(self, props):
builders = set()
for f in props.files:
if f.endswith('.ebuild'):
- builders.add('update_db_packages')
+ builders.add('update_db_check')
return list(builders)
@util.renderer
@@ -36,7 +36,7 @@ def gentoo_schedulers():
name="force",
buttonName="pushMe!",
label="My nice Force form",
- builderNames=['update_db_packages'],
+ builderNames=['update_db_check'],
# A completely customized property list. The name of the
# property is the name of the parameter
properties=[
@@ -50,7 +50,10 @@ def gentoo_schedulers():
default="gentoo", size=80),
])
])
+ update_cpv_data = schedulers.Triggerable(name="update_cpv_data",
+ builderNames=["update_cpv_data"])
s = []
s.append(test_updatedb)
#s.append(scheduler_update_db)
+ s.append(update_cpv_data)
return s
diff --git a/buildbot_gentoo_ci/config/service.py b/buildbot_gentoo_ci/config/service.py
index af283b5..c2c6054 100644
--- a/buildbot_gentoo_ci/config/service.py
+++ b/buildbot_gentoo_ci/config/service.py
@@ -1,8 +1,8 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from twisted.python import log
-from twisted.internet import reactor, defer
+from twisted.internet import defer
from buildbot import config as master_config
from buildbot.db import exceptions
@@ -39,5 +39,5 @@ class GentooCiService(BuildbotService):
yield self.db.setup(self.config)
except exceptions.DatabaseNotReadyError:
# (message was already logged)
- reactor.stop()
+ self.master.reactor.stop()
return
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index 682e72a..f95a121 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -15,8 +15,7 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.connector.py
# Modifyed by Gentoo Authors.
-# Copyright 2020 Gentoo Authors
-
+# Copyright 2021 Gentoo Authors
import textwrap
@@ -31,6 +30,8 @@ from buildbot.db import pool
from buildbot.util import service
from buildbot_gentoo_ci.db import model
+from buildbot_gentoo_ci.db import projects
+from buildbot_gentoo_ci.db import repositorys
upgrade_message = textwrap.dedent("""\
@@ -70,6 +71,8 @@ class DBConnector(service.ReconfigurableServiceMixin,
def setServiceParent(self, p):
yield super().setServiceParent(p)
self.model = model.Model(self)
+ self.projects = projects.ProjectsConnectorComponent(self)
+ self.repositorys = repositorys.RepositorysConnectorComponent(self)
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 8865517..1ab5990 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -99,8 +99,11 @@ class Model(base.DBConnectorComponent):
sa.Column('name', sa.String(255), nullable=False),
# description of the project
sa.Column('description', sa.Text, nullable=True),
+ sa.Column('project_repository_uuid', sa.String(36),
+ sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
+ nullable=False),
sa.Column('profile', sa.String(255), nullable=False),
- sa.Column('portage_repository_uuid', sa.Integer,
+ sa.Column('profile_repository_uuid', sa.String(36),
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('keyword_id', sa.Integer,
@@ -125,6 +128,7 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
)
+
keywords = sautils.Table(
"keywords", metadata,
# unique id per project
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
new file mode 100644
index 0000000..4a14bac
--- /dev/null
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -0,0 +1,69 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class ProjectsConnectorComponent(base.DBConnectorComponent):
+
+ @defer.inlineCallbacks
+ def getProjects(self):
+ def thd(conn):
+ tbl = self.db.model.projects
+ q = tbl.select()
+ q = q.where(tbl.c.enable == True,
+ tbl.c.auto == True
+ )
+ return [self._row2dict(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ #FIXME:join project_repository and profile_repository info
+ @defer.inlineCallbacks
+ def getProjectByName(self, name):
+ def thd(conn):
+ tbl = self.db.model.projects
+ q = tbl.select()
+ q = q.where(tbl.c.name == name)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def _row2dict(self, conn, row):
+ return dict(
+ uuid=row.uuid,
+ name=row.name,
+ description=row.description,
+ project_repository_uuid=row.project_repository_uuid,
+ profile=row.profile,
+ profile_repository_uuid=row.profile_repository_uuid,
+ keyword_id=row.keyword_id,
+ unstable=row.unstable,
+ auto=row.auto,
+ enabled=row.enabled,
+ created_by=row.created_by
+ )
diff --git a/buildbot_gentoo_ci/db/repositorys.py b/buildbot_gentoo_ci/db/repositorys.py
new file mode 100644
index 0000000..d93ee72
--- /dev/null
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -0,0 +1,73 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class RepositorysConnectorComponent(base.DBConnectorComponent):
+
+ def getRepositorys(self):
+ def thd(conn):
+ tbl = self.db.model.repositorys
+ q = tbl.select()
+ q = q.where(tbl.c.enable == 1,
+ tbl.c.auto == 1
+ )
+ r = conn.execute(q)
+ return [dict(row) for row in r.fetchall()]
+ return self.db.pool.do(thd)
+
+ @defer.inlineCallbacks
+ def getRepositoryByName(self, name):
+ def thd(conn):
+ tbl = self.db.model.repositorys
+ q = tbl.select()
+ q = q.where(tbl.c.name == name)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getRepositoryByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.repositorys
+ q = tbl.select()
+ q = q.where(tbl.c.uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def getRepositorysGitPoller(self):
+ def thd(conn):
+ tblr = self.db.model.repositorys
+ tblrg = self.db.model.repository_gitpuller
+ from_clause = tblr.join(tblg, tblr.c.uuid == tblrg.c.repository_uuid)
+ q = sa.select([tblrg]).select_from(
+ from_clause).where(tblr.c.enabled == 1,
+ tblr.c.auto == 1
+ )
+ r = conn.execute(q)
+ return [dict(row) for row in r.fetchall()]
+ return self.db.pool.do(thd)
+
+ def _row2dict(self, conn, row):
+ return dict(
+ uuid=row.uuid,
+ name=row.name,
+ description=row.description,
+ mirror_url=row.mirror_url,
+ auto=row.auto,
+ enabled=row.enabled,
+ ebuild=row.ebuild
+ )
diff --git a/buildbot_gentoo_ci/steps/__init__.py b/buildbot_gentoo_ci/steps/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
new file mode 100644
index 0000000..df28f35
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -0,0 +1,162 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+import os
+
+from portage import config as portage_config
+from portage.versions import catpkgsplit
+from portage.util import getconfig
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.plugins import steps
+
+#from buildbot_gentoo_ci.steps.updatedb_functions import category
+
+class GetDataGentooCiProject(BuildStep):
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+ if self.project_data is None:
+ log.err('No data for project in the database')
+ return FAILURE
+ self.project_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.project_data['project_repository_uuid'])
+ if self.project_repository_data is None:
+ log.err('No data for repository in the database')
+ return FAILURE
+ self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.project_data['profile_repository_uuid'])
+ if self.profile_repository_data is None:
+ log.err('No data for repository in the database')
+ return FAILURE
+ print(self.project_data)
+ print(self.project_repository_data)
+ print(self.profile_repository_data)
+ print(self.getProperty("cpv_changes"))
+ print(self.getProperty("repository"))
+ self.setProperty("project_data", self.project_data, 'project_data')
+ self.setProperty("project_repository_data", self.project_repository_data, 'project_repository_data')
+ self.setProperty("profile_repository_data", self.profile_repository_data, 'profile_repository_data')
+ self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
+ self.setProperty("repository", self.getProperty("repository"), 'repository')
+ return SUCCESS
+
+class CheckPathGentooCiProject(BuildStep):
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.repository_basedir = self.gentooci.config.project['repository_basedir']
+ self.profile_repository_data = self.getProperty("profile_repository_data")
+ self.project_repository_data = self.getProperty("project_repository_data")
+ #self.repository = self.getProperty("repository")
+ self.repository = 'gentoo'
+ self.project_data = self.getProperty("project_data")
+ self.project_path = yield os.path.join(self.repository_basedir, self.project_repository_data['name'] + '.git')
+ self.repository_path = yield os.path.join(self.repository_basedir, self.repository + '.git')
+ self.portage_path = yield os.path.join(self.project_path, self.project_data['name'], 'etc/portage')
+ success = True
+ for x in [
+ os.path.join(self.repository_basedir, self.profile_repository_data['name'] + '.git'),
+ self.project_path,
+ self.portage_path,
+ os.path.join(self.portage_path, 'make.profile')
+ self.repository_path
+ # check the path of make.profile is project_data['profile']
+ ]:
+ is_dir = True
+ if not os.path.isdir(x):
+ is_dir = False
+ success = False
+ print("isdir(%s): %s" %(x, is_dir))
+ if not success:
+ return FAILURE
+ self.setProperty("project_data", self.project_data, 'project_data')
+ self.setProperty("project_repository_data", self.project_repository_data, 'project_repository_data')
+ self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
+ self.setProperty("repository", self.repository, 'repository')
+ return SUCCESS
+
+class CheckProjectGentooCiProject(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.repository_basedir = self.gentooci.config.project['repository_basedir']
+ self.project_repository_data = self.getProperty("project_repository_data")
+ self.project_data = self.getProperty("project_data")
+ self.project_path = yield os.path.join(self.repository_basedir, self.project_repository_data['name'] + '.git')
+ self.config_root = yield os.path.join(self.project_path, self.project_data['name'])
+ self.make_conf_file = yield os.path.join(self.config_root, 'etc/portage', '') + 'make.conf'
+ try:
+ getconfig(self.make_conf_file, tolerant=0, allow_sourcing=True, expand=True)
+ mysettings = portage_config(config_root = self.config_root)
+ mysettings.validate()
+ except ParseError as e:
+ print("project portage conf has error %s" %(str(e)))
+ return FAILURE
+ self.setProperty("config_root", self.config_root, 'config_root')
+ self.setProperty("project_data", self.project_data, 'project_data')
+ self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
+ self.setProperty("repository", self.getProperty("repository"), 'repository')
+ return SUCCESS
+
+class CheckCPVGentooCiProject(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.config_root = self.getProperty("config_root")
+ #self.cpv_changes = self.getProperty("cpv_changes")
+ self.cpv_changes = []
+ self.project_data = self.getProperty("project_data")
+ self.cpv_changes.append('dev-lang/python-3.8')
+ self.cpv_changes.append('dev-python/prust-3.9')
+ print(self.cpv_changes)
+ self.repository = self.getProperty("repository")
+ print(self.repository)
+ # check if cpv_change is a string or a list
+ if isinstance(self.cpv_changes, list):
+ self.cpv_list = self.cpv_changes
+ else:
+ self.cpv_list = []
+ self.cpv_list.append(self.cpv_changes)
+ self.success = True
+ addStepUpdateCPVData = []
+ for cpv in sorted(self.cpv_list):
+ # check that cpv is valied
+ if catpkgsplit(cpv) is None:
+ log.msg("%s is not vaild package name" % cpv)
+ self.success = False
+ else:
+ # call update_cpv_data
+ addStepUpdateCPVData.append(
+ steps.Trigger(
+ schedulerNames=['update_cpv_data'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : cpv,
+ 'config_root' : self.config_root,
+ 'project_data' : self.project_data,
+ 'repository' : self.repository
+ }
+ )
+ )
+ yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
+ if self.success is False:
+ return FAILURE
+ return SUCCESS
diff --git a/gentooci.cfg b/gentooci.cfg
index ff12a75..4043db4 100644
--- a/gentooci.cfg
+++ b/gentooci.cfg
@@ -19,4 +19,4 @@ c['db_url'] = "mysql://buildbot:xxxx@192.168.1.x/gentooci?max_idle=300"
c['project'] = "gosbsbase"
# This specifies what the repository base dir is
-c['repository_basedir'] = "repository"
+c['repository_basedir'] = "/srv/repository"
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2021-01-10 15:34 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-01-10 15:34 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, /, buildbot_gentoo_ci/config/ Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox