* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/
@ 2021-01-10 21:41 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2021-01-10 21:41 UTC (permalink / raw
To: gentoo-commits
commit: 12c4aa125889ef4518a7ecc8232a10d55b129301
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 10 21:40:48 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 21:40:48 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=12c4aa12
Add CheckCGentooCiProject on BuildFactory update_db_cp
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 4 +-
buildbot_gentoo_ci/db/categorys.py | 76 ++++++++++++++++++++++++++++++
buildbot_gentoo_ci/db/connector.py | 2 +
buildbot_gentoo_ci/db/model.py | 10 ++--
buildbot_gentoo_ci/steps/category.py | 58 +++++++++++++++++++++++
5 files changed, 143 insertions(+), 7 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 3fae376..4a0ffcd 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -5,7 +5,7 @@ from buildbot.plugins import steps as buildbot_steps
from buildbot.plugins import util
from buildbot_gentoo_ci.steps import update_db
-#from buildbot_gentoo_ci.steps import category
+from buildbot_gentoo_ci.steps import category
#from buildbot_gentoo_ci.steps import package
def update_db_check():
@@ -37,7 +37,7 @@ def update_db_cp():
# else
# add category to db step
# return category_data
- #f.addStep(category.CheckCGentooCiProject())
+ f.addStep(category.CheckCGentooCiProject())
# if package in db
# return package_data
# add check package path step at end
diff --git a/buildbot_gentoo_ci/db/categorys.py b/buildbot_gentoo_ci/db/categorys.py
new file mode 100644
index 0000000..f2bfde4
--- /dev/null
+++ b/buildbot_gentoo_ci/db/categorys.py
@@ -0,0 +1,76 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class CategorysConnectorComponent(base.DBConnectorComponent):
+
+ @defer.inlineCallbacks
+ def getCategoryByName(self, name):
+ def thd(conn):
+ tbl = self.db.model.categorys
+ q = tbl.select()
+ q = q.where(tbl.c.name == name)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getCategoryByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.categorys
+ q = tbl.select()
+ q = q.where(tbl.c.uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def addCategory(self, name):
+ def thd(conn, no_recurse=False):
+ try:
+ tbl = self.db.model.categorys
+ q = tbl.insert()
+ r = conn.execute(q, dict(name=name))
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ uuid = None
+ else:
+ uuid = r.inserted_primary_key[0]
+ return uuid
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def _row2dict(self, conn, row):
+ return dict(
+ uuid=row.uuid,
+ name=row.name
+ )
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index f95a121..c10fc49 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -32,6 +32,7 @@ from buildbot.util import service
from buildbot_gentoo_ci.db import model
from buildbot_gentoo_ci.db import projects
from buildbot_gentoo_ci.db import repositorys
+from buildbot_gentoo_ci.db import categorys
upgrade_message = textwrap.dedent("""\
@@ -73,6 +74,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.model = model.Model(self)
self.projects = projects.ProjectsConnectorComponent(self)
self.repositorys = repositorys.RepositorysConnectorComponent(self)
+ self.categorys = categorys.CategorysConnectorComponent(self)
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 1ab5990..e01462f 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -138,7 +138,7 @@ class Model(base.DBConnectorComponent):
)
categorys = sautils.Table(
- "categories", metadata,
+ "categorys", metadata,
sa.Column('uuid', sa.String(36), primary_key=True,
default=lambda: str(uuid.uuid4())
),
@@ -152,7 +152,7 @@ class Model(base.DBConnectorComponent):
),
sa.Column('name', sa.String(255), nullable=False),
sa.Column('category_uuid', sa.String(36),
- sa.ForeignKey('categories.uuid', ondelete='CASCADE'),
+ sa.ForeignKey('categorys.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('repository_uuid', sa.String(36),
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
@@ -175,14 +175,14 @@ class Model(base.DBConnectorComponent):
sa.Column('deleted_at', sa.Integer, nullable=True),
)
- ebuildkeywords = sautils.Table(
- "ebuildkeywords", metadata,
+ ebuilds_keywords = sautils.Table(
+ "ebuilds_keywords", metadata,
# unique id per project
sa.Column('id', sa.Integer, primary_key=True),
# project's name
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE')),
- sa.Column('ebuild_uuid', sa.String(36),
+ sa.Column('ebuilds_uuid', sa.String(36),
sa.ForeignKey('ebuilds.uuid', ondelete='CASCADE')),
sa.Column('status', sa.String(255), nullable=False),
)
diff --git a/buildbot_gentoo_ci/steps/category.py b/buildbot_gentoo_ci/steps/category.py
new file mode 100644
index 0000000..e641a97
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/category.py
@@ -0,0 +1,58 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+import os
+
+from portage.xml.metadata import MetaDataXML
+from portage.checksum import perform_checksum
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+
+class AddCategory(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.category = self.getProperty("category")
+ self.category_data = {}
+ self.category_data['name'] = self.category
+ self.category_data['uuid'] = yield self.gentooci.db.categorys.addCategory(self.category)
+ print(self.category_data)
+ self.setProperty("category_data", self.category_data, 'category_data')
+ self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
+ self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
+ self.setProperty("repository", self.getProperty("repository"), 'repository')
+ self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
+ return SUCCESS
+
+class CheckCGentooCiProject(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.category = yield self.getProperty("cpv").split('/')[0]
+ print(self.category)
+ self.category_data = yield self.gentooci.db.categorys.getCategoryByName(self.category)
+ self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
+ self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
+ self.setProperty("repository", self.getProperty("repository"), 'repository')
+ self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
+ print(self.category_data)
+ if self.category_data is None:
+ self.setProperty("category", self.category, 'category')
+ yield self.build.addStepsAfterCurrentStep([AddCategory()])
+ #yield self.build.addStepsAfterLastStep([AddMetadataCategory()])
+ return SUCCESS
+ self.setProperty("category_data", self.category_data, 'category_data')
+ #yield self.build.addStepsAfterLastStep([CheckPathCategory()])
+ return SUCCESS
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/
@ 2021-01-11 17:45 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2021-01-11 17:45 UTC (permalink / raw
To: gentoo-commits
commit: a4a5adaa76b48ae04c142f0c1bf602c210892731
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 11 17:45:46 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 11 17:45:46 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a4a5adaa
Add CheckPGentooCiProject on BuildFactory update_db_cp
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 5 +-
buildbot_gentoo_ci/db/connector.py | 2 +
buildbot_gentoo_ci/db/packages.py | 80 ++++++++++++++++++++++++++++++
buildbot_gentoo_ci/steps/category.py | 10 ++--
buildbot_gentoo_ci/steps/package.py | 65 ++++++++++++++++++++++++
buildbot_gentoo_ci/steps/update_db.py | 30 +++++------
6 files changed, 170 insertions(+), 22 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 4a0ffcd..4038bb1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -6,7 +6,7 @@ from buildbot.plugins import util
from buildbot_gentoo_ci.steps import update_db
from buildbot_gentoo_ci.steps import category
-#from buildbot_gentoo_ci.steps import package
+from buildbot_gentoo_ci.steps import package
def update_db_check():
f = util.BuildFactory()
@@ -44,7 +44,8 @@ def update_db_cp():
# else
# add package to db step
# return package_data
- #f.addStep(package.CheckPGentooCiProject())
+ f.addStep(package.CheckPGentooCiProject())
# Make a for loop and trigger new builders with v from cpv
# return package_data, cpv, repository, project_data, config_root
+ #f.addStep(package.TriggerVGentooCiProject())
return f
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index c10fc49..0121d11 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -33,6 +33,7 @@ from buildbot_gentoo_ci.db import model
from buildbot_gentoo_ci.db import projects
from buildbot_gentoo_ci.db import repositorys
from buildbot_gentoo_ci.db import categorys
+from buildbot_gentoo_ci.db import packages
upgrade_message = textwrap.dedent("""\
@@ -75,6 +76,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.projects = projects.ProjectsConnectorComponent(self)
self.repositorys = repositorys.RepositorysConnectorComponent(self)
self.categorys = categorys.CategorysConnectorComponent(self)
+ self.packages = packages.PackagesConnectorComponent(self)
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py
new file mode 100644
index 0000000..13cc267
--- /dev/null
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -0,0 +1,80 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class PackagesConnectorComponent(base.DBConnectorComponent):
+
+ @defer.inlineCallbacks
+ def getPackageByName(self, name):
+ def thd(conn):
+ tbl = self.db.model.packages
+ q = tbl.select()
+ q = q.where(tbl.c.name == name)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getPackageByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.packages
+ q = tbl.select()
+ q = q.where(tbl.c.uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def addPackage(self, name, repository_uuid, category_uuid):
+ def thd(conn, no_recurse=False):
+ try:
+ tbl = self.db.model.packages
+ q = tbl.insert()
+ r = conn.execute(q, dict(name=name,
+ repository_uuid=repository_uuid,
+ category_uuid=category_uuid))
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ uuid = None
+ else:
+ uuid = r.inserted_primary_key[0]
+ return uuid
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def _row2dict(self, conn, row):
+ return dict(
+ uuid=row.uuid,
+ name=row.name,
+ repository_uuid=row.repository_uuid,
+ category_uuid=row.category_uuid
+ )
diff --git a/buildbot_gentoo_ci/steps/category.py b/buildbot_gentoo_ci/steps/category.py
index e641a97..1aca99a 100644
--- a/buildbot_gentoo_ci/steps/category.py
+++ b/buildbot_gentoo_ci/steps/category.py
@@ -21,15 +21,14 @@ class AddCategory(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- self.category = self.getProperty("category")
self.category_data = {}
- self.category_data['name'] = self.category
- self.category_data['uuid'] = yield self.gentooci.db.categorys.addCategory(self.category)
+ self.category_data['name'] = self.getProperty("category")
+ self.category_data['uuid'] = yield self.gentooci.db.categorys.addCategory(self.category_data['name'])
print(self.category_data)
self.setProperty("category_data", self.category_data, 'category_data')
self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
- self.setProperty("repository", self.getProperty("repository"), 'repository')
+ self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
return SUCCESS
@@ -42,10 +41,11 @@ class CheckCGentooCiProject(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.category = yield self.getProperty("cpv").split('/')[0]
print(self.category)
+ print(self.getProperty("repository"))
self.category_data = yield self.gentooci.db.categorys.getCategoryByName(self.category)
self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
- self.setProperty("repository", self.getProperty("repository"), 'repository')
+ self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
print(self.category_data)
if self.category_data is None:
diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
new file mode 100644
index 0000000..916bb73
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -0,0 +1,65 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+import os
+
+from portage.xml.metadata import MetaDataXML
+from portage.checksum import perform_checksum
+from portage.versions import catpkgsplit
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+
+class AddPackage(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.package_data = {}
+ self.package_data['name'] = self.getProperty("package")
+ self.package_data['repository_uuid'] = self.getProperty("repository_data")['uuid']
+ self.package_data['category_uuid'] = self.getProperty("category_data")['uuid']
+ self.package_data['uuid'] = yield self.gentooci.db.packages.addPackage(
+ self.package_data['name'],
+ self.package_data['repository_uuid'],
+ self.package_data['category_uuid']
+ )
+ print(self.package_data)
+ self.setProperty("package_data", self.package_data, 'package_data')
+ self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
+ self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
+ self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
+ self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
+ return SUCCESS
+
+class CheckPGentooCiProject(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.package = yield catpkgsplit(self.getProperty("cpv"))[1]
+ print(self.package)
+ self.package_data = yield self.gentooci.db.packages.getPackageByName(self.package)
+ self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
+ self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
+ self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
+ self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
+ print(self.package_data)
+ if self.package_data is None:
+ self.setProperty("category_data", self.getProperty("category_data"), 'category_data')
+ self.setProperty("package", self.package, 'package')
+ yield self.build.addStepsAfterCurrentStep([AddPackage()])
+ #yield self.build.addStepsAfterLastStep([AddMetadataPackage()])
+ return SUCCESS
+ self.setProperty("package_data", self.package_data, 'package_data')
+ #yield self.build.addStepsAfterLastStep([CheckPathPackage()])
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index df28f35..d00407c 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -1,5 +1,6 @@
# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+
import os
from portage import config as portage_config
@@ -36,16 +37,19 @@ class GetDataGentooCiProject(BuildStep):
if self.profile_repository_data is None:
log.err('No data for repository in the database')
return FAILURE
+ #self.repository = self.getProperty("repository")
+ self.repository = 'gentoo'
+ self.repository_data = yield self.gentooci.db.repositorys.getRepositoryByName(self.repository)
print(self.project_data)
print(self.project_repository_data)
print(self.profile_repository_data)
print(self.getProperty("cpv_changes"))
- print(self.getProperty("repository"))
+ print(self.repository_data)
self.setProperty("project_data", self.project_data, 'project_data')
self.setProperty("project_repository_data", self.project_repository_data, 'project_repository_data')
self.setProperty("profile_repository_data", self.profile_repository_data, 'profile_repository_data')
self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
- self.setProperty("repository", self.getProperty("repository"), 'repository')
+ self.setProperty("repository_data", self.repository_data, 'repository_data')
return SUCCESS
class CheckPathGentooCiProject(BuildStep):
@@ -59,18 +63,17 @@ class CheckPathGentooCiProject(BuildStep):
self.repository_basedir = self.gentooci.config.project['repository_basedir']
self.profile_repository_data = self.getProperty("profile_repository_data")
self.project_repository_data = self.getProperty("project_repository_data")
- #self.repository = self.getProperty("repository")
- self.repository = 'gentoo'
+ self.repository_data = self.getProperty("repository_data")
self.project_data = self.getProperty("project_data")
self.project_path = yield os.path.join(self.repository_basedir, self.project_repository_data['name'] + '.git')
- self.repository_path = yield os.path.join(self.repository_basedir, self.repository + '.git')
+ self.repository_path = yield os.path.join(self.repository_basedir, self.repository_data['name'] + '.git')
self.portage_path = yield os.path.join(self.project_path, self.project_data['name'], 'etc/portage')
success = True
for x in [
os.path.join(self.repository_basedir, self.profile_repository_data['name'] + '.git'),
self.project_path,
self.portage_path,
- os.path.join(self.portage_path, 'make.profile')
+ os.path.join(self.portage_path, 'make.profile'),
self.repository_path
# check the path of make.profile is project_data['profile']
]:
@@ -84,7 +87,7 @@ class CheckPathGentooCiProject(BuildStep):
self.setProperty("project_data", self.project_data, 'project_data')
self.setProperty("project_repository_data", self.project_repository_data, 'project_repository_data')
self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
- self.setProperty("repository", self.repository, 'repository')
+ self.setProperty("repository_data", self.repository_data, 'repository_data')
return SUCCESS
class CheckProjectGentooCiProject(BuildStep):
@@ -110,7 +113,7 @@ class CheckProjectGentooCiProject(BuildStep):
self.setProperty("config_root", self.config_root, 'config_root')
self.setProperty("project_data", self.project_data, 'project_data')
self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes')
- self.setProperty("repository", self.getProperty("repository"), 'repository')
+ self.setProperty("repository_data", self.getProperty("repository_data"), 'repository')
return SUCCESS
class CheckCPVGentooCiProject(BuildStep):
@@ -119,15 +122,12 @@ class CheckCPVGentooCiProject(BuildStep):
@defer.inlineCallbacks
def run(self):
- self.config_root = self.getProperty("config_root")
#self.cpv_changes = self.getProperty("cpv_changes")
self.cpv_changes = []
- self.project_data = self.getProperty("project_data")
self.cpv_changes.append('dev-lang/python-3.8')
self.cpv_changes.append('dev-python/prust-3.9')
print(self.cpv_changes)
- self.repository = self.getProperty("repository")
- print(self.repository)
+ print(self.getProperty("repository_data"))
# check if cpv_change is a string or a list
if isinstance(self.cpv_changes, list):
self.cpv_list = self.cpv_changes
@@ -150,9 +150,9 @@ class CheckCPVGentooCiProject(BuildStep):
updateSourceStamp=False,
set_properties={
'cpv' : cpv,
- 'config_root' : self.config_root,
- 'project_data' : self.project_data,
- 'repository' : self.repository
+ 'config_root' : self.getProperty("config_root"),
+ 'project_data' : self.getProperty("project_data"),
+ 'repository_data' : self.getProperty("repository_data"),
}
)
)
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/
@ 2021-01-17 9:25 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2021-01-17 9:25 UTC (permalink / raw
To: gentoo-commits
commit: 81da08ad4476408915e0e082a33f5a1c2ebe159e
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 17 09:25:07 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 17 09:25:07 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=81da08ad
Add BuildFactory update_db_v
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 16 ++
buildbot_gentoo_ci/config/buildfactorys.py | 36 +++-
buildbot_gentoo_ci/config/schedulers.py | 6 +
buildbot_gentoo_ci/db/connector.py | 4 +
buildbot_gentoo_ci/db/keywords.py | 76 +++++++++
buildbot_gentoo_ci/db/model.py | 28 +--
buildbot_gentoo_ci/db/versions.py | 112 ++++++++++++
buildbot_gentoo_ci/steps/package.py | 34 +++-
buildbot_gentoo_ci/steps/update_db.py | 6 +-
buildbot_gentoo_ci/steps/version.py | 264 +++++++++++++++++++++++++++++
10 files changed, 562 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index c069841..9fef46a 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -22,4 +22,20 @@ def gentoo_builders(b=[]):
factory=buildfactorys.update_db_cp()
)
)
+ # FIXME: get workers from db
+ # Use multiplay workers
+ b.append(util.BuilderConfig(
+ name='update_v_data',
+ workername='updatedb_1',
+ factory=buildfactorys.update_db_v()
+ )
+ )
+ # FIXME: get workers from db
+ # Use multiplay workers
+ b.append(util.BuilderConfig(
+ name='build_request_data',
+ workername='updatedb_1',
+ factory=buildfactorys.build_request_check()
+ )
+ )
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 4038bb1..ba96e5a 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -7,6 +7,7 @@ from buildbot.plugins import util
from buildbot_gentoo_ci.steps import update_db
from buildbot_gentoo_ci.steps import category
from buildbot_gentoo_ci.steps import package
+from buildbot_gentoo_ci.steps import version
def update_db_check():
f = util.BuildFactory()
@@ -45,7 +46,36 @@ def update_db_cp():
# add package to db step
# return package_data
f.addStep(package.CheckPGentooCiProject())
- # Make a for loop and trigger new builders with v from cpv
- # return package_data, cpv, repository, project_data, config_root
- #f.addStep(package.TriggerVGentooCiProject())
+ # Trigger new builders with v from cpv
+ # return package_data, cpv, repository_data, project_data, config_root
+ f.addStep(package.TriggerCheckVGentooCiProject())
+ return f
+
+def update_db_v():
+ f = util.BuildFactory()
+ # FIXME: 3
+ # if version in db
+ # return version_data
+ f.addStep(version.GetVData())
+ # check path and hash
+ f.addStep(version.CheckPathHash())
+ # if not path
+ # if not hash
+ # add deleted stage att end
+ # add version to db stage
+ # add version metadata to db
+ # add version to build check
+ # else
+ # add deleted stage att end
+ # add version to build check stage att end
+ # else
+ # add version to db
+ # add version metadata to db
+ # add version to build check
+ f.addStep(version.CheckV())
+ return f
+
+def build_request_check():
+ f = util.BuildFactory()
+ # FIXME: 4
return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index 88d0e49..687b52f 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -52,8 +52,14 @@ def gentoo_schedulers():
])
update_cpv_data = schedulers.Triggerable(name="update_cpv_data",
builderNames=["update_cpv_data"])
+ update_v_data = schedulers.Triggerable(name="update_v_data",
+ builderNames=["update_v_data"])
+ build_request_data = schedulers.Triggerable(name="build_request_data",
+ builderNames=["build_request_data"])
s = []
s.append(test_updatedb)
#s.append(scheduler_update_db)
s.append(update_cpv_data)
+ s.append(update_v_data)
+ s.append(build_request_data)
return s
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index 0121d11..35febe6 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -34,6 +34,8 @@ from buildbot_gentoo_ci.db import projects
from buildbot_gentoo_ci.db import repositorys
from buildbot_gentoo_ci.db import categorys
from buildbot_gentoo_ci.db import packages
+from buildbot_gentoo_ci.db import versions
+from buildbot_gentoo_ci.db import keywords
upgrade_message = textwrap.dedent("""\
@@ -77,6 +79,8 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.repositorys = repositorys.RepositorysConnectorComponent(self)
self.categorys = categorys.CategorysConnectorComponent(self)
self.packages = packages.PackagesConnectorComponent(self)
+ self.versions = versions.VersionsConnectorComponent(self)
+ self.keywords = keywords.KeywordsConnectorComponent(self)
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/keywords.py b/buildbot_gentoo_ci/db/keywords.py
new file mode 100644
index 0000000..165dbe4
--- /dev/null
+++ b/buildbot_gentoo_ci/db/keywords.py
@@ -0,0 +1,76 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class KeywordsConnectorComponent(base.DBConnectorComponent):
+
+ @defer.inlineCallbacks
+ def getKeywordByName(self, name):
+ def thd(conn):
+ tbl = self.db.model.keywords
+ q = tbl.select()
+ q = q.where(tbl.c.name == name)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getKeywordById(self, id):
+ def thd(conn):
+ tbl = self.db.model.keywords
+ q = tbl.select()
+ q = q.where(tbl.c.id == id)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def addKeyword(self, name):
+ def thd(conn, no_recurse=False):
+ try:
+ tbl = self.db.model.keywords
+ q = tbl.insert()
+ r = conn.execute(q, dict(name=name))
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ uuid = None
+ else:
+ uuid = r.inserted_primary_key[0]
+ return uuid
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def _row2dict(self, conn, row):
+ return dict(
+ id=row.id,
+ name=row.name
+ )
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index e01462f..9ad1331 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -131,10 +131,10 @@ class Model(base.DBConnectorComponent):
keywords = sautils.Table(
"keywords", metadata,
- # unique id per project
+ # unique uuid per keyword
sa.Column('id', sa.Integer, primary_key=True),
# project's name
- sa.Column('keyword', sa.String(255), nullable=False),
+ sa.Column('name', sa.String(255), nullable=False),
)
categorys = sautils.Table(
@@ -158,11 +158,11 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('deleted', sa.Boolean, default=False),
- sa.Column('deleted_at', sa.Integer, nullable=True),
+ sa.Column('deleted_at', sa.DateTime, nullable=True),
)
- ebuilds = sautils.Table(
- "ebuilds", metadata,
+ versions = sautils.Table(
+ "versions", metadata,
sa.Column('uuid', sa.String(36), primary_key=True,
default=lambda: str(uuid.uuid4()),
),
@@ -170,21 +170,23 @@ class Model(base.DBConnectorComponent):
sa.Column('package_uuid', sa.String(36),
sa.ForeignKey('packages.uuid', ondelete='CASCADE'),
nullable=False),
- sa.Column('ebuild_hash', sa.String(255), nullable=False),
+ sa.Column('file_hash', sa.String(255), nullable=False),
sa.Column('deleted', sa.Boolean, default=False),
- sa.Column('deleted_at', sa.Integer, nullable=True),
+ sa.Column('deleted_at', sa.DateTime, nullable=True),
)
- ebuilds_keywords = sautils.Table(
- "ebuilds_keywords", metadata,
+ versions_keywords = sautils.Table(
+ "versions_keywords", metadata,
# unique id per project
- sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('uuid', sa.String(36), primary_key=True,
+ default=lambda: str(uuid.uuid4()),
+ ),
# project's name
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE')),
- sa.Column('ebuilds_uuid', sa.String(36),
- sa.ForeignKey('ebuilds.uuid', ondelete='CASCADE')),
- sa.Column('status', sa.String(255), nullable=False),
+ sa.Column('versions_uuid', sa.String(36),
+ sa.ForeignKey('versions.uuid', ondelete='CASCADE')),
+ sa.Column('status', sa.Enum('stable','unstable','negative','all'), nullable=False),
)
# Tables related to users
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
new file mode 100644
index 0000000..25f2061
--- /dev/null
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -0,0 +1,112 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+from buildbot.util import epoch2datetime
+
+class VersionsConnectorComponent(base.DBConnectorComponent):
+
+ @defer.inlineCallbacks
+ def getVersionByName(self, name):
+ def thd(conn):
+ tbl = self.db.model.versions
+ q = tbl.select()
+ q = q.where(tbl.c.name == name)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getVersionByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.versions
+ q = tbl.select()
+ q = q.where(tbl.c.uuid == uuid)
+ res = conn.execute(q)
+ row = res.fetchone()
+ if not row:
+ return None
+ return self._row2dict(conn, row)
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def addVersion(self, name, package_uuid, file_hash):
+ def thd(conn, no_recurse=False):
+ try:
+ tbl = self.db.model.versions
+ q = tbl.insert()
+ r = conn.execute(q, dict(name=name,
+ package_uuid=package_uuid,
+ file_hash=file_hash))
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ uuid = None
+ else:
+ uuid = r.inserted_primary_key[0]
+ return uuid
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def delVersion(self, uuid):
+ deleted_at = epoch2datetime(int(self.master.reactor.seconds()))
+ def thd(conn, no_recurse=False):
+
+ tbl = self.db.model.versions
+ q = tbl.update()
+ q = q.where(tbl.c.uuid == uuid)
+ conn.execute(q, deleted=True,
+ deleted_at=deleted_at)
+ yield self.db.pool.do(thd)
+
+ @defer.inlineCallbacks
+ def addKeyword(self, version_uuid, keyword_id, status):
+ def thd(conn, no_recurse=False):
+ try:
+ tbl = self.db.model.versions_keywords
+ q = tbl.insert()
+ r = conn.execute(q, dict(version_uuid=version_uuid,
+ keyword_id=keyword_id,
+ status=status))
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ uuid = None
+ else:
+ uuid = r.inserted_primary_key[0]
+ return uuid
+ res = yield self.db.pool.do(thd)
+ return res
+
+ def _row2dict(self, conn, row):
+ return dict(
+ uuid=row.uuid,
+ name=row.name,
+ package_uuid=row.package_uuid,
+ file_hash=row.file_hash,
+ deleted=row.deleted,
+ deleted_at=row.deleted_at
+ )
diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
index 916bb73..3a28da2 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -14,6 +14,7 @@ from twisted.python import log
from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
+from buildbot.plugins import steps
class AddPackage(BuildStep):
def __init__(self, **kwargs):
@@ -36,6 +37,7 @@ class AddPackage(BuildStep):
self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
+ self.setProperty("category_data", self.getProperty("category_data"), 'category_data')
self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
return SUCCESS
@@ -52,10 +54,10 @@ class CheckPGentooCiProject(BuildStep):
self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
+ self.setProperty("category_data", self.getProperty("category_data"), 'category_data')
self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
print(self.package_data)
if self.package_data is None:
- self.setProperty("category_data", self.getProperty("category_data"), 'category_data')
self.setProperty("package", self.package, 'package')
yield self.build.addStepsAfterCurrentStep([AddPackage()])
#yield self.build.addStepsAfterLastStep([AddMetadataPackage()])
@@ -63,3 +65,33 @@ class CheckPGentooCiProject(BuildStep):
self.setProperty("package_data", self.package_data, 'package_data')
#yield self.build.addStepsAfterLastStep([CheckPathPackage()])
return SUCCESS
+
+class TriggerCheckVGentooCiProject(BuildStep):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ addStepUpdateVData = []
+ addStepUpdateVData.append(
+ steps.Trigger(
+ schedulerNames=['update_v_data'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : self.getProperty("cpv"),
+ 'package_data' : self.getProperty("package_data"),
+ 'config_root' : self.getProperty("config_root"),
+ 'repository_data' : self.getProperty("repository_data"),
+ 'category_data' : self.getProperty("category_data"),
+ }
+ )
+ )
+ yield self.build.addStepsAfterCurrentStep(addStepUpdateVData)
+ self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
+ self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
+ self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
+ self.setProperty("category_data", self.getProperty("category_data"), 'category_data')
+ self.setProperty("package_data", self.getProperty("package_data"), 'package_data')
+ self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index d00407c..3817d44 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -101,7 +101,7 @@ class CheckProjectGentooCiProject(BuildStep):
self.project_repository_data = self.getProperty("project_repository_data")
self.project_data = self.getProperty("project_data")
self.project_path = yield os.path.join(self.repository_basedir, self.project_repository_data['name'] + '.git')
- self.config_root = yield os.path.join(self.project_path, self.project_data['name'])
+ self.config_root = yield os.path.join(self.project_path, self.project_data['name'], '')
self.make_conf_file = yield os.path.join(self.config_root, 'etc/portage', '') + 'make.conf'
try:
getconfig(self.make_conf_file, tolerant=0, allow_sourcing=True, expand=True)
@@ -124,8 +124,8 @@ class CheckCPVGentooCiProject(BuildStep):
def run(self):
#self.cpv_changes = self.getProperty("cpv_changes")
self.cpv_changes = []
- self.cpv_changes.append('dev-lang/python-3.8')
- self.cpv_changes.append('dev-python/prust-3.9')
+ self.cpv_changes.append('dev-lang/python-3.9.1')
+ self.cpv_changes.append('dev-python/scrypt-0.8.16')
print(self.cpv_changes)
print(self.getProperty("repository_data"))
# check if cpv_change is a string or a list
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
new file mode 100644
index 0000000..e274e65
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -0,0 +1,264 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+import os
+
+from portage.xml.metadata import MetaDataXML
+from portage.checksum import perform_checksum
+from portage.versions import cpv_getversion, pkgsplit
+from portage import auxdbkeys
+from portage import config as portage_config
+from portage import portdbapi
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.plugins import steps
+
+class GetVData(BuildStep):
+
+ name = 'GetVData'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.version = yield cpv_getversion(self.getProperty("cpv"))
+ print(self.version)
+ self.old_version_data = yield self.gentooci.db.versions.getVersionByName(self.version)
+ print(self.old_version_data)
+ self.setProperty("old_version_data", self.old_version_data, 'old_version_data')
+ self.setProperty("version", self.version, 'version')
+ return SUCCESS
+
+class AddVersion(BuildStep):
+
+ name = 'AddVersion'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.version_data = {}
+ self.version_data['name'] = self.getProperty("version")
+ self.version_data['package_uuid'] = self.getProperty("package_data")['uuid']
+ self.version_data['file_hash'] = self.getProperty("ebuild_file_hash")
+ self.version_data['uuid'] = yield self.gentooci.db.versions.addVersion(
+ self.version_data['name'],
+ self.version_data['package_uuid'],
+ self.version_data['file_hash']
+ )
+ print(self.version_data)
+ self.setProperty("version_data", self.version_data, 'version_data')
+ return SUCCESS
+
+class GetAuxMetadata(BuildStep):
+
+ name = 'GetAuxMetadata'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.mysettings = yield portage_config(config_root = self.getProperty("config_root"))
+ self.myportdb = yield portdbapi(mysettings=self.mysettings)
+ try:
+ auxdb_list = yield self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys, myrepo=self.getProperty("repository_data")['name'])
+ except:
+ print("Failed to get aux data for %s" % self.getProperty("cpv"))
+ yield self.myportdb.close_caches()
+ yield portdbapi.portdbapi_instances.remove(self.myportdb)
+ return FAILURE
+ self.setProperty('aux_metadata', auxdb_list, 'aux_metadata')
+ yield self.myportdb.close_caches()
+ yield portdbapi.portdbapi_instances.remove(self.myportdb)
+ return SUCCESS
+
+class AddVersionKeyword(BuildStep):
+
+ name = 'AddVersionKeyword'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def add_keyword_data(self, keyword):
+ keyword_data = {}
+ keyword_data['keyword'] = keyword
+ keyword_data['id'] = yield self.gentooci.db.keywords.addKeyword(keyword_data['keyword'])
+ return keyword_data
+
+ @defer.inlineCallbacks
+ def check_keyword_data(self, keyword):
+ keyword_data = yield self.gentooci.db.keywords.getKeywordByName(keyword)
+ if keyword_data is None:
+ keyword_data = yield self.add_keyword_data(keyword)
+ return keyword_data['id']
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.version_keyword_dict = {}
+ for keyword in self.getProperty("aux_metadata")[8].split():
+ status = 'stable'
+ if keyword[0] in ["~"]:
+ keyword = keyword[1:]
+ status = 'unstable'
+ elif keyword[0] in ["-"]:
+ keyword = keyword[1:]
+ status = 'negative'
+ elif keyword[0] in ["*"]:
+ keyword = keyword[1:]
+ status = 'all'
+ version_keyword_data = {}
+ version_keyword_data['version_uuid'] = self.getProperty("version_data")['uuid']
+ version_keyword_data['keyword_id'] = yield self.check_keyword_data(keyword)
+ version_keyword_data['status'] = status
+ version_keyword_data['uuid'] = yield self.gentooci.db.versions.addKeyword(
+ version_keyword_data['version_uuid'],
+ version_keyword_data['keyword_id'],
+ version_keyword_data['status'])
+ self.version_keyword_dict[keyword] = version_keyword_data
+ self.setProperty('version_keyword_dict', self.version_keyword_dict, 'version_keyword_dict')
+ return SUCCESS
+
+class CheckPathHash(BuildStep):
+
+ name = 'CheckPathHash'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.repository_basedir = self.gentooci.config.project['repository_basedir']
+ self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'] + '.git')
+ self.cp_path = yield pkgsplit(self.getProperty("cpv"))[0]
+ self.file_name = yield self.getProperty("package_data")['name'] + '-' + self.getProperty("version") + '.ebuild'
+ self.ebuild_file = yield os.path.join(self.repository_path, self.cp_path, self.file_name)
+ if os.path.isfile(self.ebuild_file):
+ self.ebuild_file_hash = yield perform_checksum(self.ebuild_file, "SHA256")[0]
+ else:
+ self.ebuild_file = None
+ self.ebuild_file_hash = None
+ self.setProperty('ebuild_file', self.ebuild_file, 'ebuild_file')
+ self.setProperty('ebuild_file_hash', self.ebuild_file_hash, 'ebuild_file_hash')
+ return SUCCESS
+
+class TriggerBuildCheck(BuildStep):
+
+ name = 'TriggerBuildCheck'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ yield steps.Trigger(
+ schedulerNames=['build_request_data'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : self.getProperty("cpv"),
+ 'version_data' : self.getProperty("version_data"),
+ 'version_keyword_dict' : self.getProperty('version_keyword_dict'),
+ 'repository_data' : self.getProperty("repository_data"),
+ }
+ )
+ return SUCCESS
+
+class DeleteOldVersion(BuildStep):
+
+ name = 'DeleteOldVersion'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ yield self.gentooci.db.versions.delVersion(self.getProperty("old_version_data")['uuid'])
+ return SUCCESS
+
+class CheckV(BuildStep):
+
+ name = 'CheckV'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.old_version_data = self.getProperty("old_version_data")
+ self.ebuild_file = self.getProperty("ebuild_file")
+ addStepVData = []
+ if self.getProperty("ebuild_file") is None and self.getProperty("old_version_data") is not None:
+ addStepVData.append(TriggerBuildCheck())
+ addStepVData.append(DeleteOldVersion())
+ if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None:
+ if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']:
+ addStepVData.append(AddVersion())
+ addStepVData.append(GetAuxMetadata())
+ addStepVData.append(AddVersionKeyword())
+ addStepVData.append(TriggerBuildCheck())
+ addStepVData.append(DeleteOldVersion())
+ else:
+ return SUCCESS
+ if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None:
+ addStepVData.append(AddVersion())
+ addStepVData.append(GetAuxMetadata())
+ addStepVData.append(AddVersionKeyword())
+ addStepVData.append(TriggerBuildCheck())
+ yield self.build.addStepsAfterCurrentStep(addStepVData)
+ return SUCCESS
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/
@ 2021-02-22 20:28 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2021-02-22 20:28 UTC (permalink / raw
To: gentoo-commits
commit: 4324b82edd9d69a27b1db9f4272564ddc51a75dd
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 22 20:27:14 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 20:27:14 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4324b82e
Add RunPkgCheck and CheckPkgCheckLogs
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 11 ++-
buildbot_gentoo_ci/db/model.py | 2 +-
buildbot_gentoo_ci/db/projects.py | 6 +-
buildbot_gentoo_ci/steps/builders.py | 120 +++++++++++++++++++++++++++--
4 files changed, 129 insertions(+), 10 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 3c77762..cfe4131 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -92,13 +92,15 @@ def run_build_request():
# set needed Propertys
f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
+ #FIXME: Is don't like symlinks
f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
workdir='/etc/'))
f.addStep(buildbot_steps.MakeDirectory(dir="portage",
workdir='/etc/'))
# setup the profile
- f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
- workdir='/etc/portage/'))
+ #NOTE: pkgcheck do not support it as a dir
+ #f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
+ # workdir='/etc/portage/'))
f.addStep(builders.SetMakeProfile())
# setup repos.conf dir
f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
@@ -132,4 +134,9 @@ def run_build_request():
# check log
# setup make.conf if build id has changes make.conf as dict from SetMakeConf
# setup package.* env if build id has changes
+ # setup pkgcheck.conf if needed
+ #f.addStep(builders.SetPkgCheckConf())
+ # run pkgcheck if wanted
+ # check log
+ f.addStep(builders.RunPkgCheck())
return f
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0a2c145..4d691ea 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -128,7 +128,7 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('auto', sa.Boolean, default=False),
- sa.Column('pkgcheck', sa.Boolean, default=False),
+ sa.Column('pkgcheck', sa.Enum('package','full','none'), default='none'),
)
# projects etc/portage settings
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index eb8f727..9aeca50 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -160,12 +160,16 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
)
def _row2dict_projects_repositorys(self, conn, row):
+ if row.pkgcheck == 'none':
+ pkgcheck = False
+ else:
+ pkgcheck=row.pkgcheck
return dict(
id=row.id,
project_uuid=row.project_uuid,
repository_uuid=row.repository_uuid,
auto=row.auto,
- pkgcheck=row.pkgcheck
+ pkgcheck=pkgcheck
)
def _row2dict_projects_portage(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index c0567c1..6d46ae6 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,6 +4,8 @@
import os
import re
+from portage.versions import catpkgsplit
+
from twisted.internet import defer
from twisted.python import log
@@ -75,6 +77,26 @@ def PersOutputOfEmerge(rc, stdout, stderr):
'emerge_output' : emerge_output
}
+def PersOutputOfPkgCheck(rc, stdout, stderr):
+ pkgcheck_output = {}
+ pkgcheck_output['rc'] = rc
+ #FIXME: Handling of stdout output
+ pkgcheck_xml_list = []
+ # split the lines
+ for line in stdout.split('\n'):
+ # pkgcheck output list
+ if line.startswith('<checks'):
+ pkgcheck_xml_list.append(line)
+ if line.startswith('<result'):
+ pkgcheck_xml_list.append(line)
+ if line.startswith('</checks'):
+ pkgcheck_xml_list.append(line)
+ pkgcheck_output['pkgcheck_xml'] = pkgcheck_xml_list
+ #FIXME: Handling of stderr output
+ return {
+ 'pkgcheck_output' : pkgcheck_output
+ }
+
class TriggerRunBuildRequest(BuildStep):
name = 'TriggerRunBuildRequest'
@@ -190,16 +212,31 @@ class SetMakeProfile(BuildStep):
project_data = self.getProperty('project_data')
profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
makeprofiles_paths = []
+ #NOTE: pkgcheck don't support make.profile as a dir
+ # we only support one line in db
makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
for makeprofile in makeprofiles_data:
makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', makeprofile['value'], '')
- makeprofiles_paths.append('../../..' + makeprofile_path)
- separator = '\n'
- makeprofile_path_string = separator.join(makeprofiles_paths)
+ # makeprofiles_paths.append('../../..' + makeprofile_path)
+ #separator = '\n'
+ #makeprofile_path_string = separator.join(makeprofiles_paths)
+ # yield self.build.addStepsAfterCurrentStep([
+ # steps.StringDownload(makeprofile_path_string + separator,
+ # workerdest="make.profile/parent",
+ # workdir='/etc/portage/')
+ # ])
+ #NOTE: pkgcheck profile link
+ shell_commad_list = [
+ 'ln',
+ '-s'
+ ]
+ shell_commad_list.append(makeprofile_path)
+ shell_commad_list.append('/etc/portage/make.profile')
yield self.build.addStepsAfterCurrentStep([
- steps.StringDownload(makeprofile_path_string + separator,
- workerdest="make.profile/parent",
- workdir='/etc/portage/')
+ steps.ShellCommandNewStyle(
+ command=shell_commad_list,
+ workdir='/'
+ )
])
return SUCCESS
@@ -515,3 +552,74 @@ class CheckEmergeLogs(BuildStep):
if not self.step is None:
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+
+class RunPkgCheck(BuildStep):
+
+ name = 'RunPkgCheck'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ projectrepository_data = self.getProperty('projectrepository_data')
+ if not projectrepository_data['pkgcheck']:
+ return SUCCESS
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ portage_repos_path = self.getProperty('portage_repos_path')
+ repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
+ repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
+ cpv = self.getProperty("cpv")
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ shell_commad_list = [
+ 'pkgcheck',
+ 'scan',
+ '-v'
+ ]
+ shell_commad_list.append('-R')
+ shell_commad_list.append('XmlReporter')
+ aftersteps_list = []
+ if projectrepository_data['pkgcheck'] == 'full':
+ pkgcheck_workdir = yield os.path.join(repository_path, '')
+ else:
+ pkgcheck_workdir = yield os.path.join(repository_path, c, p, '')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfPkgCheck,
+ workdir=pkgcheck_workdir
+ ))
+ aftersteps_list.append(CheckPkgCheckLogs())
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
+
+class CheckPkgCheckLogs(BuildStep):
+
+ name = 'CheckPkgCheckLogs'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ #@defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ pkgcheck_output = self.getProperty('pkgcheck_output')
+ print(pkgcheck_output)
+ #FIXME:
+ # Perse the logs
+ # tripp irc request with pkgcheck info
+ return SUCCESS
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/
@ 2022-04-22 12:38 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2022-04-22 12:38 UTC (permalink / raw
To: gentoo-commits
commit: 2fac94e594b5f50b00780361788df91800a59ea7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 22 12:37:35 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Apr 22 12:37:35 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2fac94e5
Add support stage4 build
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 9 +
buildbot_gentoo_ci/config/schedulers.py | 28 ++-
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/steps/nodes.py | 322 +++++++++++++++++++++++++++++
4 files changed, 344 insertions(+), 16 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 86fa162..a714e47 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -13,6 +13,7 @@ from buildbot_gentoo_ci.steps import builders
from buildbot_gentoo_ci.steps import portage
from buildbot_gentoo_ci.steps import logs
from buildbot_gentoo_ci.steps import repos
+from buildbot_gentoo_ci.steps import nodes
def update_db_check():
f = util.BuildFactory()
@@ -216,3 +217,11 @@ def parse_build_log():
# setup things for the irc bot
#f.addStep(logs.SetIrcInfo())
return f
+
+def run_build_stage4_request():
+ f = util.BuildFactory()
+ # set needed Propertys
+ f.addStep(nodes.SetupPropertys())
+ # set the needed steps for making the stage4
+ f.addStep(nodes.SetupStage4Steps())
+ return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index e520076..8c08006 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -46,24 +46,18 @@ def gentoo_schedulers():
builderNames = builderUpdateDbNames,
change_filter=util.ChangeFilter(branch='master'),
)
- test_updatedb = schedulers.ForceScheduler(
- name="force",
- buttonName="pushMe!",
- label="My nice Force form",
- builderNames=['update_db_check'],
+ create_stage4 = schedulers.ForceScheduler(
+ name="create_stage4",
+ buttonName="Create stage4",
+ label="Create stage4 form",
+ builderNames=['run_build_stage4_request'],
# A completely customized property list. The name of the
# property is the name of the parameter
properties=[
- util.NestedParameter(name="options", label="Build Options",
- layout="vertical", fields=[
- util.StringParameter(name="cpv_changes",
- label="Package to check",
- default="dev-lang/python-3.8", size=80),
- util.StringParameter(name="repository",
- label="repo",
- default="gentoo", size=80),
- ])
- ])
+ util.StringParameter(name="project_uuid",
+ label="Project uuid",
+ default="e89c2c1a-46e0-4ded-81dd-c51afeb7fcfd", size=36),
+ ])
update_cpv_data = schedulers.Triggerable(name="update_cpv_data",
builderNames=["update_cpv_data"])
update_repo_check = schedulers.Triggerable(name="update_repo_check",
@@ -76,8 +70,10 @@ def gentoo_schedulers():
builderNames=["run_build_request"])
parse_build_log = schedulers.Triggerable(name="parse_build_log",
builderNames=["parse_build_log"])
+ run_build_stage4_request = schedulers.Triggerable(name="run_build_stage4_request",
+ builderNames=["run_build_stage4_request"])
s = []
- s.append(test_updatedb)
+ s.append(create_stage4)
s.append(scheduler_update_db)
s.append(update_repo_check)
s.append(update_cpv_data)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 18fb7ff..908ebb5 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -110,6 +110,7 @@ class Model(base.DBConnectorComponent):
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE'),
nullable=False),
+ sa.Column('image', sa.String(255), nullable=False),
sa.Column('status', sa.Enum('stable','unstable','all'), nullable=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py
new file mode 100644
index 0000000..37cf917
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -0,0 +1,322 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import re
+import json
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+
+from portage.versions import catpkgsplit, cpv_getversion
+from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
+from buildbot.plugins import steps, util
+
+from buildbot_gentoo_ci.steps import portage as portage_steps
+from buildbot_gentoo_ci.steps import builders as builders_steps
+
+class SetupPropertys(BuildStep):
+ name = 'Setup propertys for stage4 image'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ # we need project uuid and worker uuid
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_uuid'))
+ self.setProperty('project_data', project_data, 'project_data')
+ #FIXME: set it in db node config
+ self.workerbase = yield os.path.join('/', 'srv', 'gentoo', 'stage4')
+ self.setProperty('workerbase', self.workerbase, 'workerbase')
+ self.setProperty('stage3', 'image', 'stage3')
+ # we only support docker for now
+ self.setProperty('type', 'docker', 'type')
+ return SUCCESS
+
+class SetupStage4Steps(BuildStep):
+ name = 'Setup steps for stage4 image'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.setProperty('portage_repos_path', self.gentooci.config.project['project']['worker_portage_repos_path'], 'portage_repos_path')
+ aftersteps_list = []
+ separator = '\n'
+ log = yield self.addLog('makeing_stage4')
+ if self.getProperty("type") == 'docker':
+ print('build this stage4 %s on %s for %s' % (self.getProperty('project_uuid'), self.getProperty('workername'), self.getProperty('project_data')['name']))
+ self.descriptionDone = ' '.join(['build this stage4', self.getProperty('project_uuid'), 'on', self.getProperty('workername'), 'for', self.getProperty('project_data')['name']])
+ #FIXME: package list should be in the db project
+ package_list = ['dev-vcs/git', 'app-text/ansifilter', 'dev-util/pkgcheck', 'dev-lang/rust-bin', 'app-admin/eclean-kernel', 'app-portage/gentoolkit', 'sys-kernel/gentoo-kernel-bin', 'app-editors/nano']
+ if 'systemd' or 'openrc' in self.getProperty('project_data')['image']:
+ workerdest = yield os.path.join(self.getProperty("workerbase"), self.getProperty('project_uuid'))
+ workerdest_etc = yield os.path.join(workerdest, 'etc')
+ print(workerdest_etc)
+ # create dir
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Create stage4 dir',
+ command=['mkdir', self.getProperty('project_uuid')],
+ workdir=self.getProperty("workerbase")
+ ))
+ # download stage3
+ aftersteps_list.append(GetSteg3())
+ # setup portage
+ aftersteps_list.append(builders_steps.UpdateRepos(workdir=workerdest))
+ aftersteps_list.append(portage_steps.SetReposConf(workdir=workerdest))
+ aftersteps_list.append(portage_steps.SetMakeConf(workdir=workerdest))
+ # add localegen
+ #FIXME: set that in config
+ locale_conf = []
+ locale_conf.append('en_US.UTF-8 UTF-8')
+ locale_conf.append('en_US ISO-8859-1')
+ locale_conf.append('C.UTF8 UTF-8')
+ locale_conf_string = separator.join(locale_conf)
+ aftersteps_list.append(
+ steps.StringDownload(locale_conf_string + separator,
+ workerdest="locale.gen",
+ workdir=workerdest_etc
+ ))
+ yield log.addStdout('File: ' + 'locale.gen' + '\n')
+ for line in locale_conf:
+ yield log.addStdout(line + '\n')
+ aftersteps_list.append(
+ steps.StringDownload('LANG="en_US.utf8"' + separator,
+ workerdest="locale.conf",
+ workdir=workerdest_etc
+ ))
+ yield log.addStdout('Setting LANG to: ' + 'en_US.utf8' + '\n')
+ aftersteps_list.append(SetSystemdNspawnConf())
+ # run localgen
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Run locale-gen on the chroot',
+ command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'locale-gen'],
+ workdir=self.getProperty("workerbase")
+ ))
+ # update timezone
+ # install packages in world file config
+ command_list = ['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge']
+ for package in package_list:
+ command_list.append(package)
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Install programs on the chroot',
+ command=command_list,
+ workdir=self.getProperty("workerbase")
+ ))
+ # update container
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Run update on the chroot',
+ command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge', '--update', '--deep', '--newuse', '@world'],
+ workdir=self.getProperty("workerbase")
+ ))
+ # install buildbot-worker
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Install buildbot worker on the chroot',
+ command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge', 'buildbot-worker'],
+ workdir=self.getProperty("workerbase")
+ ))
+ #FIXME: move this to image build for chroot type part
+ if self.getProperty("type") == 'chroot':
+ # set hostname
+ aftersteps_list.append(steps.StringDownload(
+ self.getProperty("worker") + separator,
+ workerdest="hostname",
+ workdir=workerdest_etc
+ ))
+ yield log.addStdout('Setting hostname to: ' + self.getProperty("worker") + '\n')
+ # config buildbot-worker
+ # get password from db if set else generate one in uuid
+ worker_passwd = 'test1234'
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ SecretString=[worker_passwd, '<WorkerPassword>'],
+ name='Install buildbot worker on the chroot',
+ command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'buildbot-worker', 'create-worker', '/var/lib/buildbot_worker', '192.168.1.5', self.getProperty("worker"), worker_passwd],
+ workdir=self.getProperty("workerbase")
+ ))
+ if self.getProperty("type") == 'docker':
+ # copy docker_buildbot.tac to worker dir
+ buildbot_worker_config_file = yield os.path.join(self.master.basedir, 'files', 'docker_buildbot_worker.tac')
+ aftersteps_list.append(steps.FileDownload(
+ flunkOnFailure=True,
+ name='Upload buildbot worker config to the stage4',
+ mastersrc=buildbot_worker_config_file,
+ workerdest='var/lib/buildbot_worker/buildbot.tac',
+ workdir=workerdest
+ ))
+ # add info to the buildbot worker
+ worker_info_list = []
+ worker_info_list.append(self.getProperty('project_data')['name'])
+ worker_info_list.append(self.getProperty("stage3"))
+ worker_info_list.append(self.getProperty("type"))
+ #FIXME: worker name of self.getProperty('workername') from node table
+ worker_info_list.append('node1')
+ print(worker_info_list)
+ worker_info = ' '.join(worker_info_list)
+ aftersteps_list.append(steps.StringDownload(
+ worker_info + separator,
+ workerdest='var/lib/buildbot_worker/info/host',
+ workdir=workerdest
+ ))
+ # if self.getProperty("type") == 'chroot' and 'systemd' in self.getProperty('project_data')['image']:
+ # set buildbot worker to run
+ # depclean
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Depclean on the chroot',
+ command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge', '--depclean'],
+ workdir=self.getProperty("workerbase")
+ ))
+ # remove the gentoo repo
+ # compress it
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Compress the stage4',
+ command=['tar', '-cf', '--numeric-owner', self.getProperty('project_uuid') + '.tar', self.getProperty('project_uuid')],
+ workdir=self.getProperty("workerbase")
+ ))
+ # signing the stage4
+ # remove the dir
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Remove the stage4 dir',
+ command=['rm', '-R', self.getProperty('project_uuid')],
+ workdir=self.getProperty("workerbase")
+ ))
+ if aftersteps_list != []:
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
+
+class GetSteg3(BuildStep):
+ name = 'Get the steg3 image'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ compresstype = '.tar.xz'
+ downloadbaseurl = 'https://gentoo.osuosl.org/releases'
+ #downloadbaseurl = 'https://bouncer.gentoo.org/fetch/root/all/releases'
+ aftersteps_list = []
+ project_data = self.getProperty('project_data')
+ project_keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
+ keyword = project_keyword_data['name']
+ # if the stage3 end with latest we neeed to get the date
+ if project_data['image'].endswith('latest'):
+ stage3latest = 'latest' + '-' + project_data['image'].replace('-latest', '') + '.txt'
+ downloadurllatest = '/'.join([downloadbaseurl, keyword, 'autobuilds', stage3latest])
+ print(downloadurllatest)
+ session = requests.Session()
+ retry = Retry(connect=3, backoff_factor=0.5)
+ adapter = HTTPAdapter(max_retries=retry)
+ session.mount('http://', adapter)
+ session.mount('https://', adapter)
+ r = session.get(downloadurllatest, timeout=5)
+ if r.status_code == requests.codes.ok:
+ urltext = r.text.split('\n')[2]
+ print(urltext)
+ date = urltext.split('/')[0]
+ image = urltext.split('/')[1].split(' ')[0]
+ else:
+ r.raise_for_status()
+ return FAILURE
+ else:
+ image = project_data['image'] + compresstype
+ date = project_data['image'].split('-')[-1]
+ self.descriptionDone = image
+ self.setProperty('stage3', image, 'stage3')
+ downloadurlstage3 = '/'.join([downloadbaseurl, keyword, 'autobuilds', date, image])
+ print(downloadurlstage3)
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Download the stage3 file',
+ command=['wget', '-N', '-nv', downloadurlstage3],
+ workdir=self.getProperty("workerbase")
+ ))
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Download the stage3 DIGESTS file',
+ command=['wget', '-N', '-nv', downloadurlstage3 + '.asc'],
+ workdir=self.getProperty("workerbase")
+ ))
+ #FIXME: validate the stage3 file
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Unpack the stage3 file',
+ command=['tar', 'xpf', image, '--xattrs-include=\'*.*\'', '--numeric-owner', '-C', self.getProperty('project_uuid')],
+ workdir=self.getProperty("workerbase")
+ ))
+ if aftersteps_list != []:
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
+
+class SetSystemdNspawnConf(BuildStep):
+
+ name = 'SetSystemdNspawnConf'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ nspawn_conf_path = '/etc/systemd/nspawn/'
+ log = yield self.addLog(self.getProperty('project_uuid') + '.nspawn')
+ #FIXME: set it in config
+ separator = '\n'
+ nspawn_conf = []
+ nspawn_conf.append('[Files]')
+ nspawn_conf.append('TemporaryFileSystem=/run/lock')
+ # db node config portage cache bind
+ nspawn_conf.append('Bind=/srv/gentoo/portage/' + self.getProperty('project_uuid') + ':/var/cache/portage')
+ nspawn_conf.append('[Exec]')
+ nspawn_conf.append('Capability=CAP_NET_ADMIN')
+ nspawn_conf.append('[Network]')
+ nspawn_conf.append('VirtualEthernet=no')
+ nspawn_conf_string = separator.join(nspawn_conf)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(nspawn_conf_string + separator,
+ workerdest=self.getProperty('project_uuid') + '.nspawn',
+ workdir=nspawn_conf_path)
+ ])
+ yield log.addStdout('File: ' + self.getProperty('project_uuid') + '.nspawn' + '\n')
+ for line in nspawn_conf:
+ yield log.addStdout(line + '\n')
+ return SUCCESS
^ permalink raw reply related [flat|nested] 5+ messages in thread
end of thread, other threads:[~2022-04-22 12:38 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-01-17 9:25 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2022-04-22 12:38 Magnus Granberg
2021-02-22 20:28 Magnus Granberg
2021-01-11 17:45 Magnus Granberg
2021-01-10 21:41 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox