* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/
@ 2021-02-07 9:00 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2021-02-07 9:00 UTC (permalink / raw
To: gentoo-commits
commit: 080b3d481d7e239a941ae866d804cef49df537c5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 7 09:01:13 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Feb 7 09:01:13 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=080b3d48
Add run pretend on packages update on worker
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 35 +++++++++++++++++++
buildbot_gentoo_ci/db/model.py | 2 +-
buildbot_gentoo_ci/steps/builders.py | 54 +++++++++++++++++++++++++++++-
buildbot_gentoo_ci/steps/update_db.py | 2 +-
4 files changed, 90 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 861a4e1..2916847 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -108,4 +108,39 @@ def run_build_request():
f.addStep(builders.UpdateRepos())
# setup make.conf
f.addStep(builders.SetMakeConf())
+ # setup package.*
+ #f.addStep(portages.SetPackageUse())
+ # setup env
+ # setup files in /etc if needed
+ # run --regen if needed on repo
+ # update packages before any tests
+ # run pretend on packages update on worker
+ shell_commad_list = [
+ 'emerge',
+ '-uDNv',
+ '--changed-deps',
+ '--changed-use',
+ '--pretend',
+ '@world'
+ ]
+ f.addStep(buildbot_steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=builders.PersOutputOfEmerge,
+ workdir='/'
+ ))
+ # look at the log to see if we need to do stuff
+ # run update package on worker
+ # check log
+ # run pretend @preserved-rebuild if needed
+ # look at the log to see if we need to do stuff
+ # run @preserved-rebuild
+ # check log
+ # run depclean if set
+ # depclean pretend
+ # look at the log to see if we need to do stuff
+ # depclean
+ # check log
+ # setup make.conf if build id has changes make.conf as dict from SetMakeConf
+ # setup package.* env if build id has changes
return f
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index bd111be..4c5e04c 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -216,7 +216,7 @@ class Model(base.DBConnectorComponent):
# project's name
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE')),
- sa.Column('versions_uuid', sa.String(36),
+ sa.Column('version_uuid', sa.String(36),
sa.ForeignKey('versions.uuid', ondelete='CASCADE')),
sa.Column('status', sa.Enum('stable','unstable','negative','all'), nullable=False),
)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 085d096..862f0d4 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import os
+import re
from twisted.internet import defer
from twisted.python import log
@@ -11,6 +12,56 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
+def PersOutputOfEmerge(rc, stdout, stderr):
+ emerge_output = {}
+ emerge_output['rc'] = rc
+ package_dict = {}
+ print(stderr)
+ for line in stdout.split('\n'):
+ # package list
+ subdict = {}
+ if line.startswith('[ebuild'):
+ # split the line
+ # if binaries
+ if line.startswith('[ebuild'):
+ subdict['binary'] = False
+ else:
+ subdict['binary'] = True
+ # action [ N ] stuff
+ subdict['action'] = line[8:15].replace(' ', '')
+ # cpv
+ cpv_split = re.search('] (.+?) ', line).group(1).split(':')
+ cpv = cpv_split[0]
+ # repository
+ # slot
+ if cpv_split[1] == '':
+ subdict['slot'] = None
+ subdict['repository'] = cpv_split[2]
+ else:
+ subdict['slot'] = cpv_split[1]
+ subdict['repository'] = cpv_split[3]
+ # if action U version cpv
+ if 'U' in subdict['action']:
+ subdict['old_version'] = re.search(' \[(.+?)] ', line).group(1).split(':')
+ else:
+ subdict['old_version'] = None
+ # Use list
+ if 'USE=' in line:
+ subdict['use'] = re.search('USE="(.+?)" ', line).group(1).split(' ')
+ else:
+ subdict['use'] = None
+ # PYTHON_TARGETS list
+ if 'PYTHON_TARGETS=' in line:
+ subdict['python_targets'] = re.search('PYTHON_TARGETS="(.+?)" ', line).group(1).split(' ')
+ else:
+ subdict['python_targets'] = None
+ # CPU_FLAGS_X86 list
+ package_dict[cpv] = subdict
+ #FIXME: Handling of stderr output
+ return {
+ 'emerge_output' : emerge_output
+ }
+
class TriggerRunBuildRequest(BuildStep):
name = 'TriggerRunBuildRequest'
@@ -259,6 +310,7 @@ class SetMakeConf(BuildStep):
makeconf_variable_list.append('--rebuild-if-new-rev=y')
makeconf_variable_list.append('--rebuilt-binaries=y')
makeconf_variable_list.append('--usepkg=y')
+ makeconf_variable_list.append('--binpkg-respect-use=y')
makeconf_variable_list.append('--nospinner')
makeconf_variable_list.append('--color=n')
makeconf_variable_list.append('--ask=n')
@@ -279,7 +331,7 @@ class SetMakeConf(BuildStep):
if k['variable'] == 'ACCEPT_RESTRICT':
makeconf_variable_list.append('-fetch')
for v in makeconf_variables_values_data:
- if v['build_id'] is 0:
+ if v['build_id'] == 0:
makeconf_variable_list.append(v['value'])
if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list != []:
makeconf_variable_list.append('ACCEPT_LICENSE="*"')
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index 1ffbbff..ee2ba88 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -124,7 +124,7 @@ class CheckCPVGentooCiProject(BuildStep):
def run(self):
#self.cpv_changes = self.getProperty("cpv_changes")
self.cpv_changes = []
- self.cpv_changes.append('dev-lang/python-3.9.1-r1')
+ self.cpv_changes.append('dev-python/django-3.1.5')
self.cpv_changes.append('dev-python/scrypt-0.8.16')
print(self.cpv_changes)
print(self.getProperty("repository_data"))
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/
@ 2021-03-03 23:15 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2021-03-03 23:15 UTC (permalink / raw
To: gentoo-commits
commit: 2b37e2f71e33f58d1ee3a3af5d6994c3dccd0357
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 3 23:14:32 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Mar 3 23:14:32 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2b37e2f7
Add Project build id
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 1 +
buildbot_gentoo_ci/db/builds.py | 81 ++++++++++++++++++++++++++++++
buildbot_gentoo_ci/db/connector.py | 2 +
buildbot_gentoo_ci/db/model.py | 19 +++++++
buildbot_gentoo_ci/steps/builders.py | 47 +++++++++++++++++
5 files changed, 150 insertions(+)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index b592541..31543f9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -158,4 +158,5 @@ def run_build_request():
f.addStep(builders.RunEmerge(step='depclean'))
f.addStep(builders.RunEmerge(step='preserved-libs'))
f.addStep(builders.RunEmerge(step='depclean'))
+ f.addStep(builders.setBuildStatus())
return f
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
new file mode 100644
index 0000000..79814b7
--- /dev/null
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -0,0 +1,81 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+from buildbot.util import epoch2datetime
+
+class BuildsConnectorComponent(base.DBConnectorComponent):
+
+ #@defer.inlineCallbacks
+ def addBuild(self, project_build_data):
+ created_at = epoch2datetime(int(self.master.reactor.seconds()))
+ def thd(conn, no_recurse=False):
+ tbl = self.db.model.projects_builds
+ # get the highest current number
+ r = conn.execute(sa.select([sa.func.max(tbl.c.build_id)],
+ whereclause=(tbl.c.project_uuid == project_build_data['project_uuid'])))
+ number = r.scalar()
+ new_number = 1 if number is None else number + 1
+ try:
+ q = tbl.insert()
+ r = conn.execute(q, dict(project_uuid=project_build_data['project_uuid'],
+ version_uuid=project_build_data['version_uuid'],
+ buildbot_build_id=project_build_data['buildbot_build_id'],
+ status=project_build_data['status'],
+ requested=project_build_data['requested'],
+ created_at=created_at,
+ build_id=new_number))
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ id = None
+ new_number = None
+ else:
+ id = r.inserted_primary_key[0]
+ return id, new_number
+ return self.db.pool.do(thd)
+
+ @defer.inlineCallbacks
+ def setSatusBuilds(self, build_id, project_uuid, status):
+ updated_at = epoch2datetime(int(self.master.reactor.seconds()))
+ def thd(conn, no_recurse=False):
+
+ tbl = self.db.model.projects_builds
+ q = tbl.update()
+ q = q.where(tbl.c.build_id == build_id)
+ q = q.where(tbl.c.project_uuid == project_uuid)
+ conn.execute(q, updated_at=updated_at,
+ status=status)
+ yield self.db.pool.do(thd)
+
+ @defer.inlineCallbacks
+ def setBuildbotBuildIdBuilds(self, build_id, project_uuid, buildbot_build_id):
+ updated_at = epoch2datetime(int(self.master.reactor.seconds()))
+ def thd(conn, no_recurse=False):
+
+ tbl = self.db.model.projects_builds
+ q = tbl.update()
+ q = q.where(tbl.c.build_id == build_id)
+ q = q.where(tbl.c.project_uuid == project_uuid)
+ conn.execute(q, updated_at=updated_at,
+ buildbot_build_id=buildbot_build_id)
+ yield self.db.pool.do(thd)
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index 7d218ab..7256cab 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -37,6 +37,7 @@ from buildbot_gentoo_ci.db import packages
from buildbot_gentoo_ci.db import versions
from buildbot_gentoo_ci.db import keywords
from buildbot_gentoo_ci.db import portages
+from buildbot_gentoo_ci.db import builds
upgrade_message = textwrap.dedent("""\
@@ -83,6 +84,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.versions = versions.VersionsConnectorComponent(self)
self.keywords = keywords.KeywordsConnectorComponent(self)
self.portages = portages.PortagesConnectorComponent(self)
+ self.builds = builds.BuildsConnectorComponent(self)
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index aa4eabc..2221ba4 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -198,6 +198,25 @@ class Model(base.DBConnectorComponent):
sa.Column('preserved_libs', sa.Boolean, default=True),
)
+ projects_builds = sautils.Table(
+ "projects_builds", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('build_id', sa.Integer),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('version_uuid', sa.String(36),
+ sa.ForeignKey('versions.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('buildbot_build_id', sa.Integer),
+ sa.Column('status', sa.Enum('failed','completed','in-progress','waiting', 'warning'), nullable=False),
+ sa.Column('requested', sa.Boolean, default=False),
+ sa.Column('created_at', sa.DateTime, nullable=True),
+ sa.Column('updated_at', sa.DateTime, nullable=True),
+ sa.Column('deleted', sa.Boolean, default=False),
+ sa.Column('deleted_at', sa.DateTime, nullable=True),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 21472c7..d09c4b6 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -124,6 +124,7 @@ class TriggerRunBuildRequest(BuildStep):
'projectrepository_data' : self.getProperty('projectrepository_data'),
'use_data' : self.getProperty("use_data"),
'fullcheck' : self.getProperty("fullcheck"),
+ 'project_build_data' : None
}
)])
return SUCCESS
@@ -194,6 +195,28 @@ class SetupPropertys(BuildStep):
self.setProperty('preserved_libs', False, 'preserved-libs')
self.setProperty('depclean', False, 'depclean')
self.setProperty('cpv_build', False, 'cpv_build')
+ print(self.getProperty("buildnumber"))
+ if self.getProperty('project_build_data') is None:
+ project_build_data = {}
+ project_build_data['project_uuid'] = project_data['uuid']
+ project_build_data['version_uuid'] = self.getProperty("version_data")['uuid']
+ project_build_data['status'] = 'in-progress'
+ project_build_data['requested'] = False
+ project_build_data['buildbot_build_id'] = self.getProperty("buildnumber")
+ project_build_data['id'], project_build_data['build_id'] = yield self.gentooci.db.builds.addBuild(
+ project_build_data)
+ else:
+ project_build_data = self.getProperty('project_build_data')
+ yield self.gentooci.db.builds.setSatusBuilds(
+ project_build_data['build_id'],
+ project_build_data['project_uuid'],
+ 'in-progress')
+ yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
+ project_build_data['build_id'],
+ project_build_data['project_uuid'],
+ self.getProperty("buildnumber"))
+ self.setProperty('project_build_data', project_build_data, 'project_build_data')
+ print(self.getProperty("project_build_data"))
return SUCCESS
class UpdateRepos(BuildStep):
@@ -546,3 +569,27 @@ class RunBuild(BuildStep):
self.setProperty('preserved_libs', False, 'preserved-libs')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+
+class setBuildStatus(BuildStep):
+
+ name = 'setBuildStatus'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_build_data = self.getProperty('project_build_data')
+ if project_build_data['status'] == 'in-progress':
+ yield self.gentooci.db.builds.setSatusBuilds(
+ project_build_data['build_id'],
+ project_build_data['project_uuid'],
+ 'completed')
+ self.setProperty('project_build_data', project_build_data, 'project_build_data')
+ return SUCCESS
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/
@ 2021-03-28 8:38 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2021-03-28 8:38 UTC (permalink / raw
To: gentoo-commits
commit: 5dc7b0dd7863aac81b462302c2fc11635411927d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 28 08:37:27 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Mar 28 08:37:27 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5dc7b0dd
Add log parser steps
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 8 +-
buildbot_gentoo_ci/config/schedulers.py | 2 +-
buildbot_gentoo_ci/db/model.py | 14 ++
buildbot_gentoo_ci/db/projects.py | 39 +++++
buildbot_gentoo_ci/steps/logs.py | 229 +++++++++++++++++++++++++++++
5 files changed, 288 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index df7baef..d8db8f8 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -168,16 +168,18 @@ def parse_build_log():
# set needed Propertys
f.addStep(logs.SetupPropertys())
# pers the build log for info qa errors
- #f.addStep(logs.ParserBuildLog())
+ f.addStep(logs.ParserBuildLog())
# pers the log from pkg check
#f.addStep(logs.ParserPkgCheckLog())
# Upload the log to the cloud and remove the log
#f.addStep(logs.Upload())
# check the sum log if we need to make a issue/bug/pr report
# set it SUCCESS/FAILURE/WARNINGS
- #f.addStep(logs.MakeIssue())
+ f.addStep(logs.MakeIssue())
+ # add sum log to buildbot log
+ #f.addStep(logs.setBuildbotLog)
# set BuildStatus
- #f.addStep(logs.setBuildStatus())
+ f.addStep(logs.setBuildStatus())
# setup things for the irc bot
#f.addStep(logs.SetIrcInfo())
return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index efdac75..1941a82 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -74,7 +74,7 @@ def gentoo_schedulers():
builderNames=["build_request_data"])
run_build_request = schedulers.Triggerable(name="run_build_request",
builderNames=["run_build_request"])
- pers_build_log = schedulers.Triggerable(name="parse_build_log",
+ parse_build_log = schedulers.Triggerable(name="parse_build_log",
builderNames=["parse_build_log"])
s = []
s.append(test_updatedb)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 939059c..3abc686 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -221,6 +221,20 @@ class Model(base.DBConnectorComponent):
sa.Column('deleted_at', sa.DateTime, nullable=True),
)
+ projects_pattern = sautils.Table(
+ "projects_pattern", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('search', sa.String(50), nullable=False),
+ sa.Column('search_end', sa.String(50), nullable=True),
+ sa.Column('start', sa.Integer, default=0),
+ sa.Column('end', sa.Integer, default=0),
+ sa.Column('status', sa.Enum('info', 'warning', 'ignore', 'error'), default='info'),
+ sa.Column('type', sa.Enum('info', 'qa', 'compile', 'configure', 'install', 'postinst', 'prepare', 'setup', 'test', 'unpack', 'ignore'), default='info'),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 37ae2f1..2edc14a 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -166,6 +166,29 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectLogSearchPatternByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_pattern
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ return [self._row2dict_projects_pattern(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getProjectLogSearchPatternByUuidAndIgnore(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_pattern
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.status == 'ignore')
+ return [self._row2dict_projects_pattern(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -242,3 +265,19 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
depclean=row.depclean,
preserved_libs=row.preserved_libs
)
+
+ def _row2dict_projects_pattern(self, conn, row):
+ if row.search_end == '':
+ search_end = None
+ else:
+ search_end = row.search_end
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ search=row.search,
+ search_end=search_end,
+ start=row.start,
+ end=row.end,
+ status=row.status,
+ type=row.type
+ )
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
new file mode 100644
index 0000000..ae41fe5
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -0,0 +1,229 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import re
+import gzip
+import io
+import hashlib
+
+from portage.versions import catpkgsplit
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import WARNINGS
+from buildbot.plugins import steps
+
+class SetupPropertys(BuildStep):
+
+ name = 'SetupPropertys'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ # set this in config
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
+ default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+ version_data = yield self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
+ self.setProperty("project_data", project_data, 'project_data')
+ self.setProperty("default_project_data", default_project_data, 'default_project_data')
+ self.setProperty("version_data", version_data, 'version_data')
+ self.setProperty("status", 'completed', 'status')
+ return SUCCESS
+
+class ParserBuildLog(BuildStep):
+
+ name = 'ParserBuildLog'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ self.logfile_text_dict = {}
+ self.summery_dict = {}
+ self.index = 1
+ self.log_search_pattern_list = []
+ self.max_text_lines = self.index -1
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def get_log_search_pattern(self):
+ # get pattern from the profile
+ # add that to log_search_pattern_list
+ for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
+ self.log_search_pattern_list.append(project_pattern)
+ # get the default profile pattern
+ # add if not pattern is in project ignore
+ for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])):
+ match = True
+ for project_pattern_ignore in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('default_project_data')['uuid'])):
+ if project_pattern['search'] == project_pattern_ignore['search']:
+ match = False
+ if match:
+ self.log_search_pattern_list.append(project_pattern)
+ print(self.log_search_pattern_list)
+
+ def search_buildlog(self, tmp_index):
+ # get text line to search
+ text_line = self.logfile_text_dict[tmp_index]
+ # loop true the pattern list for match
+ for search_pattern in self.log_search_pattern_list:
+ if re.search(search_pattern['search'], text_line) and not search_pattern['status'] == 'ignore':
+ self.summery_dict[tmp_index] = {}
+ self.summery_dict[tmp_index]['text'] = text_line
+ self.summery_dict[tmp_index]['status'] = search_pattern['status']
+ # add upper text lines if requested
+ # max 10
+ if search_pattern['start'] is not 0:
+ i = tmp_index
+ i_start = i - search_pattern['start']
+ match = True
+ while match:
+ i = i - 1
+ if i < 0 or i < i_start:
+ match = False
+ else:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['status'] = 'info'
+ # add lower text lines if requested
+ # max 10
+ if search_pattern['end'] is not 0:
+ i = tmp_index
+ i_end = i + search_pattern['end']
+ match = True
+ while match:
+ i = i + 1
+ if i > self.max_text_lines or i > i_end:
+ match = False
+ else:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['status'] = 'info'
+ # add text lines if requested that we need to search for the end
+ # max 10
+ if search_pattern['search_end'] is not None:
+ i = tmp_index
+ match = True
+ while match:
+ i = i + 1
+ if i > self.max_text_lines:
+ match = False
+ if re.search(search_pattern['search_end'], self.logfile_text_dict[i]):
+ if not i + 1 > self.max_text_lines or not re.search(search_pattern['search_end'], self.logfile_text_dict[i + 1]):
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['status'] = 'info'
+ else:
+ match = False
+ else:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['status'] = 'info'
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ #FIXME:
+ # get the log parser pattern from db
+ yield self.get_log_search_pattern()
+ # open the log file
+ # read it to a buffer
+ # make a dict of the buffer
+ # maby use mulitiprocces to speed up the search
+ file_path = yield os.path.join(self.master.basedir, 'cpv_logs', self.getProperty('log_build_data')['full_logname'])
+ with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f:
+ for text_line in f:
+ self.logfile_text_dict[self.index] = text_line
+ # run the parse patten on the line
+ # have a buffer on 5 before we run pattern check
+ if self.index >= 10:
+ yield self.search_buildlog(self.index - 9)
+ # remove text line that we don't need any more
+ if self.index >= 20:
+ del self.logfile_text_dict[self.index - 19]
+ self.index = self.index + 1
+ f.close()
+ print(self.summery_dict)
+ # check last 5 lines in logfile_text_dict
+ # setProperty summery_dict
+ self.setProperty("summery_log_dict", self.summery_dict, 'summery_log_dict')
+ return SUCCESS
+
+class MakeIssue(BuildStep):
+
+ name = 'MakeIssue'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ #@defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ summery_log_dict = self.getProperty('summery_log_dict')
+ error = False
+ warning = False
+ self.summery_log_list = []
+ log_hash = hashlib.sha256()
+ for k, v in sorted(summery_log_dict.items()):
+ if v['status'] == 'error':
+ error = True
+ if v['status'] == 'warning':
+ warning = True
+ self.summery_log_list.append(v['text'])
+ log_hash.update(v['text'].encode('utf-8'))
+ # add build log
+ # add issue/bug/pr report
+ self.setProperty("summery_log_list", self.summery_log_list, 'summery_log_list')
+ if error:
+ self.setProperty("status", 'failed', 'status')
+ if warning:
+ self.setProperty("status", 'warning', 'status')
+ return SUCCESS
+
+class setBuildStatus(BuildStep):
+
+ name = 'setBuildStatus'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = False
+ flunkOnFailure = True
+ warnOnWarnings = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ project_build_data = self.getProperty('project_build_data')
+ yield self.gentooci.db.builds.setSatusBuilds(
+ project_build_data['build_id'],
+ project_build_data['project_uuid'],
+ self.getProperty('status')
+ )
+ if self.getProperty('status') == 'failed':
+ return FAILURE
+ if self.getProperty('status') == 'warning':
+ return WARNINGS
+ return SUCCESS
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-03-28 8:38 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-03-28 8:38 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2021-03-03 23:15 Magnus Granberg
2021-02-07 9:00 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox