* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
@ 2021-02-28 23:26 Magnus Granberg
0 siblings, 0 replies; 4+ messages in thread
From: Magnus Granberg @ 2021-02-28 23:26 UTC (permalink / raw
To: gentoo-commits
commit: f60d63024794752620e12f445e13d6a317030de3
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 28 23:25:37 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Feb 28 23:25:37 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f60d6302
Add git log commit id to version data
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/versions.py | 6 ++++--
buildbot_gentoo_ci/steps/version.py | 35 ++++++++++++++++++++++++++++++++++-
3 files changed, 39 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index b91f158..aa4eabc 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -240,6 +240,7 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('packages.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('file_hash', sa.String(255), nullable=False),
+ sa.Column('commit_id', sa.String(255), nullable=False),
sa.Column('deleted', sa.Boolean, default=False),
sa.Column('deleted_at', sa.DateTime, nullable=True),
)
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
index 25f2061..df9b4d3 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -56,14 +56,15 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
return res
@defer.inlineCallbacks
- def addVersion(self, name, package_uuid, file_hash):
+ def addVersion(self, name, package_uuid, file_hash, commit_id):
def thd(conn, no_recurse=False):
try:
tbl = self.db.model.versions
q = tbl.insert()
r = conn.execute(q, dict(name=name,
package_uuid=package_uuid,
- file_hash=file_hash))
+ file_hash=file_hash,
+ commit_id=commit_id))
except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
uuid = None
else:
@@ -107,6 +108,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
name=row.name,
package_uuid=row.package_uuid,
file_hash=row.file_hash,
+ commit_id=row.commit_id,
deleted=row.deleted,
deleted_at=row.deleted_at
)
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index 5430e6e..d47a253 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -3,6 +3,7 @@
import re
import os
+import git
from portage.xml.metadata import MetaDataXML
from portage.checksum import perform_checksum
@@ -61,10 +62,12 @@ class AddVersion(BuildStep):
self.version_data['name'] = self.getProperty("version")
self.version_data['package_uuid'] = self.getProperty("package_data")['uuid']
self.version_data['file_hash'] = self.getProperty("ebuild_file_hash")
+ self.version_data['commit_id'] = self.getProperty("commit_id")
self.version_data['uuid'] = yield self.gentooci.db.versions.addVersion(
self.version_data['name'],
self.version_data['package_uuid'],
- self.version_data['file_hash']
+ self.version_data['file_hash'],
+ self.version_data['commit_id']
)
print(self.version_data)
self.setProperty("version_data", self.version_data, 'version_data')
@@ -98,6 +101,33 @@ class GetAuxMetadata(BuildStep):
yield portdbapi.portdbapi_instances.remove(self.myportdb)
return SUCCESS
+class GetCommitdata(BuildStep):
+
+ name = 'GetCommitdata'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ #FIXME: Could be a better way to get the log
+ git_log_ebuild = ''
+ g = git.Git(self.getProperty("repository_path"))
+ index = 1
+ git_log_dict = {}
+ git_log = yield g.log('-n 1', self.getProperty("ebuild_file"))
+ print(git_log)
+ for line in git_log.splitlines():
+ git_log_dict[index] = line
+ index = index + 1
+ self.setProperty('commit_id', re.sub('commit ', '', git_log_dict[1]), 'commit_id')
+ return SUCCESS
+
class AddVersionKeyword(BuildStep):
name = 'AddVersionKeyword'
@@ -178,6 +208,7 @@ class CheckPathHash(BuildStep):
self.ebuild_file_hash = None
self.setProperty('ebuild_file', self.ebuild_file, 'ebuild_file')
self.setProperty('ebuild_file_hash', self.ebuild_file_hash, 'ebuild_file_hash')
+ self.setProperty('repository_path', self.repository_path, 'repository_path')
return SUCCESS
class TriggerBuildCheck(BuildStep):
@@ -248,6 +279,7 @@ class CheckV(BuildStep):
addStepVData.append(DeleteOldVersion())
if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None:
if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']:
+ addStepVData.append(GetCommitdata())
addStepVData.append(AddVersion())
addStepVData.append(GetAuxMetadata())
addStepVData.append(AddVersionKeyword())
@@ -256,6 +288,7 @@ class CheckV(BuildStep):
else:
return SUCCESS
if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None:
+ addStepVData.append(GetCommitdata())
addStepVData.append(AddVersion())
addStepVData.append(GetAuxMetadata())
addStepVData.append(AddVersionKeyword())
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
@ 2021-04-17 17:28 Magnus Granberg
0 siblings, 0 replies; 4+ messages in thread
From: Magnus Granberg @ 2021-04-17 17:28 UTC (permalink / raw
To: gentoo-commits
commit: 0fbd4c06d15a194eb41e56e69583b31ccd847e29
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 17 17:27:51 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Apr 17 17:27:51 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0fbd4c06
Add support for search type in search pattern
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/model.py | 2 +-
buildbot_gentoo_ci/db/projects.py | 6 +--
buildbot_gentoo_ci/steps/logs.py | 94 ++++++++++++++++++---------------------
3 files changed, 46 insertions(+), 56 deletions(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index ede5cb7..ca9932a 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -225,11 +225,11 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('search', sa.String(50), nullable=False),
- sa.Column('search_end', sa.String(50), nullable=True),
sa.Column('start', sa.Integer, default=0),
sa.Column('end', sa.Integer, default=0),
sa.Column('status', sa.Enum('info', 'warning', 'ignore', 'error'), default='info'),
sa.Column('type', sa.Enum('info', 'qa', 'compile', 'configure', 'install', 'postinst', 'prepare', 'setup', 'test', 'unpack', 'ignore'), default='info'),
+ sa.Column('search_type', sa.Enum('in', 'startswith', 'endswith', 'search'), default='in'),
)
keywords = sautils.Table(
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 2393011..176be92 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -266,15 +266,11 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
)
def _row2dict_projects_pattern(self, conn, row):
- if row.search_end == '':
- search_end = None
- else:
- search_end = row.search_end
return dict(
id=row.id,
project_uuid=row.project_uuid,
search=row.search,
- search_end=search_end,
+ search_type=row.search_type,
start=row.start,
end=row.end,
status=row.status,
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 4c3bf01..178d71e 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -75,13 +75,13 @@ class ParserBuildLog(BuildStep):
match = False
if match:
self.log_search_pattern_list.append(project_pattern)
- print(self.log_search_pattern_list)
def search_buildlog(self, tmp_index):
# get text line to search
text_line = self.logfile_text_dict[tmp_index]
# loop true the pattern list for match
for search_pattern in self.log_search_pattern_list:
+ search_hit = False
# we add all line that start with ' * ' as info
# we add all line that start with '>>>' but not '>>> /' as info
if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
@@ -89,63 +89,57 @@ class ParserBuildLog(BuildStep):
self.summery_dict[tmp_index]['text'] = text_line
self.summery_dict[tmp_index]['type'] = 'info'
self.summery_dict[tmp_index]['status'] = 'info'
- if re.search(search_pattern['search'], text_line):
+ self.summery_dict[tmp_index]['search_pattern_id'] = 0
+ if search_pattern['search_type'] == 'in':
+ if search_pattern['search'] in text_line:
+ search_hit = True
+ if search_pattern['search_type'] == 'startswith':
+ if text_line.startswith(search_pattern['search']):
+ search_hit = True
+ if search_pattern['search_type'] == 'endswith':
+ if text_line.endswith(search_pattern['search']):
+ search_hit = True
+ if search_pattern['search_type'] == 'search':
+ if search_pattern['search'] in text_line:
+ search_hit = True
+ if search_hit:
+ print(text_line)
+ print(search_pattern['search'])
self.summery_dict[tmp_index] = {}
self.summery_dict[tmp_index]['text'] = text_line
self.summery_dict[tmp_index]['type'] = search_pattern['type']
self.summery_dict[tmp_index]['status'] = search_pattern['status']
+ self.summery_dict[tmp_index]['search_pattern_id'] = search_pattern['id']
# add upper text lines if requested
# max 10
- if search_pattern['start'] != 0:
- i = tmp_index
- i_start = i - search_pattern['start']
- match = True
- while match:
- i = i - 1
- if i < 0 or i < i_start:
- match = False
- else:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] = self.logfile_text_dict[i]
- self.summery_dict[i]['type'] = search_pattern['type']
- self.summery_dict[i]['status'] = 'info'
+ if search_pattern['start'] != 0 and search_hit:
+ i = tmp_index
+ i_start = i - search_pattern['start']
+ match = True
+ while match:
+ i = i - 1
+ if i < 0 or i < i_start:
+ match = False
+ else:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['type'] = search_pattern['type']
+ self.summery_dict[i]['status'] = 'info'
# add lower text lines if requested
# max 10
- if search_pattern['end'] != 0:
- i = tmp_index
- i_end = i + search_pattern['end']
- match = True
- while match:
- i = i + 1
- if i > self.max_text_lines or i > i_end:
- match = False
- else:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] = self.logfile_text_dict[i]
- self.summery_dict[i]['type'] = search_pattern['type']
- self.summery_dict[i]['status'] = 'info'
- # add text lines if requested that we need to search for the end
- # max 10
- if search_pattern['search_end'] is not None:
- i = tmp_index
- match = True
- while match:
- i = i + 1
- if i > self.max_text_lines:
- match = False
- if re.search(search_pattern['search_end'], self.logfile_text_dict[i]):
- if not i + 1 > self.max_text_lines or not re.search(search_pattern['search_end'], self.logfile_text_dict[i + 1]):
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] = self.logfile_text_dict[i]
- self.summery_dict[i]['type'] = search_pattern['type']
- self.summery_dict[i]['status'] = 'info'
- else:
- match = False
- else:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] = self.logfile_text_dict[i]
- self.summery_dict[i]['type'] = search_pattern['type']
- self.summery_dict[i]['status'] = 'info'
+ if search_pattern['end'] != 0 and search_hit:
+ i = tmp_index
+ i_end = i + search_pattern['end']
+ match = True
+ while match:
+ i = i + 1
+ if i > self.max_text_lines or i > i_end:
+ match = False
+ else:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['type'] = search_pattern['type']
+ self.summery_dict[i]['status'] = 'info'
@defer.inlineCallbacks
def run(self):
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
@ 2021-10-25 21:24 Magnus Granberg
0 siblings, 0 replies; 4+ messages in thread
From: Magnus Granberg @ 2021-10-25 21:24 UTC (permalink / raw
To: gentoo-commits
commit: f5cbe07ace4abc03c04e21a43e028f1edc24121d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 25 21:24:51 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Oct 25 21:24:51 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f5cbe07a
Add AddVersionRestrictions and AddVersionIUse to Version buildstep
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/model.py | 10 ++++++
buildbot_gentoo_ci/db/versions.py | 38 ++++++++++++++++++++
buildbot_gentoo_ci/steps/version.py | 72 +++++++++++++++++++++++++++++++++++++
3 files changed, 120 insertions(+)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 12d5ee9..9e380c8 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -301,6 +301,16 @@ class Model(base.DBConnectorComponent):
sa.Column('status', sa.Enum('stable','unstable','negative','all'), nullable=False),
)
+ versions_metadata = sautils.Table(
+ "versions_metadata", metadata,
+ # unique id per project
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('version_uuid', sa.String(36),
+ sa.ForeignKey('versions.uuid', ondelete='CASCADE')),
+ sa.Column('metadata', sa.Enum('restrict', 'properties', 'iuse', 'required use', 'keyword'), nullable=False),
+ sa.Column('value', sa.String(255), nullable=False),
+ )
+
workers = sautils.Table(
"workers", metadata,
# unique id per project
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
index fcc19ee..6b534f3 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -101,6 +101,36 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def addMetadata(self, version_uuid, metadata, value):
+ def thd(conn, no_recurse=False):
+ try:
+ tbl = self.db.model.versions_metadata
+ q = tbl.insert()
+ r = conn.execute(q, dict(version_uuid=version_uuid,
+ metadata=metadata,
+ value=value))
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ id = None
+ else:
+ id = r.inserted_primary_key[0]
+ return uuid
+ res = yield self.db.pool.do(thd)
+ return res
+
+ #FIXME: return sorted by id
+ @defer.inlineCallbacks
+ def getMetadataByUuidAndMatadata(self, uuid, metadata):
+ def thd(conn):
+ tbl = self.db.model.versions_metadata
+ q = tbl.select()
+ q = q.where(tbl.c.version_uuid == uuid)
+ q = q.where(tbl.c.metadata == metadata)
+ return [self._row2dict_version_metadata(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -111,3 +141,11 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
deleted=row.deleted,
deleted_at=row.deleted_at
)
+
+ def _row2dict_version_metadata(self, conn, row):
+ return dict(
+ id=row.id,
+ version_uuid=row.version_uuid,
+ metadata=row.metadata,
+ value=row.value
+ )
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index df97536..bff1e57 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -16,10 +16,21 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
+from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import portage as portage_steps
+def getIUseValue(auxdb_iuse):
+ status = False
+ if auxdb_iuse[0] in ['+']:
+ status = True
+ if auxdb_iuse[0] in ['+'] or auxdb_iuse[0] in ['-']:
+ iuse = auxdb_iuse[1:]
+ else:
+ iuse = auxdb_iuse
+ return iuse, status
+
class GetVData(BuildStep):
name = 'GetVData'
@@ -96,6 +107,7 @@ class GetCommitdata(BuildStep):
self.setProperty('commit_id', self.getProperty("change_data")['revision'], 'commit_id')
return SUCCESS
+#FIXME: use versions_metadata table
class AddVersionKeyword(BuildStep):
name = 'AddVersionKeyword'
@@ -155,6 +167,64 @@ class AddVersionKeyword(BuildStep):
self.setProperty('version_keyword_dict', self.version_keyword_dict, 'version_keyword_dict')
return SUCCESS
+class AddVersionRestrictions(BuildStep):
+
+ name = 'AddVersionRestrictions'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ auxdb = self.getProperty("auxdb")['RESTRICT']
+ if auxdb is None or not isinstance(auxdb, list):
+ return SKIPPED
+ for restrict in auxdb:
+ version_metadata_data = {}
+ version_metadata_data['version_uuid'] = self.getProperty("version_data")['uuid']
+ version_metadata_data['metadata'] = 'restrict'
+ version_metadata_data['value'] = restrict
+ version_metadata_data['id'] = yield self.gentooci.db.versions.addMetadata(
+ version_metadata_data['version_uuid'],
+ version_metadata_data['metadata'],
+ version_metadata_data['value'])
+ return SUCCESS
+
+class AddVersionIUse(BuildStep):
+
+ name = 'AddVersionIUse'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ auxdb = self.getProperty("auxdb")['IUSE']
+ if auxdb is None or not isinstance(auxdb, list):
+ return SKIPPED
+ for iuse in auxdb:
+ version_metadata_data = {}
+ version_metadata_data['version_uuid'] = self.getProperty("version_data")['uuid']
+ version_metadata_data['metadata'] = 'iuse'
+ version_metadata_data['value'] = iuse
+ version_metadata_data['id'] = yield self.gentooci.db.versions.addMetadata(
+ version_metadata_data['version_uuid'],
+ version_metadata_data['metadata'],
+ version_metadata_data['value'])
+ return SUCCESS
+
class CheckPathHash(BuildStep):
name = 'CheckPathHash'
@@ -272,6 +342,8 @@ class CheckV(BuildStep):
addStepVData.append(portage_steps.GetAuxMetadata())
addStepVData.append(AddVersion())
addStepVData.append(AddVersionKeyword())
+ addStepVData.append(AddVersionRestrictions())
+ addStepVData.append(AddVersionIUse())
addStepVData.append(TriggerBuildCheck())
yield self.build.addStepsAfterCurrentStep(addStepVData)
return SUCCESS
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/
@ 2024-03-14 21:01 Magnus Granberg
0 siblings, 0 replies; 4+ messages in thread
From: Magnus Granberg @ 2024-03-14 21:01 UTC (permalink / raw
To: gentoo-commits
commit: 0693e5f1e3191f2b6ecb3938610731c745d69f4a
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 14 21:00:44 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Mar 14 21:00:44 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0693e5f1
Get faild_version_data when faild and add more info in find bugs
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/packages.py | 3 ++-
buildbot_gentoo_ci/db/versions.py | 3 ++-
buildbot_gentoo_ci/steps/bugs.py | 16 +++++++++++++---
buildbot_gentoo_ci/steps/builders.py | 23 +++++++++++++++--------
buildbot_gentoo_ci/steps/logs.py | 16 +++++++++++-----
5 files changed, 43 insertions(+), 18 deletions(-)
diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py
index 1f68637..ad4a244 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -27,12 +27,13 @@ from buildbot.db import base
class PackagesConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
- def getPackageByName(self, name, c_uuid, repo_uuid):
+ def getPackageByName(self, name, c_uuid, repo_uuid, deleted=False):
def thd(conn):
tbl = self.db.model.packages
q = tbl.select()
q = q.where(tbl.c.name == name)
q = q.where(tbl.c.category_uuid == c_uuid)
+ q = q.where(tbl.c.deleted == deleted)
q = q.where(tbl.c.repository_uuid == repo_uuid)
res = conn.execute(q)
row = res.fetchone()
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
index 8d35602..afb9e52 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -26,12 +26,13 @@ from buildbot.db import base
class VersionsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
- def getVersionByName(self, name, p_uuid):
+ def getVersionByName(self, name, p_uuid, deleted=False):
def thd(conn):
tbl = self.db.model.versions
q = tbl.select()
q = q.where(tbl.c.name == name)
q = q.where(tbl.c.package_uuid == p_uuid)
+ q = q.where(tbl.c.deleted == deleted)
res = conn.execute(q)
row = res.fetchone()
if not row:
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 6a097d4..52bd08d 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -123,26 +123,36 @@ class GetBugs(BuildStep):
# we splite the lines to lists and try to match the words
matches = 0
bug_words = list(bug['summary'].split())
+ # add c/p and v
+ for word in bug_words:
+ if pkgsplit(word) is not None:
+ bug_words.append(pkgsplit(word)[0])
+ bug_words.append(cpv_getversion(word))
print(bug_words)
print(self.getProperty('error_dict')['words'])
for word in bug_words:
# clean the word
word = finishTitle(stripQuotesAndMore(word))
- print(word)
for match_word in self.getProperty('error_dict')['words']:
- if match_word in word:
+ if match_word == word:
matches = matches + 1
print(f"Bug: {str(bug['id'])} Matched words: {str(matches)} Summary: {bug['summary']}")
- if matches >= len(self.getProperty('error_dict')['title_nice']) - self.bug_config['min_match_word']:
+ len_match_words = len(self.getProperty('error_dict')['words']) - self.bug_config['min_match_word']
+ print(f"Min words to matchs: {str(len_match_words)}")
+ if matches >= len_match_words or matches >= self.bug_config['max_match_word']:
match = {}
match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
yield log.addStdout(f"Line to match: {self.getProperty('error_dict')['title_issue']}\n")
+ yield log.addStdout(f"Nice line to match: {self.getProperty('error_dict')['title_nice']}\n")
+ yield log.addStdout(f"Words to match: {self.getProperty('error_dict')['words']}\n")
if match:
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary: {match['summary']}\n")
self.setProperty("bgo", match, 'bgo')
+ print(f"Match bug: YES")
+ print(f"Bug: {str(match['id'])} Summary: {match['summary']}")
return match
yield log.addStdout('Match bug: NO\n')
return match
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 117fb70..09739d7 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -282,6 +282,7 @@ class GetProjectRepositoryData(BuildStep):
continue
# check if auto, enabled and not in config.project['project']
if project_data['auto'] is True and project_data['enabled'] is True and project_data['name'] != self.gentooci.config.project['project']['update_db']:
+ print(project_data)
# set Property projectrepository_data so we can use it in the trigger
# get name o project keyword
project_keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
@@ -729,13 +730,14 @@ class CheckEmergeLogs(BuildStep):
self.faild_cpv = False
@defer.inlineCallbacks
- def getVersionData(self, cpv):
+ def getVersionData(self, cpv, repository):
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
category_data = yield self.gentooci.db.categorys.getCategoryByName(c)
+ repository_data = yield self.gentooci.db.repositorys.getRepositoryByName(repository)
package_data = yield self.gentooci.db.packages.getPackageByName(p,
- category_data['uuid'],
- self.getProperty('repository_data')['uuid'])
+ category_data['uuid'],
+ repository_data['uuid'])
if package_data is None:
return None
version = yield cpv_getversion(cpv)
@@ -868,6 +870,7 @@ class CheckEmergeLogs(BuildStep):
]
emerge_output = self.getProperty('emerge_output')
self.faild_cpv = emerge_output['failed']
+ self.faild_version_data = False
package_dict = emerge_output['packages']
#FIXME: Prosees the logs and do stuff
@@ -1006,7 +1009,6 @@ class CheckEmergeLogs(BuildStep):
print('log_dict not emty')
# requested cpv
cpv = self.getProperty('cpv')
- faild_version_data = False
if cpv in log_dict or self.faild_cpv in log_dict:
print(cpv)
print(self.faild_cpv)
@@ -1022,14 +1024,19 @@ class CheckEmergeLogs(BuildStep):
if not rebuild or self.getProperty('rerun') >= retrays:
# failed and build requested cpv
if cpv == self.faild_cpv:
- faild_version_data = self.getProperty("version_data")
+ self.faild_version_data = self.getProperty("version_data")
else:
# failed but not build requested cpv
self.log_data[self.faild_cpv] = log_dict[self.faild_cpv]
yield self.getLogFile(self.faild_cpv, log_dict)
- faild_version_data = yield self.getVersionData(self.faild_cpv)
+ repository = False
+ for cpv, v in package_dict.items():
+ if cpv == self.faild_cpv:
+ repository = v['repository']
+ if repository:
+ self.faild_version_data = yield self.getVersionData(self.faild_cpv, repository)
self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
- print(faild_version_data)
+ print(self.faild_version_data)
print('getEmergeFiles')
self.getEmergeFiles(self.faild_cpv)
self.getBuildWorkDirs(self.faild_cpv)
@@ -1045,7 +1052,7 @@ class CheckEmergeLogs(BuildStep):
updateSourceStamp=False,
set_properties={
'cpv' : self.getProperty("cpv"),
- 'faild_version_data' : faild_version_data,
+ 'faild_version_data' : self.faild_version_data,
'project_build_data' : self.getProperty('project_build_data'),
'log_build_data' : self.log_data,
'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 84f5058..f214ad6 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -8,7 +8,7 @@ import io
import hashlib
import json
-from portage.versions import catpkgsplit
+from portage.versions import catpkgsplit, cpv_getversion
from twisted.internet import defer
from twisted.python import log
@@ -209,6 +209,7 @@ class MakeIssue(BuildStep):
p = catpkgsplit(self.error_dict['cpv'])[1]
cp = c + '/' + p
word_list.append(cp)
+ word_list.append(cpv_getversion(self.error_dict['cpv']))
for word in self.error_dict['title_phase'].split(' '):
word_list.append(word)
for word in self.error_dict['title_issue_nice'].split(' '):
@@ -273,16 +274,21 @@ class MakeIssue(BuildStep):
emails['maintainers'] = Maintainers
emails['owners'] = self.getProperty('owners')
emails['packages_emails'] = []
- pmail_ids = yield self.gentooci.db.packages.getEmailsIdsByPackageUuid(self.getProperty("version_data")['package_uuid'])
+ if self.getProperty('failed_version_data'):
+ version_data = self.getProperty('failed_version_data')
+ else:
+ version_data = self.getProperty("version_data")
+ pmail_ids = yield self.gentooci.db.packages.getEmailsIdsByPackageUuid(version_data['package_uuid'])
for pmail_id in pmail_ids:
pmail = yield self.gentooci.db.packages.getEmailByEmailId(pmail_id['email_id'])
- emails['packages_emails'].append(pmail['email'])
- changes = yield self.master.db.changes.getChange(self.getProperty("version_data")['change_id'])
+ pmail_id['email'] = pmail['email']
+ emails['packages_emails'].append(pmail_id)
+ changes = yield self.master.db.changes.getChange(version_data['change_id'])
cemail = {}
cemail['authors'] = changes['author']
#cemail['committer'] = changes['committer']
cemail['changes_uids_email'] = []
- changes_uids = yield self.master.db.changes.getChangeUids(self.getProperty("version_data")['change_id'])
+ changes_uids = yield self.master.db.changes.getChangeUids(version_data['change_id'])
for changes_uid in changes_uids:
cuemail = yield self.master.db.users.getUser(changes_uid)
cemail['changes_uids_email'].append(cuemail['identifier'])
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2024-03-14 21:01 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-10-25 21:24 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2024-03-14 21:01 Magnus Granberg
2021-04-17 17:28 Magnus Granberg
2021-02-28 23:26 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox