From: "Magnus Granberg" <zorry@gentoo.org>
To: gentoo-commits@lists.gentoo.org
Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, py/
Date: Tue, 21 Feb 2023 01:40:23 +0000 (UTC) [thread overview]
Message-ID: <1676943594.021a017b6e7ba89d28fac0e40db2ebb8baad9ce2.zorry@gentoo> (raw)
commit: 021a017b6e7ba89d28fac0e40db2ebb8baad9ce2
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 21 01:39:54 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Feb 21 01:39:54 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=021a017b
Get all search pattern that match
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 4 +-
buildbot_gentoo_ci/steps/logs.py | 178 ++++-----------------------------------
py/log_parser.py | 53 +++++++-----
3 files changed, 49 insertions(+), 186 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 801fc98..6c85aac 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -96,8 +96,8 @@ class GetBugs(BuildStep):
for match_word in match_search_text:
if match_word in match_bug_text:
matches = matches + 1
- if matches >= 10:
- print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}")
+ print(f"Bug: {str(bug['id'])} Matched words: {str(matches)} Summary: {bug['summary']}")
+ if matches >= 5:
match = {}
match['match'] = True
match['id'] = bug['id']
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 2a52308..e4fc951 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -19,6 +19,7 @@ from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
+from buildbot.plugins import util
from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
@@ -35,11 +36,15 @@ def PersOutputOfLogParser(rc, stdout, stderr):
for k, v in json.loads(line).items():
summary_log_dict[int(k)] = {
'text' : v['text'],
- 'type' : v['type'],
- 'status' : v['status'],
- 'id' : v['id'],
- 'search_pattern' : v['search_pattern']
+ 'pattern_infos' : [],
}
+ for s in v['pattern_info']:
+ summary_log_dict[int(k)]['pattern_infos'].append({
+ 'type' : s['type'],
+ 'status' : s['status'],
+ 'id' : s['id'],
+ 'search_pattern' : s['search_pattern'],
+ })
build_summery_output['summary_log_dict'] = summary_log_dict
#FIXME: Handling of stderr output
return {
@@ -140,6 +145,8 @@ class SetupParserBuildLoger(BuildStep):
command.append(log_cpv['full_logname'])
command.append('-u')
command.append(self.getProperty('project_data')['uuid'])
+ command.append('-d')
+ command.append(util.Secret("log_parser_database"))
self.aftersteps_list.append(steps.SetPropertyFromCommand(
name = 'RunBuildLogParser',
haltOnFailure = True,
@@ -152,159 +159,6 @@ class SetupParserBuildLoger(BuildStep):
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
-class ParserBuildLog(BuildStep):
-
- name = 'ParserBuildLog'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- self.logfile_text_dict = {}
- self.summery_dict = {}
- self.index = 1
- self.log_search_pattern_list = []
- self.max_text_lines = 0
- super().__init__(**kwargs)
-
- #FIXME: ansifilter
- def ansiFilter(self, text):
- return text
-
- @defer.inlineCallbacks
- def get_log_search_pattern(self):
- # get pattern from the projects
- # add that to log_search_pattern_list
- for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
- # check if the search pattern is vaild
- try:
- re.compile(project_pattern['search'])
- except re.error:
- print("Non valid regex pattern")
- print(project_pattern)
- else:
- self.log_search_pattern_list.append(project_pattern)
- # get the default project pattern
- # add if not pattern is in project ignore
- self.project_pattern_ignore = yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('project_data')['uuid'])
- for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])):
- if not project_pattern['search'] in self.project_pattern_ignore:
- # check if the search pattern is vaild
- try:
- re.compile(project_pattern['search'])
- except re.error:
- print("Non valid regex pattern")
- print(project_pattern)
- else:
- self.log_search_pattern_list.append(project_pattern)
-
- def search_buildlog(self, tmp_index):
- # get text line to search
- text_line = self.ansiFilter(self.logfile_text_dict[tmp_index])
- # loop true the pattern list for match
- for search_pattern in self.log_search_pattern_list:
- search_hit = False
- if search_pattern['search_type'] == 'in':
- if search_pattern['search'] in text_line:
- search_hit = True
- if search_pattern['search_type'] == 'startswith':
- if text_line.startswith(search_pattern['search']):
- search_hit = True
- if search_pattern['search_type'] == 'endswith':
- if text_line.endswith(search_pattern['search']):
- search_hit = True
- if search_pattern['search_type'] == 'search':
- if re.search(search_pattern['search'], text_line):
- search_hit = True
- # add the line if the pattern match
- if search_hit:
- print(text_line)
- print(search_pattern)
- print(tmp_index)
- self.summery_dict[tmp_index] = {}
- self.summery_dict[tmp_index]['text'] = text_line
- self.summery_dict[tmp_index]['type'] = search_pattern['type']
- self.summery_dict[tmp_index]['status'] = search_pattern['status']
- self.summery_dict[tmp_index]['search_pattern_id'] = search_pattern['id']
- # add upper text lines if requested
- # max 5
- if search_pattern['start'] != 0:
- i = tmp_index - search_pattern['start'] - 1
- match = True
- while match:
- i = i + 1
- if i < (tmp_index - 9) or i == tmp_index:
- match = False
- else:
- if not i in self.summery_dict:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i])
- self.summery_dict[i]['type'] = 'info'
- self.summery_dict[i]['status'] = 'info'
- # add lower text lines if requested
- # max 5
- if search_pattern['end'] != 0:
- i = tmp_index
- end = tmp_index + search_pattern['end']
- match = True
- while match:
- i = i + 1
- if i > self.max_text_lines or i > end:
- match = False
- else:
- if not i in self.summery_dict:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i])
- self.summery_dict[i]['type'] = 'info'
- self.summery_dict[i]['status'] = 'info'
- else:
- # we add all line that start with ' * ' as info
- # we add all line that start with '>>>' but not '>>> /' as info
- if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
- if not tmp_index in self.summery_dict:
- self.summery_dict[tmp_index] = {}
- self.summery_dict[tmp_index]['text'] = text_line
- self.summery_dict[tmp_index]['type'] = 'info'
- self.summery_dict[tmp_index]['status'] = 'info'
-
- @defer.inlineCallbacks
- def run(self):
- self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- yield self.get_log_search_pattern()
- # open the log file
- # read it to a buffer
- # make a dict of the buffer
- # maybe use mulitiprocces to speed up the search
- print(self.getProperty('log_build_data'))
- if self.getProperty('faild_cpv'):
- log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
- else:
- log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')]
- file_path = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']) ,log_cpv['full_logname'])
- #FIXME: decode it to utf-8
- with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f:
- for text_line in f:
- self.logfile_text_dict[self.index] = text_line.strip('\n')
- # run the parse patten on the line
- # have a buffer on 10 before we run pattern check
- if self.index >= 10:
- yield self.search_buildlog(self.index - 9)
- # remove text line that we don't need any more
- if self.index >= 20:
- del self.logfile_text_dict[self.index - 19]
- self.index = self.index + 1
- self.max_text_lines = self.index
- f.close()
- # check last 10 lines in logfile_text_dict
- yield self.search_buildlog(self.index - 10)
- print(self.summery_dict)
- # remove all lines with ignore in the dict
- # setProperty summery_dict
- self.setProperty("summary_log_dict", self.summery_dict, 'summary_log_dict')
- return SUCCESS
-
class MakeIssue(BuildStep):
name = 'MakeIssue'
@@ -353,8 +207,9 @@ class MakeIssue(BuildStep):
text_phase_list = []
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
- if v['type'] == self.error_dict['phase'] and v['status'] == 'error':
- text_issue_list.append(v['text'])
+ for s in v['pattern_infos']:
+ if s['type'] == self.error_dict['phase'] and s['status'] == 'error':
+ text_issue_list.append(v['text'])
# add the issue error
if text_issue_list != []:
self.error_dict['title_issue'] = text_issue_list[0].replace('*', '').strip()
@@ -379,8 +234,9 @@ class MakeIssue(BuildStep):
for k, v in sorted(self.summary_log_dict.items()):
self.summary_log_list.append(v['text'])
#self.error_dict['hash'].update(v['text'].encode('utf-8'))
- if v['status'] == 'warning':
- warning = True
+ for s in v['pattern_infos']:
+ if s['status'] == 'warning':
+ warning = True
# check if the build did fail
if v['text'].startswith(' * ERROR:') and v['text'].endswith(' phase):'):
# get phase error
diff --git a/py/log_parser.py b/py/log_parser.py
index dd48295..f5c4eb5 100644
--- a/py/log_parser.py
+++ b/py/log_parser.py
@@ -64,37 +64,41 @@ def get_log_search_pattern(Session, uuid, default_uuid):
return log_search_pattern
def get_search_pattern_match(log_search_pattern, text_line):
+ match_list = []
for search_pattern in log_search_pattern:
if re.search(search_pattern['search'], text_line):
- return search_pattern
- return False
+ match_list.append(search_pattern)
+ return match_list
def search_buildlog(log_search_pattern, text_line, index):
summary = {}
#FIXME: add check for test
# don't log ignore lines
- if get_search_pattern_match(log_search_pattern['ignore'], text_line):
+ if get_search_pattern_match(log_search_pattern['ignore'], text_line) != []:
return False
# search default pattern
- search_pattern_match = get_search_pattern_match(log_search_pattern['default'], text_line)
- if search_pattern_match:
- summary[index] = dict(
- text = text_line,
- type = search_pattern_match['type'],
- status = search_pattern_match['status'],
- id = search_pattern_match['id'],
- search_pattern = search_pattern_match['search']
- )
- return summary
+ summary[index] = {
+ 'text' : text_line,
+ 'pattern_info' : [],
+ }
+ search_pattern_match_list = get_search_pattern_match(log_search_pattern['default'], text_line)
+ if search_pattern_match_list != []:
+ for search_pattern_match in search_pattern_match_list:
+ summary[index]['pattern_info'].append({
+ 'type' : search_pattern_match['type'],
+ 'status' : search_pattern_match['status'],
+ 'id' : search_pattern_match['id'],
+ 'search_pattern' : search_pattern_match['search'],
+ })
# we add all line that start with ' * ' or '>>>' as info
if text_line.startswith(' * ') or text_line.startswith('>>>'):
- summary[index] = dict(
- text = text_line,
- type = 'info',
- status = 'info',
- id = 0,
- search_pattern = 'auto'
- )
+ summary[index]['pattern_info'].append({
+ 'type' : 'info',
+ 'status' : 'info',
+ 'id' : 0,
+ 'search_pattern' : 'auto',
+ })
+ if summary[index]['pattern_info'] != []:
return summary
return False
@@ -104,8 +108,8 @@ def getConfigSettings():
config = json.load(f)
return config
-def getDBSession(config):
- engine = sa.create_engine(config['database'])
+def getDBSession(args):
+ engine = sa.create_engine(args.database)
Session = sa.orm.sessionmaker(bind = engine)
return Session()
@@ -126,7 +130,7 @@ def runLogParser(args):
index = 1
logfile_text_dict = {}
config = getConfigSettings()
- Session = getDBSession(config)
+ Session = getDBSession(args)
#mp_pool = getMultiprocessingPool(config)
summary = {}
#NOTE: The patten is from https://github.com/toralf/tinderbox/tree/master/data files.
@@ -151,6 +155,9 @@ def main():
parser = argparse.ArgumentParser()
parser.add_argument("-f", "--file", required=True)
parser.add_argument("-u", "--uuid", required=True)
+ parser.add_argument("-e", "--default-uuid", required=False)
+ parser.add_argument("-c", "--cpu", required=False)
+ parser.add_argument("-d", "--database", required=True)
args = parser.parse_args()
runLogParser(args)
sys.exit()
next reply other threads:[~2023-02-21 1:40 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-02-21 1:40 Magnus Granberg [this message]
-- strict thread matches above, loose matches on Subject: below --
2024-02-21 21:16 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, py/ Magnus Granberg
2024-02-22 23:58 Magnus Granberg
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1676943594.021a017b6e7ba89d28fac0e40db2ebb8baad9ce2.zorry@gentoo \
--to=zorry@gentoo.org \
--cc=gentoo-commits@lists.gentoo.org \
--cc=gentoo-dev@lists.gentoo.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox