* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/logs/
@ 2022-01-24 0:49 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2022-01-24 0:49 UTC (permalink / raw
To: gentoo-commits
commit: c6162bc10dd6f86d170b31c1a33932d96dd5360b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 24 00:49:01 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 24 00:49:01 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c6162bc1
disable some small checks like ('./\w+/')
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/logs/log_parser.py | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/buildbot_gentoo_ci/logs/log_parser.py b/buildbot_gentoo_ci/logs/log_parser.py
index 11faf48..7531532 100644
--- a/buildbot_gentoo_ci/logs/log_parser.py
+++ b/buildbot_gentoo_ci/logs/log_parser.py
@@ -75,8 +75,8 @@ def search_buildlog(log_search_pattern_list, logfile_text_dict, tmp_index, max_t
#FIXME take the ignore line pattern from db
if text_line.startswith('>>> /'):
pass
- if else re.search('./\w+/'):
- pass
+ #if else re.search('./\w+/'):
+ # pass
else:
# search for match
if search_pattern['search_type'] == 'in':
@@ -137,7 +137,7 @@ def search_buildlog(log_search_pattern_list, logfile_text_dict, tmp_index, max_t
else:
# we add all line that start with ' * ' as info
# we add all line that start with '>>>' but not '>>> /' as info
- if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
+ if text_line.startswith(' * ') or text_line.startswith('>>>'):
if not tmp_index in summery_dict:
summery_dict[tmp_index] = {}
summery_dict[tmp_index]['text'] = text_line
@@ -145,7 +145,6 @@ def search_buildlog(log_search_pattern_list, logfile_text_dict, tmp_index, max_t
summery_dict[tmp_index]['status'] = 'info'
summery_dict[tmp_index]['id'] = 0
summery_dict[tmp_index]['search_pattern'] = 'auto'
- #FIXME: print json
if summery_dict == {}:
return None
return summery_dict
@@ -167,13 +166,13 @@ def runLogParser(args):
# Is stored in a db instead of files.
log_search_pattern_list = get_log_search_pattern(Session, args.uuid, config['default_uuid'])
Session.close()
+ #FIXME: UnicodeDecodeError: 'utf-8' codec can't decode byte ... in some logs
with io.TextIOWrapper(io.BufferedReader(gzip.open(args.file, 'rb'))) as f:
- #FIXME: add support for multiprocessing
- for text_line in f:
- logfile_text_dict[index] = text_line.strip('\n')
- index = index + 1
- max_text_lines = index
- f.close()
+ for text_line in f:
+ logfile_text_dict[index] = text_line.strip('\n')
+ index = index + 1
+ max_text_lines = index
+ f.close()
# run the parse patten on the line
for tmp_index, text in logfile_text_dict.items():
res = mp_pool.apply_async(search_buildlog, (log_search_pattern_list, logfile_text_dict, tmp_index, max_text_lines,))
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/logs/
@ 2022-02-03 21:23 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2022-02-03 21:23 UTC (permalink / raw
To: gentoo-commits
commit: 57d8ef85943af7ddfeaa51d0cacdacba80a75a13
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 3 21:23:28 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Feb 3 21:23:28 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=57d8ef85
Fix UnicodeDecodeError in log parser
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/logs/log_parser.py | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/buildbot_gentoo_ci/logs/log_parser.py b/buildbot_gentoo_ci/logs/log_parser.py
index 7531532..448192c 100644
--- a/buildbot_gentoo_ci/logs/log_parser.py
+++ b/buildbot_gentoo_ci/logs/log_parser.py
@@ -166,13 +166,10 @@ def runLogParser(args):
# Is stored in a db instead of files.
log_search_pattern_list = get_log_search_pattern(Session, args.uuid, config['default_uuid'])
Session.close()
- #FIXME: UnicodeDecodeError: 'utf-8' codec can't decode byte ... in some logs
- with io.TextIOWrapper(io.BufferedReader(gzip.open(args.file, 'rb'))) as f:
- for text_line in f:
- logfile_text_dict[index] = text_line.strip('\n')
- index = index + 1
- max_text_lines = index
- f.close()
+ for text_line in io.TextIOWrapper(io.BufferedReader(gzip.open(args.file)), encoding='utf8', errors='ignore'):
+ logfile_text_dict[index] = text_line.strip('\n')
+ index = index + 1
+ max_text_lines = index
# run the parse patten on the line
for tmp_index, text in logfile_text_dict.items():
res = mp_pool.apply_async(search_buildlog, (log_search_pattern_list, logfile_text_dict, tmp_index, max_text_lines,))
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/logs/
@ 2022-06-11 9:55 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2022-06-11 9:55 UTC (permalink / raw
To: gentoo-commits
commit: 1d74ddc1298a77bab0de91d0d8f91a15c9c4ea17
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 11 09:55:13 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Jun 11 09:55:13 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1d74ddc1
Add ignore_line = False
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/logs/log_parser.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/buildbot_gentoo_ci/logs/log_parser.py b/buildbot_gentoo_ci/logs/log_parser.py
index 448192c..b890c12 100644
--- a/buildbot_gentoo_ci/logs/log_parser.py
+++ b/buildbot_gentoo_ci/logs/log_parser.py
@@ -71,10 +71,11 @@ def search_buildlog(log_search_pattern_list, logfile_text_dict, tmp_index, max_t
# loop true the pattern list for match
for search_pattern in log_search_pattern_list:
search_hit = False
+ ignore_line = False
# check if should ignore the line
#FIXME take the ignore line pattern from db
- if text_line.startswith('>>> /'):
- pass
+ if re.search('^>>> /', text_line):
+ ignore_line = True
#if else re.search('./\w+/'):
# pass
else:
@@ -134,9 +135,9 @@ def search_buildlog(log_search_pattern_list, logfile_text_dict, tmp_index, max_t
summery_dict[i]['status'] = 'info'
summery_dict[i]['id'] = 0
summery_dict[i]['search_pattern'] = 'auto'
- else:
+ if not ignore_line or not search_hit:
# we add all line that start with ' * ' as info
- # we add all line that start with '>>>' but not '>>> /' as info
+ # we add all line that start with '>>>' as info
if text_line.startswith(' * ') or text_line.startswith('>>>'):
if not tmp_index in summery_dict:
summery_dict[tmp_index] = {}
@@ -146,7 +147,7 @@ def search_buildlog(log_search_pattern_list, logfile_text_dict, tmp_index, max_t
summery_dict[tmp_index]['id'] = 0
summery_dict[tmp_index]['search_pattern'] = 'auto'
if summery_dict == {}:
- return None
+ return False
return summery_dict
def getConfigSettings():
@@ -173,7 +174,7 @@ def runLogParser(args):
# run the parse patten on the line
for tmp_index, text in logfile_text_dict.items():
res = mp_pool.apply_async(search_buildlog, (log_search_pattern_list, logfile_text_dict, tmp_index, max_text_lines,))
- if res.get() is not None:
+ if res.get():
print(json.dumps(res.get()))
mp_pool.close()
mp_pool.join()
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2022-06-11 9:55 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-02-03 21:23 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/logs/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2022-06-11 9:55 Magnus Granberg
2022-01-24 0:49 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox