From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 05BA1158041 for ; Sun, 31 Mar 2024 19:41:01 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 46D292BC013; Sun, 31 Mar 2024 19:41:00 +0000 (UTC) Received: from smtp.gentoo.org (mail.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 2FE652BC013 for ; Sun, 31 Mar 2024 19:41:00 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 6C45F34319D for ; Sun, 31 Mar 2024 19:40:59 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id F03F51618 for ; Sun, 31 Mar 2024 19:40:57 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1711914033.e8e5b67dc70b8b2b0222486574a20d6edb0577c2.zorry@gentoo> Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/ X-VCS-Repository: proj/tinderbox-cluster X-VCS-Files: buildbot_gentoo_ci/steps/logs.py X-VCS-Directories: buildbot_gentoo_ci/steps/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: e8e5b67dc70b8b2b0222486574a20d6edb0577c2 X-VCS-Branch: master Date: Sun, 31 Mar 2024 19:40:57 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: fa5c5555-8677-4de6-b9a7-af00496259df X-Archives-Hash: af534aa0f87d8383d9aaff873bb98f65 commit: e8e5b67dc70b8b2b0222486574a20d6edb0577c2 Author: Magnus Granberg gentoo org> AuthorDate: Sun Mar 31 19:40:33 2024 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Sun Mar 31 19:40:33 2024 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e8e5b67d Add step logs to a log file Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/logs.py | 53 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index 896778c..6b8eba5 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -7,6 +7,7 @@ import gzip import io import hashlib import json +import lzma from portage.versions import catpkgsplit, cpv_getversion @@ -74,6 +75,18 @@ def PersOutputOfEmergeInfo(rc, stdout, stderr): 'emerge_info_output' : emerge_info_output } +@defer.inlineCallbacks +def WriteTextToFile(path, text_list, separator=False): + separator2 = '\n' + if separator: + text_string = separator2.join(text_list) + else: + text_string = text_list + text_string = text_string + separator2 + with lzma.open(path,"wt") as f: + yield f.write(text_string) + yield f.close + class SetupPropertys(BuildStep): name = 'SetupPropertys' @@ -109,14 +122,16 @@ class SetupPropertys(BuildStep): # get steps/log id's from build build_log_steps_data = {} steps_data = yield self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id']) - print(steps_data) for step in steps_data: + log_data = None logs_data = yield self.master.db.logs.getLogs(step['id']) - print(logs_data) + for log_info in logs_data: + if log_info['name'] != 'property changes': + log_data = log_info step_info = {} step_info['name'] = step['name'] step_info['number'] = step['number'] - step_info['log_data'] = logs_data + step_info['log_data'] = log_data build_log_steps_data[step['id']] = step_info self.setProperty("build_log_steps_data", build_log_steps_data, 'build_log_steps_data') return SUCCESS @@ -509,6 +524,38 @@ class ParserPkgCheckLog(BuildStep): returnstatus = WARNINGS return returnstatus +class writeStepLogsToFile(BuildStep): + + name = 'writeStepLogsToFile' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = False + flunkOnFailure = True + warnOnWarnings = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + buildsteplogfile_path = yield os.path.join(self.getProperty('logsdir'), 'buildsteps.log.xz') + stepstologlist = ['Run emerge step build', 'Run emerge step build_1'] + logs_texts = [] + for k, v in self.getProperty('build_log_steps_data').items(): + if v['name'] in stepstologlist: + logs_texts.append(f">>> BEGINING OF STEP: {v['name']}") + logs_texts.append('') + log_data = v['log_data'] + log_text = yield self.master.db.logs.getLogLines(log_data['id'], 1, log_data['num_lines']) + for line in log_text.split('\n'): + if line.startswith('h BUILDMASTER='): + line = 'h BUILDMASTER=XXX.XXX.XXX.XXX' + logs_texts.append(line[1:]) + logs_texts.append(f">>> END OF STEP: {v['name']}") + yield WriteTextToFile(buildsteplogfile_path, logs_texts, separator=True) + return SUCCESS + class SetupBugReportSteps(BuildStep): name = 'SetupBugReportSteps' description = 'Running'