From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 98B96158086 for ; Sat, 8 Jan 2022 17:25:18 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 92EA4E07BA; Sat, 8 Jan 2022 17:25:17 +0000 (UTC) Received: from smtp.gentoo.org (dev.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 593CCE07BA for ; Sat, 8 Jan 2022 17:25:16 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 67A67342C5B for ; Sat, 8 Jan 2022 17:25:15 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id EC0B41BF for ; Sat, 8 Jan 2022 17:25:13 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1641662699.5e9aea3e883a20bc0adedceeeca2767230dd0ea3.zorry@gentoo> Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/ X-VCS-Repository: proj/tinderbox-cluster X-VCS-Files: buildbot_gentoo_ci/steps/builders.py X-VCS-Directories: buildbot_gentoo_ci/steps/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 5e9aea3e883a20bc0adedceeeca2767230dd0ea3 X-VCS-Branch: master Date: Sat, 8 Jan 2022 17:25:13 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 009d4b55-d7dc-40f3-9025-113abec1f495 X-Archives-Hash: 21f8a7f2e286ede40d51fa5da494eeb6 commit: 5e9aea3e883a20bc0adedceeeca2767230dd0ea3 Author: Magnus Granberg gentoo org> AuthorDate: Sat Jan 8 17:24:59 2022 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Sat Jan 8 17:24:59 2022 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5e9aea3e Upload emerge and elogs Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/builders.py | 126 +++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 35 deletions(-) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index e76fe3d..d8314e4 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -170,6 +170,17 @@ def PersOutputOfEmergeInfo(rc, stdout, stderr): 'emerge_info_output' : emerge_info_output } +def PersOutputOfElogLs(rc, stdout, stderr): + elog_ls_output = {} + elog_ls_output['rc'] = rc + elog_ls_list = [] + for line in stdout.split('\n'): + elog_ls_list.append(line) + elog_ls_output['elog_ls'] = elog_ls_list + return { + 'elog_ls_output' : elog_ls_output + } + class TriggerRunBuildRequest(BuildStep): name = 'TriggerRunBuildRequest' @@ -553,6 +564,42 @@ class RunEmerge(BuildStep): yield self.build.addStepsAfterCurrentStep(aftersteps_list) return SUCCESS +class CheckElogLogs(BuildStep): + + name = 'CheckElogLogs' + description = 'Running' + descriptionDone = 'Ran' + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.aftersteps_list = [] + + def addFileUploade(self, sourcefile, destfile): + self.aftersteps_list.append(steps.FileUpload( + workersrc=sourcefile, + masterdest=destfile + )) + + @defer.inlineCallbacks + def run(self): + elog_ls_output = self.getProperty('elog_ls_output') + workdir = yield os.path.join('/', 'var', 'cache', 'portage', 'logs', 'elog') + for elogfile in elog_ls_output['elog_ls']: + if self.getProperty('faild_cpv'): + cpv = self.getProperty('faild_cpv') + else: + cpv = self.getProperty('cpv') + if elogfile.replace(':', '/').startswith(cpv): + print(elogfile) + destfile = yield os.path.join(self.getProperty('masterdest'), elogfile.replace('.log', '.elog')) + sourcefile = yield os.path.join(workdir, elogfile) + self.addFileUploade(sourcefile, destfile) + if self.aftersteps_list != []: + yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) + return SUCCESS + class CheckEmergeLogs(BuildStep): name = 'CheckEmergeLogs' @@ -567,6 +614,7 @@ class CheckEmergeLogs(BuildStep): self.descriptionSuffix = self.step self.aftersteps_list = [] self.log_data = {} + self.faild_cpv = False @defer.inlineCallbacks def getVersionData(self, cpv): @@ -603,16 +651,38 @@ class CheckEmergeLogs(BuildStep): self.addFileUploade(sourcefile, destfile) @defer.inlineCallbacks - def getEmergeFiles(self, faild_version_data): + def getElogFiles(self, cpv): + workdir = yield os.path.join('/', 'var', 'cache', 'portage', 'logs', 'elog') + elog_cpv = cpv.replace('/', ':') + shell_commad_list = [] + shell_commad_list.append('ls') + #shell_commad_list.append(elog_cpv + '*') + self.aftersteps_list.append( + steps.SetPropertyFromCommand( + command=shell_commad_list, + strip=True, + extract_fn=PersOutputOfElogLs, + workdir=workdir, + timeout=None + )) + self.aftersteps_list.append(CheckElogLogs()) + + @defer.inlineCallbacks + def getEmergeFiles(self, cpv): # get emerge info destfile = yield os.path.join(self.masterdest, 'emerge_info.txt') sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt') self.addFileUploade(sourcefile, destfile) - #FIXME: - # if faild_version_data: # get emerge.log - # get elogs + destfile2 = yield os.path.join(self.masterdest, 'emerge.log') + sourcefile2 = yield os.path.join('/', 'var', 'log', 'emerge.log') + self.addFileUploade(sourcefile2, destfile2) # world file + destfile3 = yield os.path.join(self.masterdest, 'world') + sourcefile3 = yield os.path.join('/', 'var', 'lib', 'portage', 'world') + self.addFileUploade(sourcefile3, destfile3) + # get elogs + self.getElogFiles(cpv) def getBuildWorkdirFiles(self): #FIXME: @@ -625,6 +695,8 @@ class CheckEmergeLogs(BuildStep): project_data = self.getProperty('project_data') projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid']) emerge_output = self.getProperty('emerge_output') + self.faild_cpv = emerge_output['failed'] + self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv') shell_commad_list = [ 'emerge', '-v' @@ -632,6 +704,7 @@ class CheckEmergeLogs(BuildStep): package_dict = emerge_output['packages'] self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber"))) + self.setProperty('masterdest', self.masterdest, 'masterdest') #FIXME: Prosees the logs and do stuff # preserved-libs @@ -730,8 +803,8 @@ class CheckEmergeLogs(BuildStep): else: # trigger parse_build_log with info about pre-build and it fail pass - # Check if extra build did work - if self.step == 'extra-build': + # Make Logfile dict + if self.step == 'extra-build' or self.step == 'build': print(emerge_output) log_dict = {} # get cpv, logname and log path @@ -746,11 +819,8 @@ class CheckEmergeLogs(BuildStep): full_logname = full_logname ) print(log_dict) - # Find log for cpv that was requested or did failed - if not log_dict == {}: - # requested cpv - print(log_dict) - faild_cpv = emerge_output['failed'] + if self.step == 'extra-build': + #FIXME: Check if extra build did work self.aftersteps_list.append(RunEmerge(step='pre-build')) self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun') @@ -760,43 +830,29 @@ class CheckEmergeLogs(BuildStep): # local_log_path dir set in config # format /var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz if self.step == 'build': - print(emerge_output) - log_dict = {} - # get cpv, logname and log path - for log_path in emerge_output['log_paths']: - c = log_path.split('/')[6] - full_logname = log_path.split('/')[7] - print(full_logname) - pv = full_logname.split(':')[0] - cpv = c + '/' + pv - log_dict[cpv] = dict( - log_path = log_path, - full_logname = full_logname - ) - print(log_dict) # Find log for cpv that was requested or did failed if not log_dict == {}: # requested cpv - print(log_dict) cpv = self.getProperty('cpv') - faild_cpv = emerge_output['failed'] faild_version_data = False - if cpv in log_dict or faild_cpv in log_dict: + if cpv in log_dict or self.faild_cpv in log_dict: yield self.createDistDir() if cpv in log_dict: self.log_data[cpv] = log_dict[cpv] yield self.getLogFile(cpv, log_dict) - if faild_cpv: + if self.faild_cpv: # failed and build requested cpv - if cpv == faild_cpv: + if cpv == self.faild_cpv: faild_version_data = self.getProperty("version_data") else: # failed but not build requested cpv - self.log_data[faild_cpv] = log_dict[faild_cpv] - yield self.getLogFile(faild_cpv, log_dict) - faild_version_data = yield self.getVersionData(faild_cpv) + self.log_data[self.faild_cpv] = log_dict[self.faild_cpv] + yield self.getLogFile(self.faild_cpv, log_dict) + faild_version_data = yield self.getVersionData(self.faild_cpv) + self.getEmergeFiles(self.faild_cpv) self.getBuildWorkdirFiles() - self.getEmergeFiles(faild_version_data) + else: + self.getEmergeFiles(cpv) self.aftersteps_list.append(steps.Trigger( schedulerNames=['parse_build_log'], waitForFinish=False, @@ -808,7 +864,7 @@ class CheckEmergeLogs(BuildStep): 'log_build_data' : self.log_data, 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"), 'repository_data' : self.getProperty('repository_data'), - 'faild_cpv' : faild_cpv, + 'faild_cpv' : self.faild_cpv, 'step' : self.step, 'build_workername' : self.getProperty('workername') }