From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 49C46158094 for ; Fri, 5 Aug 2022 18:32:30 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 26608E09D5; Fri, 5 Aug 2022 18:32:29 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id DEA06E09F7 for ; Fri, 5 Aug 2022 18:32:28 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id D3C82341124 for ; Fri, 5 Aug 2022 18:32:27 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 1AEBF543 for ; Fri, 5 Aug 2022 18:32:26 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1659724331.3791e957fd2bb28f350f0ff7399c0c6e43b561d6.zorry@gentoo> Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/ X-VCS-Repository: proj/tinderbox-cluster X-VCS-Files: buildbot_gentoo_ci/steps/builders.py X-VCS-Directories: buildbot_gentoo_ci/steps/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 3791e957fd2bb28f350f0ff7399c0c6e43b561d6 X-VCS-Branch: master Date: Fri, 5 Aug 2022 18:32:26 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 8e6ee392-1e54-4043-8f41-e69473fe6305 X-Archives-Hash: 74cac264280384127d9fcb3d62582344 commit: 3791e957fd2bb28f350f0ff7399c0c6e43b561d6 Author: Magnus Granberg gentoo org> AuthorDate: Fri Aug 5 18:32:11 2022 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Fri Aug 5 18:32:11 2022 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3791e957 Add haskell-updater for haskell packages Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/steps/builders.py | 147 ++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 56 deletions(-) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index a54c68c..3b6a8d0 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -4,6 +4,7 @@ import os import re import json +from pathlib import Path from portage.versions import catpkgsplit, cpv_getversion from portage.dep import dep_getcpv, dep_getslot, dep_getrepo @@ -90,6 +91,8 @@ def PersOutputOfEmerge(rc, stdout, stderr): # split the lines #FIXME: Handling of stderr output stderr_line_list = [] + if stderr != '' and not emerge_output['failed']: + emerge_output['failed'] = True for line in stderr.split('\n'): if 'Change USE:' in line: line_list = line.split(' ') @@ -340,6 +343,7 @@ class SetupPropertys(BuildStep): print(self.getProperty("project_build_data")) self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber"))) self.setProperty('masterdest', self.masterdest, 'masterdest') + self.setProperty('build_env', {}, 'build_env') self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for project', self.getProperty('project_data')['name']]) return SUCCESS @@ -355,7 +359,6 @@ class RunEmerge(BuildStep): super().__init__(**kwargs) self.descriptionSuffix = self.step self.name = 'Setup emerge for ' + self.step + ' step' - self.build_env = {} self.build_timeout = 0 @defer.inlineCallbacks @@ -374,25 +377,28 @@ class RunEmerge(BuildStep): aftersteps_list = [] #FIXME: Set build timeout in config self.build_timeout = 6600 - # set env - # https://bugs.gentoo.org/683118 - # export TERM=linux - # export TERMINFO=/etc/terminfo - self.build_env['TERM'] = 'linux' - self.build_env['TERMINFO'] = '/etc/terminfo' - # Lang - self.build_env['LANG'] = 'C.utf8' - self.build_env['LC_MESSAGES'] = 'C' - # no color - self.build_env['CARGO_TERM_COLOR'] = 'never' - self.build_env['GCC_COLORS'] = '0' - self.build_env['OCAML_COLOR'] = 'never' - self.build_env['PY_FORCE_COLOR'] = '0' - self.build_env['PYTEST_ADDOPTS'] = '--color=no' - self.build_env['NO_COLOR'] = '1' - # not all terms support urls - self.build_env['GCC_URLS'] = 'no' - self.build_env['TERM_URLS'] = 'no' + if self.step == 'update': + # set env + build_env = {} + # https://bugs.gentoo.org/683118 + # export TERM=linux + # export TERMINFO=/etc/terminfo + build_env['TERM'] = 'linux' + build_env['TERMINFO'] = '/etc/terminfo' + # Lang + build_env['LANG'] = 'C.utf8' + build_env['LC_MESSAGES'] = 'C' + # no color + build_env['CARGO_TERM_COLOR'] = 'never' + build_env['GCC_COLORS'] = '0' + build_env['OCAML_COLOR'] = 'never' + build_env['PY_FORCE_COLOR'] = '0' + build_env['PYTEST_ADDOPTS'] = '--color=no' + build_env['NO_COLOR'] = '1' + # not all terms support urls + build_env['GCC_URLS'] = 'no' + build_env['TERM_URLS'] = 'no' + self.setProperty('build_env', build_env, 'build_env') if self.step == 'pre-update': shell_commad_list.append('-uDN') @@ -432,6 +438,7 @@ class RunEmerge(BuildStep): strip=True, extract_fn=PersOutputOfEmerge, workdir='/', + env=self.getProperty("build_env"), timeout=self.build_timeout )) aftersteps_list.append(CheckEmergeLogs('update')) @@ -548,7 +555,7 @@ class RunEmerge(BuildStep): strip=True, extract_fn=PersOutputOfEmerge, workdir='/', - env=self.build_env, + env=self.getProperty("build_env"), timeout=self.build_timeout )) aftersteps_list.append(CheckEmergeLogs('build')) @@ -682,11 +689,9 @@ class CheckEmergeLogs(BuildStep): @defer.inlineCallbacks def createDistDir(self): workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername')) - self.aftersteps_list.append(steps.MasterShellCommand( - name = 'Make directory for Uploaded files', - command = ['mkdir', str(self.getProperty("buildnumber"))], - workdir = workdir - )) + check_dir = yield os.path.join(workdir, str(self.getProperty("buildnumber"))) + if not Path(check_dir).is_dir(): + yield Path(check_dir).mkdir(parents=True) def addFileUploade(self, sourcefile, destfile, name, url, urlText): self.aftersteps_list.append(steps.FileUpload( @@ -814,7 +819,7 @@ class CheckEmergeLogs(BuildStep): print(emerge_output) # this should be set in the config retrays = 5 - if self.getProperty('rerun') <= retrays: + if self.getProperty('rerun') <= retrays and self.faild_cpv: # when we need to change use. we could rerun pre-build with # --autounmask-use=y --autounmask-write=y --autounmask-only=y # but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS @@ -895,7 +900,9 @@ class CheckEmergeLogs(BuildStep): self.aftersteps_list.append(CheckEmergeLogs('extra-build')) else: # trigger parse_build_log with info about pre-build and it fail - pass + if self.faild_cpv: + pass + self.setProperty('rerun', 0, 'rerun') # Make Logfile dict if self.step == 'extra-build' or self.step == 'build': print(emerge_output) @@ -922,7 +929,9 @@ class CheckEmergeLogs(BuildStep): # trigger a logparser # local_log_path dir set in config # format /var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz + rebuild = False if self.step == 'build': + retrays = 1 # Find log for cpv that was requested or did failed if not log_dict == {}: # requested cpv @@ -934,36 +943,62 @@ class CheckEmergeLogs(BuildStep): self.log_data[cpv] = log_dict[cpv] yield self.getLogFile(cpv, log_dict) if self.faild_cpv: - # failed and build requested cpv - if cpv == self.faild_cpv: - faild_version_data = self.getProperty("version_data") - else: - # failed but not build requested cpv - self.log_data[self.faild_cpv] = log_dict[self.faild_cpv] - yield self.getLogFile(self.faild_cpv, log_dict) - faild_version_data = yield self.getVersionData(self.faild_cpv) - self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv') - self.getEmergeFiles(self.faild_cpv) - self.getBuildWorkDirs(self.faild_cpv) + c = yield catpkgsplit(self.faild_cpv)[0] + if c == 'dev-haskell': + rebuild = 'haskell' + if not rebuild or self.getProperty('rerun') >= retrays: + # failed and build requested cpv + if cpv == self.faild_cpv: + faild_version_data = self.getProperty("version_data") + else: + # failed but not build requested cpv + self.log_data[self.faild_cpv] = log_dict[self.faild_cpv] + yield self.getLogFile(self.faild_cpv, log_dict) + faild_version_data = yield self.getVersionData(self.faild_cpv) + self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv') + self.getEmergeFiles(self.faild_cpv) + self.getBuildWorkDirs(self.faild_cpv) + rebuild = False else: self.getEmergeFiles(cpv) - self.aftersteps_list.append(steps.Trigger( - name = 'Setup properties for log parser and trigger it', - schedulerNames=['parse_build_log'], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - 'cpv' : self.getProperty("cpv"), - 'faild_version_data' : faild_version_data, - 'project_build_data' : self.getProperty('project_build_data'), - 'log_build_data' : self.log_data, - 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"), - 'repository_data' : self.getProperty('repository_data'), - 'faild_cpv' : self.faild_cpv, - 'step' : self.step, - 'build_workername' : self.getProperty('workername') - } - )) + if not rebuild: + self.aftersteps_list.append(steps.Trigger( + name = 'Setup properties for log parser and trigger it', + schedulerNames=['parse_build_log'], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + 'cpv' : self.getProperty("cpv"), + 'faild_version_data' : faild_version_data, + 'project_build_data' : self.getProperty('project_build_data'), + 'log_build_data' : self.log_data, + 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"), + 'repository_data' : self.getProperty('repository_data'), + 'faild_cpv' : self.faild_cpv, + 'step' : self.step, + 'build_workername' : self.getProperty('workername') + } + )) + if rebuild: + #FIXME: Set build timeout in config + build_timeout = 6600 + shell_commad_list = [] + # rebuild broken haskell + if rebuild == 'haskell': + shell_commad_list.append('haskell-updater') + shell_commad_list.append('--') + shell_commad_list.append('--usepkg=n') + if shell_commad_list != []: + self.aftersteps_list.append( + steps.ShellCommand( + command=shell_commad_list, + workdir='/', + env=self.getProperty("build_env"), + timeout=build_timeout + )) + self.aftersteps_list.append(RunEmerge(step='build')) + self.aftersteps_list.append(CheckEmergeLogs('build')) + self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun') if not self.step is None and self.aftersteps_list != []: yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS