* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-27 11:02 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-27 11:02 UTC (permalink / raw
To: gentoo-commits
commit: 843b4272bf91ac8cf135664cab48c515f23d6cad
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 26 12:47:11 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jul 26 12:47:11 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=843b4272
Use rootworkdir
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py
index a77c878..3e15e5f 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -76,6 +76,7 @@ class SetupStage4Steps(BuildStep):
workerdest = yield os.path.join(self.getProperty("workerbase"), self.getProperty('project_uuid'))
workerdest_etc = yield os.path.join(workerdest, 'etc')
print(workerdest_etc)
+ self.setProperty('rootworkdir', workerdest, 'rootworkdir')
# create dir
aftersteps_list.append(steps.ShellCommand(
flunkOnFailure=True,
@@ -86,9 +87,9 @@ class SetupStage4Steps(BuildStep):
# download stage3
aftersteps_list.append(GetSteg3())
# setup portage
- aftersteps_list.append(builders_steps.UpdateRepos(workdir=workerdest))
- aftersteps_list.append(portage_steps.SetReposConf(workdir=workerdest))
- aftersteps_list.append(portage_steps.SetMakeConf(workdir=workerdest))
+ aftersteps_list.append(builders_steps.UpdateRepos())
+ aftersteps_list.append(portage_steps.SetReposConf())
+ aftersteps_list.append(portage_steps.SetMakeConf())
# add localegen
#FIXME: set that in config
locale_conf = []
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-10-16 20:50 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-10-16 20:50 UTC (permalink / raw
To: gentoo-commits
commit: be93016aa5c8743c88fa3cc2c6c92fc7a9d53593
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 16 20:50:04 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Oct 16 20:50:04 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=be93016a
Use stderr_failed when stderr fail in build
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 1db8cc2..7123cf4 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -30,6 +30,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['circular_deps'] = False
emerge_output['masked'] = False
emerge_output['failed'] = False
+ emerge_output['stderr_failed'] = False
package_dict = {}
change_use = {}
log_path_list = []
@@ -92,8 +93,8 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
- if stderr != '' and not emerge_output['failed']:
- emerge_output['failed'] = True
+ if stderr != '':
+ emerge_output['stderr_failed'] = True
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -360,8 +361,9 @@ class SetupPropertys(BuildStep):
project_build_data['buildbot_build_id'])
project_build_data['builder_id'] = builder_id
project_build_data['buildnumber'] = self.getProperty("buildnumber")
+ project_build_data['bug_id'] = 0
self.setProperty('project_build_data', project_build_data, 'project_build_data')
- print(self.getProperty("project_build_data"))
+ print(f"Project_build_data: {project_build_data}")
self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
self.setProperty('masterdest', self.masterdest, 'masterdest')
self.setProperty('build_env', {}, 'build_env')
@@ -919,7 +921,7 @@ class CheckEmergeLogs(BuildStep):
print(emerge_output)
# this should be set in the config
retrays = 5
- if self.getProperty('rerun') <= retrays and emerge_output['failed']:
+ if self.getProperty('rerun') <= retrays and emerge_output['stderr_failed']:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
# but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-03-31 19:40 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-03-31 19:40 UTC (permalink / raw
To: gentoo-commits
commit: e8e5b67dc70b8b2b0222486574a20d6edb0577c2
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 31 19:40:33 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Mar 31 19:40:33 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e8e5b67d
Add step logs to a log file
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 53 +++++++++++++++++++++++++++++++++++++---
1 file changed, 50 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 896778c..6b8eba5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -7,6 +7,7 @@ import gzip
import io
import hashlib
import json
+import lzma
from portage.versions import catpkgsplit, cpv_getversion
@@ -74,6 +75,18 @@ def PersOutputOfEmergeInfo(rc, stdout, stderr):
'emerge_info_output' : emerge_info_output
}
+@defer.inlineCallbacks
+def WriteTextToFile(path, text_list, separator=False):
+ separator2 = '\n'
+ if separator:
+ text_string = separator2.join(text_list)
+ else:
+ text_string = text_list
+ text_string = text_string + separator2
+ with lzma.open(path,"wt") as f:
+ yield f.write(text_string)
+ yield f.close
+
class SetupPropertys(BuildStep):
name = 'SetupPropertys'
@@ -109,14 +122,16 @@ class SetupPropertys(BuildStep):
# get steps/log id's from build
build_log_steps_data = {}
steps_data = yield self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
- print(steps_data)
for step in steps_data:
+ log_data = None
logs_data = yield self.master.db.logs.getLogs(step['id'])
- print(logs_data)
+ for log_info in logs_data:
+ if log_info['name'] != 'property changes':
+ log_data = log_info
step_info = {}
step_info['name'] = step['name']
step_info['number'] = step['number']
- step_info['log_data'] = logs_data
+ step_info['log_data'] = log_data
build_log_steps_data[step['id']] = step_info
self.setProperty("build_log_steps_data", build_log_steps_data, 'build_log_steps_data')
return SUCCESS
@@ -509,6 +524,38 @@ class ParserPkgCheckLog(BuildStep):
returnstatus = WARNINGS
return returnstatus
+class writeStepLogsToFile(BuildStep):
+
+ name = 'writeStepLogsToFile'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = False
+ flunkOnFailure = True
+ warnOnWarnings = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ buildsteplogfile_path = yield os.path.join(self.getProperty('logsdir'), 'buildsteps.log.xz')
+ stepstologlist = ['Run emerge step build', 'Run emerge step build_1']
+ logs_texts = []
+ for k, v in self.getProperty('build_log_steps_data').items():
+ if v['name'] in stepstologlist:
+ logs_texts.append(f">>> BEGINING OF STEP: {v['name']}")
+ logs_texts.append('')
+ log_data = v['log_data']
+ log_text = yield self.master.db.logs.getLogLines(log_data['id'], 1, log_data['num_lines'])
+ for line in log_text.split('\n'):
+ if line.startswith('h BUILDMASTER='):
+ line = 'h BUILDMASTER=XXX.XXX.XXX.XXX'
+ logs_texts.append(line[1:])
+ logs_texts.append(f">>> END OF STEP: {v['name']}")
+ yield WriteTextToFile(buildsteplogfile_path, logs_texts, separator=True)
+ return SUCCESS
+
class SetupBugReportSteps(BuildStep):
name = 'SetupBugReportSteps'
description = 'Running'
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-03-31 19:40 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-03-31 19:40 UTC (permalink / raw
To: gentoo-commits
commit: 4707a4cb41619d8c35d2d9f7cb0cc5412f447492
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 31 19:38:55 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Mar 31 19:38:55 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4707a4cb
Fix a typo on faild_dep
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 53f79ab..1db8cc2 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1084,7 +1084,7 @@ class CheckEmergeLogs(BuildStep):
'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
'repository_data' : self.getProperty('repository_data'),
'faild_cpv' : emerge_output['failed'],
- 'faild_dep' : self.faild_dep
+ 'faild_dep' : self.faild_dep,
'build_workername' : self.getProperty('workername')
}
))
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-03-23 14:13 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-03-23 14:13 UTC (permalink / raw
To: gentoo-commits
commit: 16f9c0fc440bb4c15742b7e02827fad64d73db82
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 23 14:13:36 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Mar 23 14:13:36 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=16f9c0fc
Redo Trigger for parse_build_log in CheckEmergeLogs
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 180 +++++++++++++++++------------------
1 file changed, 87 insertions(+), 93 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index c033196..73e82f5 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -895,8 +895,10 @@ class CheckEmergeLogs(BuildStep):
'-v'
]
emerge_output = self.getProperty('emerge_output')
- self.faild_cpv = emerge_output['failed']
+ print(f"emerge_output: {emerge_output}")
+ self.setProperty('faild_cpv', emerge_output['failed'], 'faild_cpv')
self.faild_version_data = False
+ self.log_data = {}
package_dict = emerge_output['packages']
#FIXME: Prosees the logs and do stuff
@@ -916,7 +918,7 @@ class CheckEmergeLogs(BuildStep):
print(emerge_output)
# this should be set in the config
retrays = 5
- if self.getProperty('rerun') <= retrays and self.faild_cpv:
+ if self.getProperty('rerun') <= retrays and emerge_output['failed']:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
# but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
@@ -997,26 +999,24 @@ class CheckEmergeLogs(BuildStep):
self.aftersteps_list.append(CheckEmergeLogs('extra-build'))
else:
# trigger parse_build_log with info about pre-build and it fail
- if self.faild_cpv:
+ if emerge_output['failed']:
pass
self.setProperty('rerun', 0, 'rerun')
# Make Logfile dict
if self.step == 'extra-build' or self.step == 'build':
- print(emerge_output)
log_dict = {}
# get cpv, logname and log path
for log_path in emerge_output['log_paths']:
c = log_path.split('/')[6]
full_logname = log_path.split('/')[7]
- print(full_logname)
+ print(f"full_logname: {full_logname}")
pv = full_logname.split(':')[0]
cpv = c + '/' + pv
log_dict[cpv] = dict(
log_path = log_path,
full_logname = full_logname
)
- print('log_dict')
- print(log_dict)
+ print(f"log_dict: {log_dict}")
if self.step == 'extra-build':
#FIXME: Check if extra build did work
self.aftersteps_list.append(RunEmerge(step='pre-build'))
@@ -1028,94 +1028,88 @@ class CheckEmergeLogs(BuildStep):
# local_log_path dir set in config
# format /var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz
rebuild = False
- if self.step == 'build':
+ if self.step == 'build' and log_dict != {}:
retrays = 1
- # Find log for cpv that was requested or did failed
- if not log_dict == {}:
- print('log_dict not emty')
- # requested cpv
- cpv = self.getProperty('cpv')
- if cpv in log_dict or self.faild_cpv in log_dict:
- print(cpv)
- print(self.faild_cpv)
+ if isinstance(emerge_output['failed'], str):
+ log_cpv = emerge_output['failed']
+ if self.getProperty('rerun') < retrays:
+ # set rebuild
+ # haskell
+ if c == 'dev-haskell':
+ rebuild = 'haskell'
+ # binary
+ for cpv, v in package_dict.items():
+ if cpv == emerge_output['failed']:
+ if v['binary']:
+ rebuild = 'binary'
+ print(f"rebuild is: {rebuild}")
+ else:
+ log_cpv = self.getProperty('cpv')
+ if not rebuild:
+ if log_cpv in log_dict:
+ print(f"Package {log_cpv} in log_dict")
+ self.log_data[log_cpv] = log_dict[log_cpv]
yield self.createDistDir()
- if cpv in log_dict:
- self.log_data[cpv] = log_dict[cpv]
- yield self.getLogFile(cpv, log_dict)
- if isinstance(self.faild_cpv, str):
- print('faild_cpv is string')
- c = yield catpkgsplit(self.faild_cpv)[0]
- if c == 'dev-haskell':
- rebuild = 'haskell'
- for cpv, v in package_dict.items():
- if cpv == self.faild_cpv:
- if v['binary']:
- rebuild = 'binary'
- if not rebuild or self.getProperty('rerun') >= retrays:
- # failed and build requested cpv
- if cpv == self.faild_cpv:
- self.faild_version_data = self.getProperty("version_data")
- else:
- # failed but not build requested cpv
- self.log_data[self.faild_cpv] = log_dict[self.faild_cpv]
- yield self.getLogFile(self.faild_cpv, log_dict)
- repository = False
- for cpv, v in package_dict.items():
- if cpv == self.faild_cpv:
- repository = v['repository']
- if repository:
- self.faild_version_data = yield self.getVersionData(self.faild_cpv, repository)
- self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
- print(self.faild_version_data)
- print('getEmergeFiles')
- self.getEmergeFiles(self.faild_cpv)
- self.getBuildWorkDirs(self.faild_cpv)
- rebuild = False
- else:
- print('getEmergeFiles')
- self.getEmergeFiles(cpv)
- if not rebuild:
- self.aftersteps_list.append(steps.Trigger(
- name = 'Setup properties for log parser and trigger it',
- schedulerNames=['parse_build_log'],
- waitForFinish=False,
- updateSourceStamp=False,
- set_properties={
- 'cpv' : self.getProperty("cpv"),
- 'faild_version_data' : self.faild_version_data,
- 'project_build_data' : self.getProperty('project_build_data'),
- 'log_build_data' : self.log_data,
- 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
- 'repository_data' : self.getProperty('repository_data'),
- 'faild_cpv' : self.faild_cpv,
- 'build_workername' : self.getProperty('workername')
- }
- ))
- if rebuild:
- #FIXME: Set build timeout in config
- build_timeout = 6600
- shell_commad_list = []
- exclude = False
- # rebuild broken haskell
- if rebuild == 'haskell':
- shell_commad_list.append('haskell-updater')
- shell_commad_list.append('--')
- shell_commad_list.append('--usepkg=n')
- if rebuild == 'binary':
- c = yield catpkgsplit(self.faild_cpv)[0]
- p = yield catpkgsplit(self.faild_cpv)[1]
- exclude = c + '/' + p
- if shell_commad_list != []:
- self.aftersteps_list.append(
- steps.ShellCommand(
- command=shell_commad_list,
- workdir='/',
- env=self.getProperty("build_env"),
- timeout=build_timeout
- ))
- self.aftersteps_list.append(RunEmerge(step='build', exclude=exclude))
- self.aftersteps_list.append(CheckEmergeLogs('build'))
- self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
+ yield self.getLogFile(log_cpv, log_dict)
+ yield self.getEmergeFiles(log_cpv)
+ if isinstance(emerge_output['failed'], str):
+ # get extra logs
+ self.getBuildWorkDirs(emerge_output['failed'])
+ #set faild_version_data
+ if self.getProperty('cpv') == emerge_output['failed']:
+ self.faild_version_data = self.getProperty("version_data")
+ else:
+ repository = False
+ for cpv, v in package_dict.items():
+ if cpv == emerge_output['failed']:
+ repository = v['repository']
+ if repository:
+ self.faild_version_data = yield self.getVersionData(emerge_output['failed'], repository)
+ print(f"Failed package version data: {self.faild_version_data}")
+ else:
+ print(f"No log file in log_dict match package: {log_cpv}")
+ return WARNINGS
+ self.aftersteps_list.append(steps.Trigger(
+ name = 'Setup properties for log parser and trigger it',
+ schedulerNames=['parse_build_log'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : self.getProperty("cpv"),
+ 'faild_version_data' : self.faild_version_data,
+ 'project_build_data' : self.getProperty('project_build_data'),
+ 'log_build_data' : self.log_data,
+ 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
+ 'repository_data' : self.getProperty('repository_data'),
+ 'faild_cpv' : emerge_output['failed'],
+ 'build_workername' : self.getProperty('workername')
+ }
+ ))
+ if rebuild:
+ #FIXME: Set build timeout in config
+ build_timeout = 6600
+ shell_commad_list = []
+ exclude = False
+ # rebuild broken haskell
+ if rebuild == 'haskell':
+ shell_commad_list.append('haskell-updater')
+ shell_commad_list.append('--')
+ shell_commad_list.append('--usepkg=n')
+ if rebuild == 'binary':
+ c = yield catpkgsplit(emerge_output['failed'])[0]
+ p = yield catpkgsplit(emerge_output['failed'])[1]
+ exclude = c + '/' + p
+ if shell_commad_list != []:
+ self.aftersteps_list.append(
+ steps.ShellCommand(
+ command=shell_commad_list,
+ workdir='/',
+ env=self.getProperty("build_env"),
+ timeout=build_timeout
+ ))
+ self.aftersteps_list.append(RunEmerge(step='build', exclude=exclude))
+ self.aftersteps_list.append(CheckEmergeLogs('build'))
+ self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
if not self.step is None and self.aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-03-21 16:56 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-03-21 16:56 UTC (permalink / raw
To: gentoo-commits
commit: 6171182755e1b643be9205edae0a81d284bdab99
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 21 16:56:34 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Mar 21 16:56:34 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=61711827
Add bb log id for build steps
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 9 +++++++--
buildbot_gentoo_ci/steps/logs.py | 15 ++++++++++++++-
2 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 0ac8526..c033196 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -347,13 +347,19 @@ class SetupPropertys(BuildStep):
self.setProperty('rerun', 0, 'rerun')
project_build_data = self.getProperty('project_build_data')
project_build_data['status'] = 'in-progress'
- project_build_data['buildbot_build_id'] = self.getProperty("buildnumber")
yield self.gentooci.db.builds.setStatusBuilds(
project_build_data['id'],
project_build_data['status'])
+ builder_id = yield self.master.db.builders.findBuilderId(self.getProperty("buildername"), autoCreate=False)
+ print(builder_id)
+ build_data = yield self.master.db.builds.getBuildByNumber(builder_id ,self.getProperty("buildnumber"))
+ print(build_data)
+ project_build_data['buildbot_build_id'] = build_data['id']
yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
project_build_data['id'],
project_build_data['buildbot_build_id'])
+ project_build_data['builder_id'] = builder_id
+ project_build_data['buildnumber'] = self.getProperty("buildnumber")
self.setProperty('project_build_data', project_build_data, 'project_build_data')
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
@@ -1082,7 +1088,6 @@ class CheckEmergeLogs(BuildStep):
'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
'repository_data' : self.getProperty('repository_data'),
'faild_cpv' : self.faild_cpv,
- 'step' : self.step,
'build_workername' : self.getProperty('workername')
}
))
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index b6c97b7..896778c 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -104,8 +104,21 @@ class SetupPropertys(BuildStep):
self.setProperty("log_cpv", log_cpv, 'log_cpv')
self.setProperty("bgo", False, 'bgo')
self.descriptionDone = 'Runing log checker on ' + log_cpv
- logsdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
+ logsdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildnumber']))
self.setProperty("logsdir", logsdir, 'logsdir')
+ # get steps/log id's from build
+ build_log_steps_data = {}
+ steps_data = yield self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
+ print(steps_data)
+ for step in steps_data:
+ logs_data = yield self.master.db.logs.getLogs(step['id'])
+ print(logs_data)
+ step_info = {}
+ step_info['name'] = step['name']
+ step_info['number'] = step['number']
+ step_info['log_data'] = logs_data
+ build_log_steps_data[step['id']] = step_info
+ self.setProperty("build_log_steps_data", build_log_steps_data, 'build_log_steps_data')
return SUCCESS
class SetupParserBuildLoger(BuildStep):
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-03-19 17:19 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-03-19 17:19 UTC (permalink / raw
To: gentoo-commits
commit: 33713a909627a0378845d9d0be4f689c3857667b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Mar 19 17:18:49 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Mar 19 17:18:49 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=33713a90
Add compile to word_list as Ago use it as default
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 64bd3e7..b6c97b7 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -210,6 +210,8 @@ class MakeIssue(BuildStep):
cp = c + '/' + p
word_list.append(cp)
word_list.append(cpv_getversion(self.error_dict['cpv']))
+ # add compile as Ago use that as default
+ word_list.append('compile')
for word in self.error_dict['title_phase'].split(' '):
word_list.append(word)
for word in self.error_dict['title_issue_nice'].split(' '):
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-03-17 20:40 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-03-17 20:40 UTC (permalink / raw
To: gentoo-commits
commit: 6ca6f7fb23f0d63bbb53a457c1bd149a9cd40795
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 17 20:40:25 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Mar 17 20:40:25 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6ca6f7fb
Add more support for change use, rebuild faild bin and clean some --change-* emerge options
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 49 ++++++++++++++++++++++++++++--------
1 file changed, 39 insertions(+), 10 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 09739d7..0ac8526 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -31,6 +31,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['masked'] = False
emerge_output['failed'] = False
package_dict = {}
+ change_use = {}
log_path_list = []
print('stderr')
print(stderr)
@@ -99,7 +100,6 @@ def PersOutputOfEmerge(rc, stdout, stderr):
change_use_list = []
# get cpv
cpv_split = line_list[1].split(':')
- change_use = {}
# add use flags
if line_list[4].startswith('+') or line_list[4].startswith('-'):
# we only support tre for now
@@ -120,7 +120,21 @@ def PersOutputOfEmerge(rc, stdout, stderr):
change_use_list = False
if change_use_list:
change_use[cpv_split[0]] = change_use_list
- emerge_output['change_use'] = change_use
+ if 'USE changes are' in line:
+ change_use_list = []
+ change_use = {}
+ for text in stderr.split('\n'):
+ if text.startswith('>='):
+ line_list = text.split(' ')
+ for line2 in text.split(' '):
+ if line2.startswith('>='):
+ cpv = line2.split(':')[0].replace('>=', '')
+ else:
+ change_use_list.append(line2)
+ if change_use_list:
+ change_use[cpv] = change_use_list
+ if change_use != {}:
+ emerge_output['change_use'] = change_use
if line.startswith(' * '):
if line.endswith('.log'):
log_path = line.split(' ')[3]
@@ -355,8 +369,9 @@ class RunEmerge(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, step=None, **kwargs):
+ def __init__(self, step=None, exclude=False, **kwargs):
self.step = step
+ self.exclude = exclude
super().__init__(**kwargs)
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
@@ -367,6 +382,7 @@ class RunEmerge(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+ #FIXME: Use settings from gentoo-ci-settings json file
self.stepname = 'Run emerge ' + 'step ' + self.step
shell_commad_list = [
'emerge',
@@ -402,9 +418,9 @@ class RunEmerge(BuildStep):
self.setProperty('build_env', build_env, 'build_env')
if self.step == 'pre-update':
- shell_commad_list.append('-uDN')
- shell_commad_list.append('--changed-deps')
- shell_commad_list.append('--changed-use')
+ shell_commad_list.append('--update')
+ shell_commad_list.append('--deep')
+ shell_commad_list.append('--newuse')
shell_commad_list.append('--pretend')
shell_commad_list.append('@world')
# don't build bin for virtual and acct-*
@@ -423,9 +439,9 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckEmergeLogs('pre-update'))
if self.step == 'update':
- shell_commad_list.append('-uDNq')
- shell_commad_list.append('--changed-deps')
- shell_commad_list.append('--changed-use')
+ shell_commad_list.append('--update')
+ shell_commad_list.append('--deep')
+ shell_commad_list.append('--newuse')
shell_commad_list.append('@world')
# don't build bin for virtual and acct-*
shell_commad_list.append('--buildpkg-exclude')
@@ -600,6 +616,10 @@ class RunEmerge(BuildStep):
# we don't use the bin for the requsted cpv
shell_commad_list.append('--usepkg-exclude')
shell_commad_list.append(cp)
+ if self.exclude:
+ # we don't use the bin for the exclude cpv
+ shell_commad_list.append('--usepkg-exclude')
+ shell_commad_list.append(self.exclude)
# don't build bin for virtual and acct-*
shell_commad_list.append('--buildpkg-exclude')
shell_commad_list.append('virtual')
@@ -1021,6 +1041,10 @@ class CheckEmergeLogs(BuildStep):
c = yield catpkgsplit(self.faild_cpv)[0]
if c == 'dev-haskell':
rebuild = 'haskell'
+ for cpv, v in package_dict.items():
+ if cpv == self.faild_cpv:
+ if v['binary']:
+ rebuild = 'binary'
if not rebuild or self.getProperty('rerun') >= retrays:
# failed and build requested cpv
if cpv == self.faild_cpv:
@@ -1066,11 +1090,16 @@ class CheckEmergeLogs(BuildStep):
#FIXME: Set build timeout in config
build_timeout = 6600
shell_commad_list = []
+ exclude = False
# rebuild broken haskell
if rebuild == 'haskell':
shell_commad_list.append('haskell-updater')
shell_commad_list.append('--')
shell_commad_list.append('--usepkg=n')
+ if rebuild == 'binary':
+ c = yield catpkgsplit(self.faild_cpv)[0]
+ p = yield catpkgsplit(self.faild_cpv)[1]
+ exclude = c + '/' + p
if shell_commad_list != []:
self.aftersteps_list.append(
steps.ShellCommand(
@@ -1079,7 +1108,7 @@ class CheckEmergeLogs(BuildStep):
env=self.getProperty("build_env"),
timeout=build_timeout
))
- self.aftersteps_list.append(RunEmerge(step='build'))
+ self.aftersteps_list.append(RunEmerge(step='build', exclude=exclude))
self.aftersteps_list.append(CheckEmergeLogs('build'))
self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
if not self.step is None and self.aftersteps_list != []:
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-03-16 8:49 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-03-16 8:49 UTC (permalink / raw
To: gentoo-commits
commit: c37dbde9c1e6688aa7bc9095fe64c91717d9c97d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 16 08:49:05 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Mar 16 08:49:05 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c37dbde9
Use faild_version_data when faild to get repo bug settings
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index f214ad6..a261dac 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -500,12 +500,20 @@ class SetupBugReportSteps(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
bug_config = self.gentooci.config.project['bug_config']
- bug_settings_data = yield self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.getProperty("project_data")['uuid'])
- project_repository_settings = yield self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(self.getProperty("project_data")['uuid'], self.getProperty('repository_data')['uuid'])
- # add bug enable profile and repo
make_bugreport = False
+ print(bug_config)
if bug_config['enable'] and self.getProperty('status') == 'failed' and self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
- if bug_settings_data['enable'] and project_repository_settings['bug']:
+ # add bug enable profile, repo and project
+ bug_settings_data = yield self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.getProperty("project_data")['uuid'])
+ if self.getProperty('failed_version_data'):
+ version_data = self.getProperty('failed_version_data')
+ else:
+ version_data = self.getProperty("version_data")
+ package_data = yield self.gentooci.db.packages.getPackageByUuid(version_data['package_uuid'])
+ project_repository_settings = yield self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(self.getProperty("project_data")['uuid'], package_data['repository_uuid'])
+ print(bug_settings_data)
+ print(project_repository_settings)
+ if bug_settings_data['enabled'] and project_repository_settings['bug']:
make_bugreport = True
if make_bugreport:
aftersteps_list = []
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-02-05 18:54 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-02-05 18:54 UTC (permalink / raw
To: gentoo-commits
commit: 7f25300ddcf94e1d5018e9bc343a22155e3b043c
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 5 18:54:20 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 5 18:54:20 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7f25300d
Use settings repository for dockerfiles
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 34 ++++++++++++++--------------------
1 file changed, 14 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py
index 7a01cbb..c1928ba 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -77,30 +77,26 @@ class SetupBuildStepsForDocker(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
node_data = self.getProperty("node_data")
image_data = self.getProperty("image_data")
- dockerfile = self.getProperty("image_data")['dockerfile'] + '.Dockerfile'
+ dockerfile = os.path.join('Project', self.getProperty("project_data")['uuid'],'docker', image_data['dockerfile'] + '.Dockerfile')
+ repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid'])
dockerfile_path = os.path.join(self.master.basedir, 'docker')
#FIXME: Set build timeout in config
self.build_timeout = 6600
aftersteps_list = []
- aftersteps_list.append(steps.FileDownload(
- flunkOnFailure=True,
- name='Upload Dockerfile',
- mastersrc = os.path.join(dockerfile_path, dockerfile),
- workerdest = dockerfile
- ))
+ aftersteps_list.append(steps.GitLab(repourl=repository_data['url'],
+ name = 'RunGit',
+ descriptionDone=repository_data['name'],
+ mode=repository_data['mode'],
+ method=repository_data['method'],
+ submodules=True,
+ alwaysUseLatest=repository_data['alwaysuselatest']
+ ))
# build the gentoo docker buildbot-worker image
# set the build args
self.buildargs = {}
- # set PROJECTUUID
- self.buildargs['PROJECTUUID'] = self.getProperty("project_uuid")
- # set ACCEPT_KEYWORDS
- keyword_data = yield self.gentooci.db.keywords.getKeywordById(self.getProperty('project_data')['keyword_id'])
- if self.getProperty('project_data')['status'] == 'unstable':
- self.buildargs['KEYWORDS'] = '~' + keyword_data['name']
- else:
- self.buildargs['KEYWORDS'] = keyword_data['name']
- # set ARCH
- self.buildargs['ARCH'] = keyword_data['name']
+ # set PROJECTUUID and PROJECTNAME
+ self.buildargs['PROJECTUUID'] = self.getProperty("project_data")['uuid']
+ self.buildargs['PROJECTNAME'] = self.getProperty("project_data")['name']
# set MAKEOPTS
# set USE
# set CFLAGS
@@ -132,12 +128,10 @@ class SetupBuildStepsForDocker(BuildStep):
flunkOnFailure=True,
name='Build docker buildbot-worker image',
command=['docker',
- 'buildx',
'build',
'--file', dockerfile,
'--build-arg', 'PROJECTUUID='+ self.buildargs['PROJECTUUID'],
- '--build-arg', 'KEYWORDS=' + self.buildargs['KEYWORDS'],
- '--build-arg', 'ARCH=' + self.buildargs['ARCH'],
+ '--build-arg', 'PROJECTNAME='+ self.buildargs['PROJECTNAME'],
'--build-arg', 'BINHOSTURL=' + self.buildargs['BINHOSTURL'],
'--build-arg', 'BOOTSTRAPTAG=' + self.buildargs['BOOTSTRAPTAG'],
'--tag', image_data['tag'],
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-02-04 11:32 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-02-04 11:32 UTC (permalink / raw
To: gentoo-commits
commit: 34ce6c4b4c8420b77870ef157c60984dc5ae46ce
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 4 11:32:24 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Feb 4 11:32:24 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=34ce6c4b
Use project_data for profile link
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 775e076..f90da5a 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -92,12 +92,12 @@ class SetMakeProfile(BuildStep):
portage_repos_path = self.getProperty('portage_repos_path')
project_data = self.getProperty('project_data')
profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
- makeprofiles_paths = []
+ #makeprofiles_paths = []
#NOTE: pkgcheck don't support make.profile as a dir
# we only support one line in db
- makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
- for makeprofile in makeprofiles_data:
- makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', makeprofile['value'], '')
+ #makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
+ #for makeprofile in makeprofiles_data:
+ makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', project_data['profile'], '')
# makeprofiles_paths.append('../../..' + makeprofile_path)
#separator = '\n'
#makeprofile_path_string = separator.join(makeprofiles_paths)
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-02-04 9:49 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-02-04 9:49 UTC (permalink / raw
To: gentoo-commits
commit: aebeb68573134f1f1879b7513d5051e562dd3b1c
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 4 09:49:24 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Feb 4 09:49:24 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aebeb685
Use flavor to set MAKEOPTS -j*
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 74a8e37..775e076 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -228,7 +228,12 @@ class SetHostConf(BuildStep):
separator2 = ' '
makeconf_list = []
log = yield self.addLog('host.conf')
- makeconf_list.append('MAKEOPTS="-j14"')
+ # set MAKEOPTS from flavor
+ worker_data = yield self.gentooci.db.workers.getWorkerByUuid(self.getProperty('workername'))
+ print('worker_data')
+ worker_flavor = yield self.gentooci.db.workers.getFlavorById(worker_data['flavor_id'])
+ print('worker_flavor')
+ makeconf_list.append('MAKEOPTS="-j' + str(worker_flavor['cpu']) + '"')
makeconf_string = separator1.join(makeconf_list)
print(makeconf_string)
yield self.build.addStepsAfterCurrentStep([
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2024-02-03 10:02 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2024-02-03 10:02 UTC (permalink / raw
To: gentoo-commits
commit: eaf908437029fb291b327db680d550fbe36c86df
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 3 09:48:17 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 3 09:48:17 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=eaf90843
Remove minio
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/minio.py | 60 ---------------------------------------
1 file changed, 60 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/minio.py b/buildbot_gentoo_ci/steps/minio.py
deleted file mode 100644
index ac9ac3d..0000000
--- a/buildbot_gentoo_ci/steps/minio.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# Copyright 2021 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-from minio import Minio
-from minio.error import ResponseError, BucketAlreadyOwnedByYou, BucketAlreadyExists
-
-from twisted.internet import defer
-from twisted.python import log
-
-from buildbot.process.buildstep import BuildStep
-from buildbot.process.results import SUCCESS
-from buildbot.process.results import FAILURE
-
-#FIXME:
-# get url, user from config
-# get password from secret
-url = ''
-user = ''
-password = ''
-
-class putFileToMinio(BuildStep):
-
- name = 'putFileToMinio'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = False
- flunkOnFailure = True
- warnOnWarnings = True
-
- def __init__(self, filename, target, bucket, **kwargs):
- self.filename = filename
- self.bucket = bucket
- self.target = target
- super().__init__(**kwargs)
-
- def getMinioConnect(self, url, user, password):
- minioclient = Minio(
- url,
- access_key = user,
- secret_key = password,
- secure = False
- )
- return minioclient
-
- @defer.inlineCallbacks
- def pushFileToMinio(self):
- try:
- yield self.minio_connect.fput_object(self.bucket, self.target, self.filename)
- except ResponseError as err:
- print(err)
- return FAILURE
- return True
-
- @defer.inlineCallbacks
- def run(self):
- self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- self.minio_connect = yield self.getMinioConnect(url, user, password)
- success = yield self.pushFileToMinio()
- return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-07-26 9:23 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-07-26 9:23 UTC (permalink / raw
To: gentoo-commits
commit: db8e966eef3ede0212dcf85d6d5b82c28d3dfcd5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 26 09:21:16 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jul 26 09:21:16 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=db8e966e
Check issue titel nice
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 13 +++++++++++--
buildbot_gentoo_ci/steps/logs.py | 31 +++++++++++++++----------------
2 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index ad87be9..4d958b7 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -126,13 +126,22 @@ class GetBugs(BuildStep):
for match_word in match_search_text:
if match_word in match_bug_text:
matches = matches + 1
- print(f"Bug: {str(bug['id'])} Matched words: {str(matches)} Summary: {bug['summary']}")
- if matches >= 5:
+ # try to match the nice words
+ matches_nice = 0
+ match_search_text_nice = list(self.getProperty('error_dict')['title_issue_nice'].split())
+ #FIXME: add check for cp
+ for match_word in match_search_text_nice:
+ if match_word in match_bug_text:
+ matches_nice = matches_nice + 1
+ print(f"Bug: {str(bug['id'])} Matched words: {str(matches)} {str(matches_nice)} Summary: {bug['summary']}")
+ #FIXME: set it in bug_config
+ if matches >= 6 or matches_nice >= 6:
match = {}
match['match'] = True
match['id'] = bug['id']
match['summary'] = bug['summary']
yield log.addStdout(f"Line to match: {self.getProperty('error_dict')['title_issue']}\n")
+ yield log.addStdout(f"Line to match: {self.getProperty('error_dict')['title_issue_nice']}\n")
if match:
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary: {match['summary']}\n")
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 47112e0..d71eb99 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -184,18 +184,18 @@ class MakeIssue(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
- def getNiceErrorLine(self, line):
+ def getNiceErrorLine(self, full_line):
# strip away hex addresses, loong path names, line and time numbers and other stuff
- # https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L467
+ # https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L469
# FIXME: Add the needed line when needed
- if re.search(': line', line):
- line = re.sub(r"\d", "<snip>", line)
- # Shorten the path
- if line.startswith('/usr/'):
- line = line.replace(os.path.split(line.split(' ', 1)[0])[0], '/...')
- if re.search(': \d:\d: ', line):
- line = re.sub(r":\d:\d: ", ": ", line)
- return line
+ new_line = []
+ for line in full_line.split(' '):
+ # Shorten the path
+ if line.startswith('/usr/') or line.startswith('/var/') or line.startswith('../'):
+ split_path_line = os.path.split(line)
+ line = line.replace(split_path_line[0], '...')
+ new_line.append(line)
+ return ' '.join(new_line)
def ClassifyIssue(self):
# get the title for the issue
@@ -215,11 +215,12 @@ class MakeIssue(BuildStep):
self.error_dict['title_found'] = True
else:
self.error_dict['title_issue'] = 'title_issue : None'
- self.error_dict['title_nice'] = 'title_issue : None'
+ self.error_dict['title_issue_nice'] = 'title_issue_nice : None'
self.error_dict['title_found'] = False
- self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
+ self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase'] + ':'
#set the error title
- self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' + self.error_dict['title_issue']
+ self.error_dict['title'] = ' '.join([self.error_dict['title_phase'], self.error_dict['title_issue']])
+ self.error_dict['title_nice'] = ' '.join([self.error_dict['title_phase'], self.error_dict['title_issue_nice']])
@defer.inlineCallbacks
def run(self):
@@ -249,8 +250,6 @@ class MakeIssue(BuildStep):
yield self.ClassifyIssue()
print(self.error_dict)
self.setProperty("status", 'failed', 'status')
- #FIXME: nice description for post bug
- self.error_dict['summary_log_nice'] = ''
self.setProperty("error_dict", self.error_dict, 'error_dict')
self.aftersteps_list.append(bugs.GetBugs())
if warning:
@@ -463,7 +462,7 @@ class SetupBugReportSteps(BuildStep):
bug_args['user'] = bug_config['user']
bug_args['passwd'] = yield p.render(util.Secret("bugs_password"))
bug_params = {}
- title = separator.join([bug_config['extra_summery'], self.getProperty('error_dict')['cpv'], self.getProperty('error_dict')['title']])
+ title = separator.join([bug_config['extra_summery'], self.getProperty('error_dict')['cpv'], self.getProperty('error_dict')['title_nice']])
# bug title max 170
if len(title) >= 170:
title = title[:167] + '...'
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-06-10 10:08 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-06-10 10:08 UTC (permalink / raw
To: gentoo-commits
commit: f664663c8bebcd6700c90e2ca445d82bdf9b8432
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 10 09:53:48 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Jun 10 09:53:48 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f664663c
Add support for Post, Modify and Attach bugs
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 178 +++++++++++++++++++++++++++++++++++++--
1 file changed, 173 insertions(+), 5 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 6c85aac..ad87be9 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -1,7 +1,12 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+# Origins: bugz.cli
+# Modifyed by Gentoo Authors.
+
import re
+import os
+import xmlrpc
from twisted.internet import defer
@@ -9,6 +14,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
+from buildbot.plugins import util
from bugz.cli import check_bugz_token, login, list_bugs
from bugz.cli_argparser import make_arg_parser
@@ -19,8 +25,6 @@ from bugz.log import log_error, log_info
from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
-# Origins: bugz.cli
-# Modifyed by Gentoo Authors.
# main
def main_bugz(args):
ArgParser = make_arg_parser()
@@ -69,6 +73,32 @@ def search_bugz(args):
list_bugs(result, settings)
return result
+# post
+def post_bugs(args, params):
+ """Post a new bug"""
+ settings = main_bugz(args)
+ result = settings.call_bz(settings.bz.Bug.create, params)
+ log_info('Bug %d submitted' % result['id'])
+ return result
+
+# modify
+def modify_bugs(args, params):
+ """Modify an existing bug (eg. adding a comment or changing resolution.)"""
+ settings = main_bugz(args)
+ if len(params) < 2:
+ raise BugzError('No changes were specified')
+ result = settings.call_bz(settings.bz.Bug.update, params)
+ return result
+
+def attach_bugs(args, params):
+ """ Attach a file to a bug given a filename. """
+ settings = main_bugz(args)
+ fd = open(params['filename'], 'rb')
+ params['data'] = xmlrpc.client.Binary(fd.read())
+ fd.close()
+ result = settings.call_bz(settings.bz.Bug.add_attachment, params)
+ return result
+
class GetBugs(BuildStep):
name = 'GetBugs'
@@ -107,8 +137,9 @@ class GetBugs(BuildStep):
yield log.addStdout('Match bug: YES\n')
yield log.addStdout(f"Bug: {str(match['id'])} Summary: {match['summary']}\n")
self.setProperty("bgo", match, 'bgo')
- return
+ return match
yield log.addStdout('Match bug: NO\n')
+ return match
@defer.inlineCallbacks
def run(self):
@@ -128,5 +159,142 @@ class GetBugs(BuildStep):
print(args)
buglist = search_bugz(args)
print(buglist)
- self.find_match(buglist)
+ match = self.find_match(buglist)
+ #FIXME: set bug id on build in db
+ return SUCCESS
+
+class Post(BuildStep):
+
+ name = 'Post bug'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, bug_args, bugs_params, **kwargs):
+ self.bug_args = bug_args
+ self.bugs_params = bugs_params
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ #self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ #bug_config = self.gentooci.config.project['bug_config']
+ args = []
+ args.append('-u')
+ args.append(self.bug_args['user'])
+ args.append('-p')
+ args.append(self.bug_args['passwd'])
+ args.append('post')
+ params = {}
+ params['product'] = "Gentoo Linux"
+ params['component'] = "Current packages"
+ params['version'] = "unspecified"
+ params['op_sys'] = "Linux"
+ params['platform'] = "All"
+ params['priority'] = "Normal"
+ params['description'] = self.bugs_params['description']
+ params['summary'] = self.bugs_params['summary']
+ params['assigned_to'] = self.bugs_params['assigned_to']
+ #params['cc'] = settings.cc
+ params['url'] = self.bugs_params['url']
+ bug_info = yield post_bugs(args, params)
+ match = {}
+ match['match'] = True
+ match['id'] = bug_info['id']
+ match['summary'] = self.bugs_params['summary']
+ self.setProperty("bgo", match, 'bgo')
+ print(match)
+ self.descriptionDone = f"Bug: {bug_info['id']} submitted"
+ #FIXME: update build with bug id
+ return SUCCESS
+
+class Modify(BuildStep):
+ name = 'Modify'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, bug_args, bug_params, **kwargs):
+ self.bug_args = bug_args
+ self.bug_params = bug_params
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ bugid = str(self.getProperty('bgo')['id'])
+ args = []
+ args.append('-u')
+ args.append(self.bug_args['user'])
+ args.append('-p')
+ args.append(self.bug_args['passwd'])
+ args.append('modify')
+ args.append(bugid)
+ params = {}
+ if self.bug_params['comment'] is not None:
+ params['comment'] = {}
+ params['comment']['body'] = self.bug_params['comment']
+ params['ids'] = [bugid]
+ print(params)
+ bug_info = yield modify_bugs(args, params)
+ for bug in bug_info['bugs']:
+ changes = bug['changes']
+ if not len(changes):
+ log_info('Added comment to bug %s' % bug['id'])
+ else:
+ log_info('Modified the following fields in bug %s' % bug['id'])
+ for key in changes:
+ log_info('%-12s: removed %s' %(key, changes[key]['removed']))
+ log_info('%-12s: added %s' %(key, changes[key]['added']))
+ self.descriptionDone = f"Modified bug {self.getProperty('bgo')['id']}"
+ return SUCCESS
+
+class Attach(BuildStep):
+ name = 'Attach'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, bug_args, bug_params, **kwargs):
+ self.bug_args = bug_args
+ self.bug_params = bug_params
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ bugid = str(self.getProperty('bgo')['id'])
+ args = []
+ args.append('-u')
+ args.append(self.bug_args['user'])
+ args.append('-p')
+ args.append(self.bug_args['passwd'])
+ args.append('attach')
+ args.append(bugid)
+ args.append(self.bug_params['filename'])
+ summary = os.path.basename(self.bug_params['filename'])
+
+ if not os.path.exists(self.bug_params['filename']):
+ raise BugzError('File not found: %s' % self.bug_params['filename'])
+
+ params = {}
+ params['ids'] = bugid
+ params['file_name'] = summary
+ params['summary'] = summary
+ if self.bug_params['filename'].endswith('gz'):
+ params['content_type'] = 'application/gzip'
+ elif self.bug_params['filename'].endswith('bz2'):
+ params['content_type'] = 'application/x-bzip'
+ elif self.bug_params['filename'].endswith('xz'):
+ params['content_type'] = 'application/x-xz'
+ else:
+ params['content_type'] = 'text/plain'
+ #params['comment'] = self.bug_params['comment']
+ params['filename'] = self.bug_params['filename']
+ print(f"Add params['filename'] to bug {bugid}")
+ #params['is_patch'] = is_patch
+ bug_info = yield attach_bugs(args, params)
+ log_info("'%s' has been attached to bug %s" % (params['filename'], bugid))
+ self.descriptionDone = f"{os.path.basename(params['filename'])} has been attached to bug {bugid}"
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-06-05 8:09 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-06-05 8:09 UTC (permalink / raw
To: gentoo-commits
commit: 33a7f172aa06d773f11d2916130a48d16760c7d1
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 4 20:57:28 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jun 4 20:57:28 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=33a7f172
Support splited lines in auxdb output
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/version.py | 53 +++++++++++++++++--------------------
1 file changed, 24 insertions(+), 29 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index c6f333e..98e5403 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -26,11 +26,9 @@ def PersOutputOfGetEapi(rc, stdout, stderr):
# split the lines
for line in stdout.split('\n'):
if line.startswith('EAPI'):
- print(line[-1])
m = _pms_eapi_re.match(line)
if m is not None:
eapi = m.group(2)
- print(eapi)
if eapi is None or not eapi_is_supported(eapi):
print('ERROR: invalid eapi or not found')
eapi = False
@@ -41,37 +39,34 @@ def PersOutputOfGetEapi(rc, stdout, stderr):
}
def PersOutputOfGetAuxdb(rc, stdout, stderr):
- metadata = None
NoSplit = ['DESCRIPTION']
ignore_list = ['SRC_URI']
- #make dict of the stout
- index = 1
- metadata_line_dict = {}
- for text_line in stdout.splitlines():
- metadata_line_dict[index] = text_line
- index = index + 1
+ metadata = {}
+ i = 0
if not stderr == '':
print('stderr')
- # should have 22 lines
- if len(auxdbkeys) != index -1:
- # number of lines is incorrect.
- print('ERROR: Number of lines is incorrect')
- print(metadata_line_dict)
- return {
- 'auxdb' : metadata
- }
- # split all keys to list instead of speces
- metadata = {}
- i = 1
- for key in auxdbkeys:
- if metadata_line_dict[i][-1] == '=' or key in ignore_list:
- metadata[key] = False
- else:
- if ' ' in metadata_line_dict[i] and key not in NoSplit:
- metadata[key] = metadata_line_dict[i].replace(key + '=', '').split(' ')
- else:
- metadata[key] = []
- metadata[key].append(metadata_line_dict[i].replace(key + '=', ''))
+ # make a dict of metadata
+ metadata_line_list = stdout.splitlines()
+ index = len(metadata_line_list)
+ for metadata_line in metadata_line_list:
+ print(metadata_line)
+ for key in auxdbkeys:
+ if metadata_line.startswith(key) and key not in ignore_list:
+ print(key)
+ if metadata_line[-1] == '=':
+ metadata[key] = False
+ else:
+ if ' ' in metadata_line and key not in NoSplit:
+ if i+1 == index or metadata_line_list[i+1].startswith(auxdbkeys):
+ metadata[key] = metadata_line.replace(key + '=', '').split(' ')
+ else:
+ # support one splited line
+ print(metadata_line_list[i+1])
+ metadata_line2 = metadata_line + metadata_line_list[i+1]
+ metadata[key] = metadata_line2.replace(key + '=', '').split(' ')
+ else:
+ metadata[key] = []
+ metadata[key].append(metadata_line.replace(key + '=', ''))
i = i + 1
return {
'auxdb' : metadata
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-06-05 8:09 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-06-05 8:09 UTC (permalink / raw
To: gentoo-commits
commit: 89be950457b1ffebe2ba2a27e0b6dc7f455e3115
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 4 20:50:42 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jun 4 20:50:42 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=89be9504
Add fetchonly step but not on yet
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 8692813..a5638b5 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -564,6 +564,34 @@ class RunEmerge(BuildStep):
))
aftersteps_list.append(CheckEmergeLogs('pre-build'))
+ if self.step == 'fetchonly':
+ if projects_emerge_options['oneshot']:
+ shell_commad_list.append('-1')
+ shell_commad_list.append('-f')
+ #shell_commad_list.append('--quiet=n')
+ shell_commad_list.append('=' + self.getProperty('cpv'))
+ # we don't use the bin for the requsted cpv
+ shell_commad_list.append('--usepkg-exclude')
+ shell_commad_list.append(cp)
+ # don't build bin for virtual and acct-*
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('virtual')
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('acct-*')
+ aftersteps_list.append(
+ steps.ShellCommand(
+ warnOnWarnings = True,
+ warnOnFailure = True,
+ flunkOnFailure = False,
+ flunkOnWarnings = False,
+ name = self.stepname,
+ command=shell_commad_list,
+ #strip=True,
+ #extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ timeout=self.build_timeout
+ ))
+
if self.step == 'build':
if projects_emerge_options['oneshot']:
shell_commad_list.append('-1')
@@ -787,6 +815,7 @@ class CheckEmergeLogs(BuildStep):
self.addFileUploade(sourcefile, destfile, name, url, urlText)
# get elogs
self.getElogFiles(cpv)
+ #FIXME: add etc/portage dir to upload
@defer.inlineCallbacks
def getBuildWorkDirs(self, cpv):
@@ -1223,6 +1252,7 @@ class RunBuild(BuildStep):
aftersteps_list = []
aftersteps_list.append(RunEmerge(step='pre-build'))
aftersteps_list.append(RunEmergeInfo())
+ #aftersteps_list.append(RunEmerge(step='fetchonly'))
aftersteps_list.append(RunEmerge(step='build'))
aftersteps_list.append(RunEmerge(step='pre-depclean'))
aftersteps_list.append(RunEmerge(step='preserved-libs'))
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-05-08 20:19 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-05-08 20:19 UTC (permalink / raw
To: gentoo-commits
commit: 6f1931fa4e868471cda59e88a1b1a8863753f9db
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon May 8 20:18:11 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon May 8 20:18:11 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6f1931fa
Use deep on eclean and check string on faild_cpv
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 30 +++++++++++++++++++-----------
1 file changed, 19 insertions(+), 11 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 64a3600..8692813 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -492,23 +492,23 @@ class RunEmerge(BuildStep):
# Run eclean on Packages and Distfiles
#FIXME: cleantime set in config
- cleantime_package = '1w'
+ cleantime_package = '2d'
cleantime_distfiles = '2d'
if self.step == 'eclean':
aftersteps_list.append(
steps.ShellCommand(
flunkOnFailure=False,
name = 'Run eclean Packages',
- command=['eclean-pkg', '-Ct', cleantime_package],
+ command=['eclean-pkg', '-C','-d', '-t' + cleantime_package],
+ workdir='/'
+ ))
+ aftersteps_list.append(
+ steps.ShellCommand(
+ flunkOnFailure=False,
+ name = 'Run eclean Distfiles',
+ command=['eclean-dist', '-C', '-d', '-t' + cleantime_distfiles],
workdir='/'
))
- #aftersteps_list.append(
- # steps.ShellCommand(
- # flunkOnFailure=False,
- # name = 'Run eclean Distfiles',
- # command=['eclean-dist', '-Ct', cleantime_distfiles],
- # workdir='/'
- # ))
# Fix/Check Packages file
aftersteps_list.append(
steps.ShellCommand(
@@ -946,6 +946,7 @@ class CheckEmergeLogs(BuildStep):
log_path = log_path,
full_logname = full_logname
)
+ print('log_dict')
print(log_dict)
if self.step == 'extra-build':
#FIXME: Check if extra build did work
@@ -962,15 +963,19 @@ class CheckEmergeLogs(BuildStep):
retrays = 1
# Find log for cpv that was requested or did failed
if not log_dict == {}:
+ print('log_dict not emty')
# requested cpv
cpv = self.getProperty('cpv')
faild_version_data = False
if cpv in log_dict or self.faild_cpv in log_dict:
+ print(cpv)
+ print(self.faild_cpv)
yield self.createDistDir()
if cpv in log_dict:
self.log_data[cpv] = log_dict[cpv]
yield self.getLogFile(cpv, log_dict)
- if self.faild_cpv:
+ if isinstance(self.faild_cpv, str):
+ print('faild_cpv is string')
c = yield catpkgsplit(self.faild_cpv)[0]
if c == 'dev-haskell':
rebuild = 'haskell'
@@ -984,10 +989,13 @@ class CheckEmergeLogs(BuildStep):
yield self.getLogFile(self.faild_cpv, log_dict)
faild_version_data = yield self.getVersionData(self.faild_cpv)
self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
+ print(faild_version_data)
+ print('getEmergeFiles')
self.getEmergeFiles(self.faild_cpv)
self.getBuildWorkDirs(self.faild_cpv)
rebuild = False
else:
+ print('getEmergeFiles')
self.getEmergeFiles(cpv)
if not rebuild:
self.aftersteps_list.append(steps.Trigger(
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-03-10 22:36 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-03-10 22:36 UTC (permalink / raw
To: gentoo-commits
commit: f9eed766d660ae99972201950f311c4cefc59be8
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 10 22:33:41 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Mar 10 22:33:41 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f9eed766
Download docker images befor build
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py
index f92727b..7a01cbb 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -108,6 +108,25 @@ class SetupBuildStepsForDocker(BuildStep):
self.buildargs['BINHOSTURL'] = node_data['bin_host_url']
# set bootstrap image
self.buildargs['BOOTSTRAPTAG'] = image_data['bootstrap_tag']
+ # get the latest portage and bootstrap_tag images
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Pull ' + image_data['bootstrap_tag'],
+ command=['docker',
+ 'image',
+ 'pull',
+ image_data['bootstrap_tag']
+ ]
+ ))
+ aftersteps_list.append(steps.ShellCommand(
+ flunkOnFailure=True,
+ name='Pull gentoo/portage:latest',
+ command=['docker',
+ 'image',
+ 'pull',
+ 'gentoo/portage:latest'
+ ]
+ ))
# build the gentoo docker buildbot-worker image
aftersteps_list.append(steps.ShellCommand(
flunkOnFailure=True,
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-03-10 22:36 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-03-10 22:36 UTC (permalink / raw
To: gentoo-commits
commit: fe31d987a48588a27ded9bb0c1f0b49c69f273a8
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 10 22:35:27 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Mar 10 22:35:27 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=fe31d987
Add CheckEAPI
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/version.py | 32 +++++++++++++++++++++++++++++++-
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index 59d19dc..c6f333e 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -409,6 +409,35 @@ class SetupPropertys(BuildStep):
self.setProperty("version", self.version, 'version')
return SUCCESS
+class CheckEAPI(BuildStep):
+
+ name = 'CheckEAPI'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ #@defer.inlineCallbacks
+ def run(self):
+ # Run head 30 if we did't find it on head 10
+ if self.getProperty("eapi"):
+ return SKIPPED
+ self.build.addStepsAfterCurrentStep([
+ steps.SetPropertyFromCommand(
+ name = 'RunGetEAPI',
+ haltOnFailure = True,
+ flunkOnFailure = True,
+ command=['head', '-n','30', self.getProperty("ebuild_file")],
+ strip=False,
+ extract_fn=PersOutputOfGetEapi
+ )
+ ])
+ return SUCCESS
+
class SetupStepsForCheckV(BuildStep):
name = 'Setup steps for Checking V'
@@ -441,10 +470,11 @@ class SetupStepsForCheckV(BuildStep):
name = 'RunGetEAPI',
haltOnFailure = True,
flunkOnFailure = True,
- command=['head', '-n', '15', self.getProperty("ebuild_file")],
+ command=['head', '-n', '10', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
+ addStepVData.append(CheckEAPI())
addStepVData.append(steps.SetPropertyFromCommand(
name = 'GetPythonVersion',
haltOnFailure = True,
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-03-10 1:47 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-03-10 1:47 UTC (permalink / raw
To: gentoo-commits
commit: 1624134df50b603340e7e02d839f9d19476b52e8
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 10 01:45:07 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Mar 10 01:45:07 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1624134d
Change cleantime_package to 1w and disable eclean-dist
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index a3f82a3..64a3600 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -492,7 +492,7 @@ class RunEmerge(BuildStep):
# Run eclean on Packages and Distfiles
#FIXME: cleantime set in config
- cleantime_package = '3w'
+ cleantime_package = '1w'
cleantime_distfiles = '2d'
if self.step == 'eclean':
aftersteps_list.append(
@@ -502,13 +502,13 @@ class RunEmerge(BuildStep):
command=['eclean-pkg', '-Ct', cleantime_package],
workdir='/'
))
- aftersteps_list.append(
- steps.ShellCommand(
- flunkOnFailure=False,
- name = 'Run eclean Distfiles',
- command=['eclean-dist', '-Ct', cleantime_distfiles],
- workdir='/'
- ))
+ #aftersteps_list.append(
+ # steps.ShellCommand(
+ # flunkOnFailure=False,
+ # name = 'Run eclean Distfiles',
+ # command=['eclean-dist', '-Ct', cleantime_distfiles],
+ # workdir='/'
+ # ))
# Fix/Check Packages file
aftersteps_list.append(
steps.ShellCommand(
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-02-28 21:41 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-02-28 21:41 UTC (permalink / raw
To: gentoo-commits
commit: c71f1b00885a6bd3851182ebc27d5eea67351cbc
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 28 21:41:07 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Feb 28 21:41:07 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c71f1b00
Add eclean to building steps
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 1746122..a3f82a3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -490,6 +490,34 @@ class RunEmerge(BuildStep):
))
aftersteps_list.append(CheckDepcleanLogs('depclean'))
+ # Run eclean on Packages and Distfiles
+ #FIXME: cleantime set in config
+ cleantime_package = '3w'
+ cleantime_distfiles = '2d'
+ if self.step == 'eclean':
+ aftersteps_list.append(
+ steps.ShellCommand(
+ flunkOnFailure=False,
+ name = 'Run eclean Packages',
+ command=['eclean-pkg', '-Ct', cleantime_package],
+ workdir='/'
+ ))
+ aftersteps_list.append(
+ steps.ShellCommand(
+ flunkOnFailure=False,
+ name = 'Run eclean Distfiles',
+ command=['eclean-dist', '-Ct', cleantime_distfiles],
+ workdir='/'
+ ))
+ # Fix/Check Packages file
+ aftersteps_list.append(
+ steps.ShellCommand(
+ flunkOnFailure=False,
+ name='Fix/Check binary Package file',
+ command=['emaint', 'binhost', '--fix'],
+ workdir='/'
+ ))
+
if self.step == 'match':
shell_commad_list.append('-pO')
# don't use bin for match
@@ -1191,6 +1219,7 @@ class RunBuild(BuildStep):
aftersteps_list.append(RunEmerge(step='pre-depclean'))
aftersteps_list.append(RunEmerge(step='preserved-libs'))
aftersteps_list.append(RunEmerge(step='depclean'))
+ aftersteps_list.append(RunEmerge(step='eclean'))
self.setProperty('depclean', False, 'depclean')
self.setProperty('preserved_libs', False, 'preserved-libs')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2023-02-28 21:41 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2023-02-28 21:41 UTC (permalink / raw
To: gentoo-commits
commit: a2d7f047652600dc548a7401e3cc6b9b90cd7f1c
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 28 21:40:12 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Feb 28 21:40:12 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a2d7f047
Change head to 15 for EAPI search
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index 14d1d97..59d19dc 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -441,7 +441,7 @@ class SetupStepsForCheckV(BuildStep):
name = 'RunGetEAPI',
haltOnFailure = True,
flunkOnFailure = True,
- command=['head', '-n', '10', self.getProperty("ebuild_file")],
+ command=['head', '-n', '15', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-09-25 11:46 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-09-25 11:46 UTC (permalink / raw
To: gentoo-commits
commit: 1a48ccb70732fa8367c36c842861c080c03eb1a2
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 25 11:42:02 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Sep 25 11:42:02 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1a48ccb7
Add support for test on merge_request
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 10 ++++++++++
buildbot_gentoo_ci/steps/portage.py | 5 +++++
2 files changed, 15 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 3b6a8d0..00980e3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1296,6 +1296,16 @@ class SetupStepts(BuildStep):
if build:
yield log.addStdout('Runing Build: YES\n')
aftersteps_list.append(RunBuild())
+ # run tests
+ test = False
+ if self.getProperty('projectrepository_data')['test']:
+ test = True
+ if self.getProperty('projectrepository_data')['test_mr'] and self.getProperty("event") == 'merge_request':
+ test = True
+ if test:
+ yield log.addStdout('Runing Test: YES\n')
+ else:
+ yield log.addStdout('Runing Test: NO\n')
# run eclean pkg and dist
#if build:
#f.addStep(builders.RunEclean(step='pkg')
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index ff06c58..6d838d7 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -295,7 +295,12 @@ class SetPackageDefault(BuildStep):
yield log.addStdout(line + separator1)
# for test we need to add env and use
#FIXME: check restrictions, test use mask and required use
+ test = False
if self.getProperty('projectrepository_data')['test']:
+ test = True
+ if self.getProperty('projectrepository_data')['test_mr'] and self.getProperty("event") == 'merge_request':
+ test = True
+ if test:
auxdb_iuses = yield self.gentooci.db.versions.getMetadataByUuidAndMatadata(self.getProperty("version_data")['uuid'], 'iuse')
for auxdb_iuse in auxdb_iuses:
iuse, status = getIUseValue(auxdb_iuse['value'])
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-08-10 21:51 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-08-10 21:51 UTC (permalink / raw
To: gentoo-commits
commit: 25877447fec0f718c22bdc7cd6f31423aae75f7e
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 10 21:51:13 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Aug 10 21:51:13 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=25877447
Set head to 10 for EAPI search
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index dbeaf14..26cf9fc 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -436,7 +436,7 @@ class SetupStepsForCheckV(BuildStep):
name = 'RunGetEAPI',
haltOnFailure = True,
flunkOnFailure = True,
- command=['head', '-n', '8', self.getProperty("ebuild_file")],
+ command=['head', '-n', '10', self.getProperty("ebuild_file")],
strip=False,
extract_fn=PersOutputOfGetEapi
))
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-08-05 18:32 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-08-05 18:32 UTC (permalink / raw
To: gentoo-commits
commit: 3791e957fd2bb28f350f0ff7399c0c6e43b561d6
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Aug 5 18:32:11 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Aug 5 18:32:11 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3791e957
Add haskell-updater for haskell packages
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 147 ++++++++++++++++++++++-------------
1 file changed, 91 insertions(+), 56 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index a54c68c..3b6a8d0 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,6 +4,7 @@
import os
import re
import json
+from pathlib import Path
from portage.versions import catpkgsplit, cpv_getversion
from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
@@ -90,6 +91,8 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
+ if stderr != '' and not emerge_output['failed']:
+ emerge_output['failed'] = True
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -340,6 +343,7 @@ class SetupPropertys(BuildStep):
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
self.setProperty('masterdest', self.masterdest, 'masterdest')
+ self.setProperty('build_env', {}, 'build_env')
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for project', self.getProperty('project_data')['name']])
return SUCCESS
@@ -355,7 +359,6 @@ class RunEmerge(BuildStep):
super().__init__(**kwargs)
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
- self.build_env = {}
self.build_timeout = 0
@defer.inlineCallbacks
@@ -374,25 +377,28 @@ class RunEmerge(BuildStep):
aftersteps_list = []
#FIXME: Set build timeout in config
self.build_timeout = 6600
- # set env
- # https://bugs.gentoo.org/683118
- # export TERM=linux
- # export TERMINFO=/etc/terminfo
- self.build_env['TERM'] = 'linux'
- self.build_env['TERMINFO'] = '/etc/terminfo'
- # Lang
- self.build_env['LANG'] = 'C.utf8'
- self.build_env['LC_MESSAGES'] = 'C'
- # no color
- self.build_env['CARGO_TERM_COLOR'] = 'never'
- self.build_env['GCC_COLORS'] = '0'
- self.build_env['OCAML_COLOR'] = 'never'
- self.build_env['PY_FORCE_COLOR'] = '0'
- self.build_env['PYTEST_ADDOPTS'] = '--color=no'
- self.build_env['NO_COLOR'] = '1'
- # not all terms support urls
- self.build_env['GCC_URLS'] = 'no'
- self.build_env['TERM_URLS'] = 'no'
+ if self.step == 'update':
+ # set env
+ build_env = {}
+ # https://bugs.gentoo.org/683118
+ # export TERM=linux
+ # export TERMINFO=/etc/terminfo
+ build_env['TERM'] = 'linux'
+ build_env['TERMINFO'] = '/etc/terminfo'
+ # Lang
+ build_env['LANG'] = 'C.utf8'
+ build_env['LC_MESSAGES'] = 'C'
+ # no color
+ build_env['CARGO_TERM_COLOR'] = 'never'
+ build_env['GCC_COLORS'] = '0'
+ build_env['OCAML_COLOR'] = 'never'
+ build_env['PY_FORCE_COLOR'] = '0'
+ build_env['PYTEST_ADDOPTS'] = '--color=no'
+ build_env['NO_COLOR'] = '1'
+ # not all terms support urls
+ build_env['GCC_URLS'] = 'no'
+ build_env['TERM_URLS'] = 'no'
+ self.setProperty('build_env', build_env, 'build_env')
if self.step == 'pre-update':
shell_commad_list.append('-uDN')
@@ -432,6 +438,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
+ env=self.getProperty("build_env"),
timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('update'))
@@ -548,7 +555,7 @@ class RunEmerge(BuildStep):
strip=True,
extract_fn=PersOutputOfEmerge,
workdir='/',
- env=self.build_env,
+ env=self.getProperty("build_env"),
timeout=self.build_timeout
))
aftersteps_list.append(CheckEmergeLogs('build'))
@@ -682,11 +689,9 @@ class CheckEmergeLogs(BuildStep):
@defer.inlineCallbacks
def createDistDir(self):
workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'))
- self.aftersteps_list.append(steps.MasterShellCommand(
- name = 'Make directory for Uploaded files',
- command = ['mkdir', str(self.getProperty("buildnumber"))],
- workdir = workdir
- ))
+ check_dir = yield os.path.join(workdir, str(self.getProperty("buildnumber")))
+ if not Path(check_dir).is_dir():
+ yield Path(check_dir).mkdir(parents=True)
def addFileUploade(self, sourcefile, destfile, name, url, urlText):
self.aftersteps_list.append(steps.FileUpload(
@@ -814,7 +819,7 @@ class CheckEmergeLogs(BuildStep):
print(emerge_output)
# this should be set in the config
retrays = 5
- if self.getProperty('rerun') <= retrays:
+ if self.getProperty('rerun') <= retrays and self.faild_cpv:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
# but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
@@ -895,7 +900,9 @@ class CheckEmergeLogs(BuildStep):
self.aftersteps_list.append(CheckEmergeLogs('extra-build'))
else:
# trigger parse_build_log with info about pre-build and it fail
- pass
+ if self.faild_cpv:
+ pass
+ self.setProperty('rerun', 0, 'rerun')
# Make Logfile dict
if self.step == 'extra-build' or self.step == 'build':
print(emerge_output)
@@ -922,7 +929,9 @@ class CheckEmergeLogs(BuildStep):
# trigger a logparser
# local_log_path dir set in config
# format /var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz
+ rebuild = False
if self.step == 'build':
+ retrays = 1
# Find log for cpv that was requested or did failed
if not log_dict == {}:
# requested cpv
@@ -934,36 +943,62 @@ class CheckEmergeLogs(BuildStep):
self.log_data[cpv] = log_dict[cpv]
yield self.getLogFile(cpv, log_dict)
if self.faild_cpv:
- # failed and build requested cpv
- if cpv == self.faild_cpv:
- faild_version_data = self.getProperty("version_data")
- else:
- # failed but not build requested cpv
- self.log_data[self.faild_cpv] = log_dict[self.faild_cpv]
- yield self.getLogFile(self.faild_cpv, log_dict)
- faild_version_data = yield self.getVersionData(self.faild_cpv)
- self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
- self.getEmergeFiles(self.faild_cpv)
- self.getBuildWorkDirs(self.faild_cpv)
+ c = yield catpkgsplit(self.faild_cpv)[0]
+ if c == 'dev-haskell':
+ rebuild = 'haskell'
+ if not rebuild or self.getProperty('rerun') >= retrays:
+ # failed and build requested cpv
+ if cpv == self.faild_cpv:
+ faild_version_data = self.getProperty("version_data")
+ else:
+ # failed but not build requested cpv
+ self.log_data[self.faild_cpv] = log_dict[self.faild_cpv]
+ yield self.getLogFile(self.faild_cpv, log_dict)
+ faild_version_data = yield self.getVersionData(self.faild_cpv)
+ self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
+ self.getEmergeFiles(self.faild_cpv)
+ self.getBuildWorkDirs(self.faild_cpv)
+ rebuild = False
else:
self.getEmergeFiles(cpv)
- self.aftersteps_list.append(steps.Trigger(
- name = 'Setup properties for log parser and trigger it',
- schedulerNames=['parse_build_log'],
- waitForFinish=False,
- updateSourceStamp=False,
- set_properties={
- 'cpv' : self.getProperty("cpv"),
- 'faild_version_data' : faild_version_data,
- 'project_build_data' : self.getProperty('project_build_data'),
- 'log_build_data' : self.log_data,
- 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
- 'repository_data' : self.getProperty('repository_data'),
- 'faild_cpv' : self.faild_cpv,
- 'step' : self.step,
- 'build_workername' : self.getProperty('workername')
- }
- ))
+ if not rebuild:
+ self.aftersteps_list.append(steps.Trigger(
+ name = 'Setup properties for log parser and trigger it',
+ schedulerNames=['parse_build_log'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : self.getProperty("cpv"),
+ 'faild_version_data' : faild_version_data,
+ 'project_build_data' : self.getProperty('project_build_data'),
+ 'log_build_data' : self.log_data,
+ 'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
+ 'repository_data' : self.getProperty('repository_data'),
+ 'faild_cpv' : self.faild_cpv,
+ 'step' : self.step,
+ 'build_workername' : self.getProperty('workername')
+ }
+ ))
+ if rebuild:
+ #FIXME: Set build timeout in config
+ build_timeout = 6600
+ shell_commad_list = []
+ # rebuild broken haskell
+ if rebuild == 'haskell':
+ shell_commad_list.append('haskell-updater')
+ shell_commad_list.append('--')
+ shell_commad_list.append('--usepkg=n')
+ if shell_commad_list != []:
+ self.aftersteps_list.append(
+ steps.ShellCommand(
+ command=shell_commad_list,
+ workdir='/',
+ env=self.getProperty("build_env"),
+ timeout=build_timeout
+ ))
+ self.aftersteps_list.append(RunEmerge(step='build'))
+ self.aftersteps_list.append(CheckEmergeLogs('build'))
+ self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
if not self.step is None and self.aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-30 22:40 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-30 22:40 UTC (permalink / raw
To: gentoo-commits
commit: 3f7cd0136414f731cb97d50c486c291776d16830
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 30 22:40:02 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Jul 30 22:40:02 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3f7cd013
Fix UpdateRepos step in nodes
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py
index 3e15e5f..902f7a8 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -22,7 +22,7 @@ from buildbot.process.results import SKIPPED
from buildbot.plugins import steps, util
from buildbot_gentoo_ci.steps import portage as portage_steps
-from buildbot_gentoo_ci.steps import builders as builders_steps
+from buildbot_gentoo_ci.steps import repos as repos_steps
class SetupPropertys(BuildStep):
name = 'Setup propertys for stage4 image'
@@ -87,7 +87,7 @@ class SetupStage4Steps(BuildStep):
# download stage3
aftersteps_list.append(GetSteg3())
# setup portage
- aftersteps_list.append(builders_steps.UpdateRepos())
+ aftersteps_list.append(repos_steps.UpdateRepos())
aftersteps_list.append(portage_steps.SetReposConf())
aftersteps_list.append(portage_steps.SetMakeConf())
# add localegen
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-27 11:02 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-27 11:02 UTC (permalink / raw
To: gentoo-commits
commit: 63ea1fb9235a4585754d5b8acc41c281596ab4fc
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 26 12:57:35 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jul 26 12:57:35 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=63ea1fb9
Clean up and add use of rootworkdir
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 204 ++++--------------------------------
1 file changed, 21 insertions(+), 183 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 431fe9a..ff06c58 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -20,6 +20,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
+from buildbot.process import remotecommand
from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import master as master_steps
@@ -34,39 +35,6 @@ def WriteTextToFile(path, text_list):
yield f.write(separator)
yield f.close
-def PersOutputOfEbuildSH(rc, stdout, stderr):
- metadata = None
- NoSplit = []
- NoSplit.append('DESCRIPTION')
- #make dict of the stout
- index = 1
- metadata_line_dict = {}
- for text_line in stdout.splitlines():
- metadata_line_dict[index] = text_line
- index = index + 1
- # should have 22 lines
- if len(auxdbkeys) != index -1:
- # number of lines is incorrect.
- return {
- 'auxdb' : metadata
- }
- # split all keys to list instead of speces
- metadata = {}
- i = 1
- for key in auxdbkeys:
- if metadata_line_dict[i] == '':
- metadata[key] = None
- else:
- if ' ' in metadata_line_dict[i] and key not in NoSplit:
- metadata[key] = metadata_line_dict[i].split(' ')
- else:
- metadata[key] = []
- metadata[key].append(metadata_line_dict[i])
- i = i + 1
- return {
- 'auxdb' : metadata
- }
-
class SetMakeProfile(BuildStep):
name = 'SetMakeProfile'
@@ -76,8 +44,7 @@ class SetMakeProfile(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, workdir=False, **kwargs):
- self.rootworkdir = workdir
+ def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -105,8 +72,8 @@ class SetMakeProfile(BuildStep):
'ln',
'-s'
]
- if self.rootworkdir:
- symlink_makeprofile_path = yield os.path.join(self.rootworkdir, 'etc/portage/make.profile')
+ if self.getProperty('rootworkdir'):
+ symlink_makeprofile_path = yield os.path.join(self.getProperty('rootworkdir'), 'etc/portage/make.profile')
else:
symlink_makeprofile_path = '/etc/portage/make.profile'
shell_commad_list.append(makeprofile_path)
@@ -130,8 +97,7 @@ class SetReposConf(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, workdir=False, **kwargs):
- self.rootworkdir = workdir
+ def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -139,8 +105,8 @@ class SetReposConf(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
portage_repos_path = self.getProperty('portage_repos_path')
project_data = self.getProperty('project_data')
- if self.rootworkdir:
- portage_etc_path = yield os.path.join(self.rootworkdir, 'etc/portage/')
+ if self.getProperty('rootworkdir'):
+ portage_etc_path = yield os.path.join(self.getProperty('rootworkdir'), 'etc/portage/')
else:
portage_etc_path = '/etc/portage/'
# setup the default.conf
@@ -148,6 +114,13 @@ class SetReposConf(BuildStep):
if repos_conf_data is None:
print('Default repo is not set in repos.conf')
return FAILURE
+ self.aftersteps_list = []
+ # check if we have the dir
+ cmd = remotecommand.RemoteCommand('stat', {'file': portage_etc_path + 'repos.conf'})
+ yield self.runCommand(cmd)
+ if cmd.didFail():
+ self.aftersteps_list.append(steps.MakeDirectory(dir="repos.conf",
+ workdir=portage_etc_path))
log = yield self.addLog('repos.conf')
# check if repos_conf_data['value'] is vaild repo name
separator = '\n'
@@ -156,11 +129,11 @@ class SetReposConf(BuildStep):
default_conf.append('main-repo = ' + repos_conf_data['value'])
default_conf.append('auto-sync = no')
default_conf_string = separator.join(default_conf)
- yield self.build.addStepsAfterCurrentStep([
+ self.aftersteps_list.append(
steps.StringDownload(default_conf_string + separator,
workerdest="repos.conf/default.conf",
workdir=portage_etc_path)
- ])
+ )
# display the default.conf
yield log.addStdout('File: ' + 'default.conf' + '\n')
for line in default_conf:
@@ -178,14 +151,15 @@ class SetReposConf(BuildStep):
repository_conf.append('auto-sync = no')
repository_conf_string = separator.join(repository_conf)
filename = repository_data['name'] + '.conf'
- yield self.build.addStepsAfterCurrentStep([
+ self.aftersteps_list.append(
steps.StringDownload(repository_conf_string + separator,
workerdest='repos.conf/' + filename,
workdir=portage_etc_path)
- ])
+ )
yield log.addStdout('File: ' + filename + '\n')
for line in repository_conf:
yield log.addStdout(line + '\n')
+ yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
class SetMakeConf(BuildStep):
@@ -207,8 +181,8 @@ class SetMakeConf(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
makeconf_variables_data = yield self.gentooci.db.portages.getVariables()
- if self.rootworkdir:
- portage_etc_path = yield os.path.join(self.rootworkdir, 'etc/portage/')
+ if self.getProperty('rootworkdir'):
+ portage_etc_path = yield os.path.join(self.getProperty('rootworkdir'), 'etc/portage/')
else:
portage_etc_path = '/etc/portage/'
separator1 = '\n'
@@ -541,139 +515,3 @@ class SetMakeConfLocal(BuildStep):
makeconf_list.append('FEATURES="sandbox"')
yield WriteTextToFile(make_conf_path, makeconf_list)
return SUCCESS
-
-class SetEnvForEbuildSH(BuildStep):
-
- name = 'SetEnvForEbuildSH'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- def getEapiFromFile(self):
- with io.open(_unicode_encode(self.getProperty("ebuild_file"),
- encoding=_encodings['fs'], errors='strict'),
- mode='r', encoding=_encodings['repo.content'],
- errors='replace') as f:
- _eapi, _eapi_lineno = _parse_eapi_ebuild_head(f)
-
- return _eapi
-
- @defer.inlineCallbacks
- def run(self):
- addStepEbuildSH = []
- ebuild_commands = []
- ebuild_env = {}
- config_root = yield os.path.join(self.getProperty("builddir"), '')
- mysettings = yield portage_config(config_root = config_root)
-
- #Get EAPI from file and add it to env
- eapi = yield self.getEapiFromFile()
- print(eapi)
- if eapi is None or not eapi_is_supported(eapi):
- print('invalid eapi')
- eapi = '0'
- print(eapi_is_supported(eapi))
- ebuild_env['EAPI'] = eapi
-
- #FIXME: check manifest on ebuild_file
-
- #Setup ENV
- category = yield catpkgsplit(self.getProperty("cpv"))[0]
- package = yield catpkgsplit(self.getProperty("cpv"))[1]
- version = yield catpkgsplit(self.getProperty("cpv"))[2]
- revision = yield catpkgsplit(self.getProperty("cpv"))[3]
- portage_bin_path = mysettings["PORTAGE_BIN_PATH"]
- ebuild_sh_path = yield os.path.join(portage_bin_path, 'ebuild.sh')
- #ebuild_env['PORTAGE_DEBUG'] = '1'
- ebuild_env['EBUILD_PHASE'] = 'depend'
- ebuild_env['CATEGORY'] = category
- ebuild_env['P'] = package + '-' + version
- ebuild_env['PN'] = package
- ebuild_env['PR'] = revision
- ebuild_env['PV'] = version
- if revision == 'r0':
- ebuild_env['PF'] = ebuild_env['P']
- ebuild_env['PVR'] = version
- else:
- ebuild_env['PF'] = ebuild_env['P'] + '-' + revision
- ebuild_env['PVR'] = version + '-' + revision
- ebuild_env['PORTAGE_BIN_PATH'] = portage_bin_path
- ebuild_env['EBUILD'] = self.getProperty("ebuild_file")
- ebuild_env['PORTAGE_PIPE_FD'] = '1'
- ebuild_env['WORKDIR'] = yield os.path.join(mysettings["PORTAGE_TMPDIR"], 'portage', category, ebuild_env['PF'], 'work')
- ebuild_env['PORTAGE_ECLASS_LOCATIONS'] = self.getProperty("repository_path")
-
- #FIXME: use sandbox if in FEATURES
- ebuild_commands.append(ebuild_sh_path)
- ebuild_commands.append('depend')
-
- addStepEbuildSH.append(master_steps.MasterSetPropertyFromCommand(
- name = 'RunEbuildSH',
- haltOnFailure = True,
- flunkOnFailure = True,
- command=ebuild_commands,
- env=ebuild_env,
- workdir=self.getProperty("builddir"),
- strip=False,
- extract_fn=PersOutputOfEbuildSH
- ))
- yield self.build.addStepsAfterCurrentStep(addStepEbuildSH)
- return SUCCESS
-
-class GetAuxMetadata(BuildStep):
-
- name = 'GetAuxMetadata'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- #@defer.inlineCallbacks
- def thd_getAuxDbKeys(self):
- auxdbs = self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys, myrepo=self.getProperty("repository_data")['name'])
- return auxdbs
-
- @defer.inlineCallbacks
- def run(self):
- config_root = yield os.path.join(self.getProperty("builddir"), '')
- # setup mysettings and myportdb
- mysettings = yield portage_config(config_root = config_root)
- self.myportdb = yield portdbapi(mysettings=mysettings)
- auxdbs = yield self.thd_getAuxDbKeys()
- metadata = None
- NoSplit = []
- NoSplit.append('DESCRIPTION')
- # should have 22 lines
- if len(auxdbkeys) != len(auxdbs) or not isinstance(auxdbs, list):
- # number of lines is incorrect or not a list.
- print("Lines don't match or not a list")
- yield self.myportdb.close_caches()
- yield portdbapi.portdbapi_instances.remove(self.myportdb)
- #self.setProperty('auxdb', metadata, 'auxdb')
- return FAILURE
- # split all keys to list instead of speces
- metadata = {}
- i = 0
- for key in auxdbkeys:
- if auxdbs[i] == '':
- metadata[key] = None
- else:
- if ' ' in auxdbs[i] and key not in NoSplit:
- metadata[key] = auxdbs[i].split(' ')
- else:
- metadata[key] = []
- metadata[key].append(auxdbs[i])
- i = i + 1
- self.setProperty('auxdb', metadata, 'auxdb')
- yield self.myportdb.close_caches()
- yield portdbapi.portdbapi_instances.remove(self.myportdb)
- return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-27 11:02 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-27 11:02 UTC (permalink / raw
To: gentoo-commits
commit: 8ab6bf127155a3bddca43767fb7aa208a3ea4fe7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 26 12:59:19 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jul 26 12:59:19 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8ab6bf12
Add support for use of rootworkdir
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 38dfa79..383ae60 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -314,6 +314,7 @@ class SetupPropertys(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
self.setProperty('portage_repos_path', self.gentooci.config.project['project']['worker_portage_repos_path'], 'portage_repos_path')
+ self.setProperty('rootworkdir', False, 'rootworkdir')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
@@ -342,6 +343,7 @@ class SetupPropertys(BuildStep):
self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for project', self.getProperty('project_data')['name']])
return SUCCESS
+# Should be moved to repos.py
class UpdateRepos(BuildStep):
name = 'UpdateRepos'
@@ -351,8 +353,7 @@ class UpdateRepos(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, workdir=False, **kwargs):
- self.rootworkdir = workdir
+ def __init__(self, **kwargs):
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -364,8 +365,8 @@ class UpdateRepos(BuildStep):
projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
for project_repository_data in projects_repositorys_data:
repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
- if self.rootworkdir:
- repository_path = os.path.join(self.rootworkdir, portage_repos_path[1:], repository_data['name'])
+ if self.getProperty('rootworkdir'):
+ repository_path = os.path.join(self.getProperty('rootworkdir'), portage_repos_path[1:], repository_data['name'])
else:
repository_path = os.path.join(portage_repos_path, repository_data['name'], '')
yield self.build.addStepsAfterCurrentStep([
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-27 11:02 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-27 11:02 UTC (permalink / raw
To: gentoo-commits
commit: 59e77c4f9f2ca0d05aa4945e3e765353734791b1
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 26 12:53:07 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jul 26 12:53:07 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=59e77c4f
Add SetupPropertys for use of rootworkdir
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/package.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
index 76c90cc..81b400d 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -16,6 +16,23 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
+class SetupPropertys(BuildStep):
+ name = 'Setup propertys for CPV check'
+ description = 'Running'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ # set this in config
+ super().__init__(**kwargs)
+
+ #@defer.inlineCallbacks
+ def run(self):
+ self.setProperty('portage_repos_path', '/repositorys', 'portage_repos_path')
+ self.setProperty('rootworkdir', '/var/lib/buildbot_worker', 'rootworkdir')
+ return SUCCESS
+
class AddPackage(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
@@ -100,6 +117,7 @@ class TriggerCheckForV(BuildStep):
'category_data' : self.getProperty("category_data"),
'change_data' : self.getProperty("change_data"),
'project_data' : self.getProperty("project_data"),
+ 'cp_worker' : self.getProperty('workername'),
}
)
)
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-26 11:54 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-26 11:54 UTC (permalink / raw
To: gentoo-commits
commit: ae8716fd198fb5502781e4e6147f59bb19361b21
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 26 11:53:52 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jul 26 11:53:52 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ae8716fd
Set 1H for timeout
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index e139cf9..164bb24 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -120,7 +120,7 @@ class SetupParserBuildLoger(BuildStep):
self.aftersteps_list.append(steps.FileDownload(
mastersrc=mastersrc_log,
workerdest=log_cpv['full_logname']
- ))
+ ))
# Upload log parser py code
self.aftersteps_list.append(steps.FileDownload(
mastersrc=mastersrc_py,
@@ -145,7 +145,8 @@ class SetupParserBuildLoger(BuildStep):
flunkOnFailure = True,
command=command,
strip=False,
- extract_fn=PersOutputOfLogParser
+ extract_fn=PersOutputOfLogParser,
+ timeout=3600
))
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-13 20:01 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-13 20:01 UTC (permalink / raw
To: gentoo-commits
commit: 8d734b15cfcdc72915a0f2c63f186f4a5a4073b6
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 13 20:01:00 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jul 13 20:01:00 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8d734b15
Don't remove the repo dir
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/nodes.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py
index 2a25562..a77c878 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -172,12 +172,12 @@ class SetupStage4Steps(BuildStep):
workdir=self.getProperty("workerbase")
))
# remove the gentoo repo
- aftersteps_list.append(steps.ShellCommand(
- flunkOnFailure=True,
- name='Remove the repo dir',
- command=['rm', '-R', self.getProperty('project_uuid') + '/var/db/repos/gentoo'],
- workdir=self.getProperty("workerbase")
- ))
+ #aftersteps_list.append(steps.ShellCommand(
+ # flunkOnFailure=True,
+ # name='Remove the repo dir',
+ # command=['rm', '-R', self.getProperty('project_uuid') + '/var/db/repos/gentoo'],
+ # workdir=self.getProperty("workerbase")
+ #))
# compress it
aftersteps_list.append(steps.ShellCommand(
flunkOnFailure=True,
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-08 21:54 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-08 21:54 UTC (permalink / raw
To: gentoo-commits
commit: 26b8a9034b0ce05535fb8730a4a43dcf4f70aa14
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 8 21:53:50 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Jul 8 21:53:50 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=26b8a903
Fix a typo for Title
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index e5e61a5..6bd3279 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -301,7 +301,7 @@ class MakeIssue(BuildStep):
else:
cpv = self.getProperty('cpv')
self.error_dict['cpv'] = cpv
- yield log.addStdout('Titel:' + '\n')
+ yield log.addStdout('Title:' + '\n')
yield log.addStdout(separator2.join([cpv, '-', self.error_dict['title']]) + separator1)
yield log.addStdout('Summary:' + '\n')
for line in self.summary_log_list:
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-06 18:44 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-06 18:44 UTC (permalink / raw
To: gentoo-commits
commit: 86dae64f04e20bf74405496289564cd993429c03
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 6 18:44:08 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jul 6 18:44:08 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=86dae64f
Change title_issue and title_phase
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 2 +-
buildbot_gentoo_ci/steps/logs.py | 29 +++--------------------------
2 files changed, 4 insertions(+), 27 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index e43c926..27b1388 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -88,7 +88,7 @@ class GetBugs(BuildStep):
match = False
for bug in buglist:
yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' + bug['summary'] +'\n')
- if re.search(self.getProperty('error_dict')['title_phase'][:20], bug['summary']):
+ if re.search(self.getProperty('error_dict')['title_issue'][:20], bug['summary']):
print('Bug found')
print(bug)
match = {}
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 7a63658..e5e61a5 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -318,39 +318,16 @@ class MakeIssue(BuildStep):
text_phase_list = []
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
- if v['text'].startswith(' * ERROR:') and v['text'].endswith(' phase):'):
- issue_text = self.summary_log_dict[k + 1]['text']
- if issue_text.startswith(' * ninja -v -j'):
- issue_text = 'ninja failed'
- if issue_text.startswith(' * (no error'):
- issue_text = False
- if issue_text:
- text_issue_list.append(issue_text)
- # get the phase error
if v['type'] == self.error_dict['phase'] and v['status'] == 'error':
- text_phase_list.append(v['text'])
- # if not get the first issue
- if text_issue_list == []:
- for k, v in self.summary_log_dict.items():
- if v['type'] == 'issues':
- text_issue_list.append(v['text'])
- # if not get the first error
- if text_phase_list == []:
- for k, v in self.summary_log_dict.items():
- if v['status'] == 'error':
- text_phase_list.append(v['text'])
+ text_issue_list.append(v['text'])
# add the issue error
if text_issue_list != []:
self.error_dict['title_issue'] = text_issue_list[0].replace('*', '').strip()
else:
self.error_dict['title_issue'] = 'title_issue : None'
- # add the error line
- if text_phase_list != []:
- self.error_dict['title_phase'] = text_phase_list[0].replace('*', '').strip()
- else:
- self.error_dict['title_phase'] = 'title_phase : None'
+ self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
#set the error title
- self.error_dict['title'] = self.error_dict['title_issue'] + ' (' + self.error_dict['title_phase'] + ')'
+ self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' + self.error_dict['title_issue']
@defer.inlineCallbacks
def run(self):
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-07-04 17:44 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-07-04 17:44 UTC (permalink / raw
To: gentoo-commits
commit: 0486f29cc0842870d85029553f2c229799896fa5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 4 17:43:38 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jul 4 17:43:38 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0486f29c
Add support for search bugs on b.g.o
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/bugs.py | 124 +++++++++++++++++++++++++++++++++++++++
buildbot_gentoo_ci/steps/logs.py | 14 ++++-
2 files changed, 135 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
new file mode 100644
index 0000000..e43c926
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -0,0 +1,124 @@
+# Copyright 2022 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import re
+
+from twisted.internet import defer
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
+
+from bugz.cli import check_bugz_token, login, list_bugs
+from bugz.cli_argparser import make_arg_parser
+from bugz.configfile import load_config
+from bugz.settings import Settings
+from bugz.exceptions import BugzError
+from bugz.log import log_error, log_info
+
+from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
+
+# Origins: bugz.cli
+# Modifyed by Gentoo Authors.
+# main
+def main_bugz(args):
+ ArgParser = make_arg_parser()
+ opt = ArgParser.parse_args(args)
+
+ ConfigParser = load_config(getattr(opt, 'config_file', None))
+
+ check_bugz_token()
+ settings = Settings(opt, ConfigParser)
+ return settings
+
+# search
+def search_bugz(args):
+ settings = main_bugz(args)
+ valid_keys = ['alias', 'assigned_to', 'component', 'creator',
+ 'limit', 'offset', 'op_sys', 'platform',
+ 'priority', 'product', 'resolution', 'severity',
+ 'version', 'whiteboard', 'cc']
+
+ params = {}
+ d = vars(settings)
+ for key in d:
+ if key in valid_keys:
+ params[key] = d[key]
+ if 'search_statuses' in d:
+ if 'all' not in d['search_statuses']:
+ params['status'] = d['search_statuses']
+ if 'terms' in d:
+ params['summary'] = d['terms']
+
+ if not params:
+ raise BugzError('Please give search terms or options.')
+
+ log_info('Searching for bugs meeting the following criteria:')
+ for key in params:
+ log_info(' {0:<20} = {1}'.format(key, params[key]))
+
+ login(settings)
+
+ result = settings.call_bz(settings.bz.Bug.search, params)['bugs']
+
+ if not len(result):
+ log_info('No bugs found.')
+ return []
+ else:
+ list_bugs(result, settings)
+ return result
+
+class GetBugs(BuildStep):
+
+ name = 'GetBugs'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def find_match(self, buglist):
+ log = yield self.addLog('Bugs')
+ yield log.addStdout('Open Bugs\n')
+ match = False
+ for bug in buglist:
+ yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' + bug['summary'] +'\n')
+ if re.search(self.getProperty('error_dict')['title_phase'][:20], bug['summary']):
+ print('Bug found')
+ print(bug)
+ match = {}
+ match['id'] = bug['id']
+ match['summary'] = bug['summary']
+ if match:
+ yield log.addStdout('Match bug found\n')
+ yield log.addStdout('Bug: ' + str(match['id']) + ' Summary: ' + match['summary'] +'\n')
+ self.setProperty("bgo", match, 'bgo')
+ return
+ yield log.addStdout('NO Match bug found\n')
+ self.setProperty("bgo", False, 'bgo')
+
+ @defer.inlineCallbacks
+ def run(self):
+ # self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ cpv = self.getProperty('error_dict')['cpv']
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ cp = c + '/' + p
+ # search for open bugs
+ args = []
+ args.append('--skip-auth')
+ args.append('search')
+ # set limit
+ # set date last 30 days
+ # search for cp
+ args.append(cp)
+ print(args)
+ buglist = search_bugz(args)
+ print(buglist)
+ self.find_match(buglist)
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d6ee705..7a63658 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -22,6 +22,7 @@ from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
+from buildbot_gentoo_ci.steps import bugs
def PersOutputOfLogParser(rc, stdout, stderr):
build_summery_output = {}
@@ -299,6 +300,7 @@ class MakeIssue(BuildStep):
cpv = self.getProperty('faild_cpv')
else:
cpv = self.getProperty('cpv')
+ self.error_dict['cpv'] = cpv
yield log.addStdout('Titel:' + '\n')
yield log.addStdout(separator2.join([cpv, '-', self.error_dict['title']]) + separator1)
yield log.addStdout('Summary:' + '\n')
@@ -358,10 +360,11 @@ class MakeIssue(BuildStep):
warning = False
self.summary_log_list = []
self.error_dict = {}
- self.error_dict['hash'] = hashlib.sha256()
+ self.aftersteps_list = []
+ #self.error_dict['hash'] = hashlib.sha256()
for k, v in sorted(self.summary_log_dict.items()):
self.summary_log_list.append(v['text'])
- self.error_dict['hash'].update(v['text'].encode('utf-8'))
+ #self.error_dict['hash'].update(v['text'].encode('utf-8'))
if v['status'] == 'warning':
warning = True
# check if the build did fail
@@ -377,9 +380,14 @@ class MakeIssue(BuildStep):
print(self.error_dict)
yield self.logIssue()
self.setProperty("status", 'failed', 'status')
- self.setProperty("summary_log_list", self.summary_log_list, 'summary_log_list')
+ self.setProperty("error_dict", self.error_dict, 'error_dict')
+ self.aftersteps_list.append(bugs.GetBugs())
if warning:
self.setProperty("status", 'warning', 'status')
+ self.setProperty("bgo", False, 'bgo')
+ self.setProperty("summary_log_list", self.summary_log_list, 'summary_log_list')
+ if self.aftersteps_list is not []:
+ yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
class setBuildbotLog(BuildStep):
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-06-28 23:26 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-06-28 23:26 UTC (permalink / raw
To: gentoo-commits
commit: c556e56ad98d3e81fac06417661fd7edf2c1c9fb
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jun 28 23:26:22 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jun 28 23:26:22 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c556e56a
Change emerge retrys to 5 from 3
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index fa11290..dcd54c2 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -18,7 +18,7 @@ from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
#FIXME: should be set in config
-hosturl = 'http://77.110.8.67:8000'
+hosturl = 'http://90.231.13.235:8000'
def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output = {}
@@ -856,7 +856,8 @@ class CheckEmergeLogs(BuildStep):
if self.step == 'pre-build':
print(emerge_output)
# this should be set in the config
- if self.getProperty('rerun') <= 3:
+ retrays = 5
+ if self.getProperty('rerun') <= retrays:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
# but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-06-01 0:19 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-06-01 0:19 UTC (permalink / raw
To: gentoo-commits
commit: e15a995fa6e1a649f34ac98d446be3c4db686a9d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jun 1 00:14:18 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jun 1 00:14:18 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e15a995f
set build timeout to 6600
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 1d4f52d..fa11290 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -385,8 +385,7 @@ class RunEmerge(BuildStep):
self.descriptionSuffix = self.step
self.name = 'Setup emerge for ' + self.step + ' step'
self.build_env = {}
- #FIXME: Set build timeout in config
- self.build_timeout = 1800
+ self.build_timeout = 0
@defer.inlineCallbacks
def run(self):
@@ -399,6 +398,8 @@ class RunEmerge(BuildStep):
'-v'
]
aftersteps_list = []
+ #FIXME: Set build timeout in config
+ self.build_timeout = 6600
# set env
# https://bugs.gentoo.org/683118
# export TERM=linux
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-04-22 12:34 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-04-22 12:34 UTC (permalink / raw
To: gentoo-commits
commit: e09c9ab4e45d4a884ea6d2da386f5ea3c8f13c2c
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 22 12:34:35 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Apr 22 12:34:35 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e09c9ab4
Add support rootworkdir
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 14 +++++++++-----
buildbot_gentoo_ci/steps/portage.py | 29 ++++++++++++++++++++++-------
2 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 4ade33d..8ce6545 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -344,7 +344,8 @@ class UpdateRepos(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, **kwargs):
+ def __init__(self, workdir=False, **kwargs):
+ self.rootworkdir = workdir
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -356,14 +357,17 @@ class UpdateRepos(BuildStep):
projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
for project_repository_data in projects_repositorys_data:
repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
- repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
+ if self.rootworkdir:
+ repository_path = os.path.join(self.rootworkdir, portage_repos_path[1:], repository_data['name'])
+ else:
+ repository_path = os.path.join(portage_repos_path, repository_data['name'], '')
yield self.build.addStepsAfterCurrentStep([
steps.Git(repourl=repository_data['url'],
name = 'Git pull ' + repository_data['name'],
mode='full',
submodules=True,
alwaysUseLatest=True,
- workdir=os.path.join(repository_path, ''))
+ workdir=repository_path)
])
return SUCCESS
@@ -798,8 +802,8 @@ class CheckEmergeLogs(BuildStep):
print(cpv_build_dir)
self.setProperty('cpv_build_dir', cpv_build_dir, 'cpv_build_dir')
cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work')
- #FIXME: take find pattern from db or config
- find_pattern_list = ['meson-log.txt', 'CMakeCache.txt']
+ #FIXME: take find pattern from db
+ find_pattern_list = ['meson-log.txt', 'CMakeCache.txt', 'testlog.txt', '*.out', 'project-config.jam', 'testlog-x11.txt']
shell_commad_list = []
# we have *.log as default
shell_commad_list.append('find')
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 2332626..431fe9a 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -76,7 +76,8 @@ class SetMakeProfile(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, **kwargs):
+ def __init__(self, workdir=False, **kwargs):
+ self.rootworkdir = workdir
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -104,8 +105,12 @@ class SetMakeProfile(BuildStep):
'ln',
'-s'
]
+ if self.rootworkdir:
+ symlink_makeprofile_path = yield os.path.join(self.rootworkdir, 'etc/portage/make.profile')
+ else:
+ symlink_makeprofile_path = '/etc/portage/make.profile'
shell_commad_list.append(makeprofile_path)
- shell_commad_list.append('/etc/portage/make.profile')
+ shell_commad_list.append(symlink_makeprofile_path)
yield self.build.addStepsAfterCurrentStep([
steps.ShellCommand(
command=shell_commad_list,
@@ -125,7 +130,8 @@ class SetReposConf(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, **kwargs):
+ def __init__(self, workdir=False, **kwargs):
+ self.rootworkdir = workdir
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -133,6 +139,10 @@ class SetReposConf(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
portage_repos_path = self.getProperty('portage_repos_path')
project_data = self.getProperty('project_data')
+ if self.rootworkdir:
+ portage_etc_path = yield os.path.join(self.rootworkdir, 'etc/portage/')
+ else:
+ portage_etc_path = '/etc/portage/'
# setup the default.conf
repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'repos.conf')
if repos_conf_data is None:
@@ -149,7 +159,7 @@ class SetReposConf(BuildStep):
yield self.build.addStepsAfterCurrentStep([
steps.StringDownload(default_conf_string + separator,
workerdest="repos.conf/default.conf",
- workdir='/etc/portage/')
+ workdir=portage_etc_path)
])
# display the default.conf
yield log.addStdout('File: ' + 'default.conf' + '\n')
@@ -171,7 +181,7 @@ class SetReposConf(BuildStep):
yield self.build.addStepsAfterCurrentStep([
steps.StringDownload(repository_conf_string + separator,
workerdest='repos.conf/' + filename,
- workdir='/etc/portage/')
+ workdir=portage_etc_path)
])
yield log.addStdout('File: ' + filename + '\n')
for line in repository_conf:
@@ -187,7 +197,8 @@ class SetMakeConf(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, **kwargs):
+ def __init__(self, workdir=False, **kwargs):
+ self.rootworkdir = workdir
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -196,6 +207,10 @@ class SetMakeConf(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
makeconf_variables_data = yield self.gentooci.db.portages.getVariables()
+ if self.rootworkdir:
+ portage_etc_path = yield os.path.join(self.rootworkdir, 'etc/portage/')
+ else:
+ portage_etc_path = '/etc/portage/'
separator1 = '\n'
separator2 = ' '
makeconf_list = []
@@ -248,7 +263,7 @@ class SetMakeConf(BuildStep):
yield self.build.addStepsAfterCurrentStep([
steps.StringDownload(makeconf_string + separator1,
workerdest="make.conf",
- workdir='/etc/portage/')
+ workdir=portage_etc_path)
])
# display the make.conf
for line in makeconf_list:
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-01-28 10:09 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-01-28 10:09 UTC (permalink / raw
To: gentoo-commits
commit: adea4206026bfb0ef1cc9a5f2995a90e58fd9fa5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 28 10:09:12 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Jan 28 10:09:12 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=adea4206
Change use of repository_basedir and use pathlib for symlink
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 31 ++++++++++++++++++-------------
buildbot_gentoo_ci/steps/repos.py | 21 ++++++++++++++-------
buildbot_gentoo_ci/steps/version.py | 5 ++---
3 files changed, 34 insertions(+), 23 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index e5d0a25..2332626 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -3,6 +3,7 @@
import os
import io
+from pathlib import Path
from portage import config as portage_config
from portage import auxdbkeys
@@ -403,24 +404,29 @@ class CheckPathLocal(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- self.repository_linkname = self.gentooci.config.project['repository_basedir']
- self.repository_basedir2 = '/home/repos2/'
self.portage_path = yield os.path.join('etc', 'portage')
self.profile_path = yield os.path.join(self.portage_path, 'make.profile')
self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
- print(os.getcwd())
+ self.repository_basedir_db = yield os.path.join(self.master.basedir, 'repositorys')
+ self.build_repository_basedir_db = yield os.path.join(self.getProperty("builddir"), 'repositorys')
+ #print(self.repository_basedir_db)
+ log = yield self.addLog('CheckPathLocal')
+ #print(os.getcwd())
print(self.getProperty("builddir"))
- yield os.chdir(self.getProperty("builddir"))
- print(os.getcwd())
+ #yield os.chdir(self.getProperty("builddir"))
+ #print(os.getcwd())
for x in [
self.portage_path,
self.profile_path,
self.repos_path,
]:
- if not os.path.isdir(x):
- os.makedirs(x)
- if not os.path.islink(self.repository_linkname):
- os.symlink(self.repository_basedir2, self.repository_linkname)
+ check_dir = yield os.path.join(self.getProperty("builddir"), x)
+ if not Path(check_dir).is_dir():
+ yield Path(check_dir).mkdir(parents=True)
+ yield log.addStdout(' '.join(['Makeing missing dir', x]))
+ if not Path(self.build_repository_basedir_db).is_dir():
+ yield Path(self.build_repository_basedir_db).symlink_to(self.repository_basedir_db)
+ yield log.addStdout(' '.join(['Makeing missing link', 'repositorys', 'to', self.repository_basedir_db]))
return SUCCESS
class SetMakeProfileLocal(BuildStep):
@@ -442,7 +448,7 @@ class SetMakeProfileLocal(BuildStep):
return SKIPPED
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty('project_data')['profile_repository_uuid'])
- self.repository_basedir = self.gentooci.config.project['repository_basedir']
+ self.repository_basedir = 'repositorys'
makeprofiles_paths = []
makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'make.profile')
for makeprofile in makeprofiles_data:
@@ -468,8 +474,7 @@ class SetReposConfLocal(BuildStep):
repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf')
repos_conf_default_path = yield os.path.join(repos_conf_path, 'default.conf')
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- # the path should be set in the confg
- self.repository_basedir2 = '/home/repos2/'
+ self.repository_basedir_db = yield os.path.join(self.master.basedir, 'repositorys')
if not os.path.isfile(repos_conf_default_path):
# setup the default.conf
repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'repos.conf')
@@ -483,7 +488,7 @@ class SetReposConfLocal(BuildStep):
yield WriteTextToFile(repos_conf_default_path, default_conf)
repos_conf_repository_path = yield os.path.join(repos_conf_path, self.getProperty("repository_data")['name'] + '.conf')
if not os.path.isfile(repos_conf_repository_path):
- repository_path = yield os.path.join(self.repository_basedir2, self.getProperty("repository_data")['name'])
+ repository_path = yield os.path.join(self.repository_basedir_db, self.getProperty("repository_data")['name'])
repository_conf = []
repository_conf.append('[' + self.getProperty("repository_data")['name'] + ']')
repository_conf.append('location = ' + repository_path)
diff --git a/buildbot_gentoo_ci/steps/repos.py b/buildbot_gentoo_ci/steps/repos.py
index f5c12d2..ae63c78 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -3,6 +3,7 @@
import os
import git
+from pathlib import Path
from twisted.internet import defer
@@ -25,14 +26,20 @@ class CheckPathRepositoryLocal(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
+ @defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- # self.repository_basedir = self.gentooci.config.project['mirror_repository_basedir']
- repository_basedir = '/home/repos2/'
- self.setProperty("repository_basedir", repository_basedir, 'repository_basedir')
- if os.path.isdir(repository_basedir):
- return SUCCESS
- return FAILURE
+ self.repository_basedir_db = yield os.path.join(self.master.basedir, 'repositorys')
+ print(self.repository_basedir_db)
+ print(self.gentooci.config.project['repository_basedir'])
+ p = Path(self.repository_basedir_db)
+ self.setProperty("repository_basedir_db", self.repository_basedir_db, 'repository_basedir_db')
+ log = yield self.addLog('CheckPathRepositoryLocal')
+ if not Path(self.repository_basedir_db).is_dir():
+ yield log.addStdout(' '.join(['Missing link', self.repository_basedir_db]))
+ p.symlink_to(self.gentooci.config.project['repository_basedir'])
+ yield log.addStdout(' '.join(['Makeing missing link', 'repositorys', 'to', self.gentooci.config.project['repository_basedir']]))
+ return SUCCESS
class CheckRepository(BuildStep):
@@ -105,7 +112,7 @@ class CheckRepository(BuildStep):
@defer.inlineCallbacks
def checkRepos(self, repository_data):
success = False
- repository_path = yield os.path.join(self.getProperty("repository_basedir"), repository_data['name'])
+ repository_path = yield os.path.join(self.getProperty("repository_basedir_db"), repository_data['name'])
try:
repo = git.Repo(repository_path)
except:
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index 2b0c1e1..86d5963 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -230,9 +230,8 @@ class CheckPathHash(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- self.repository_basedir = self.gentooci.config.project['repository_basedir']
- self.repository_path = yield os.path.join('/home', 'repos2', self.getProperty("repository_data")['name'])
- #self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'])
+ self.repository_basedir_db = yield os.path.join(self.master.basedir, 'repositorys')
+ self.repository_path = yield os.path.join(self.repository_basedir_db, self.getProperty("repository_data")['name'])
self.cp_path = yield pkgsplit(self.getProperty("cpv"))[0]
self.file_name = yield self.getProperty("package_data")['name'] + '-' + self.getProperty("version") + '.ebuild'
self.ebuild_file = yield os.path.join(self.repository_path, self.cp_path, self.file_name)
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-01-24 8:41 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-01-24 8:41 UTC (permalink / raw
To: gentoo-commits
commit: d3fb5e15c10d25fa06c6862fbbcd996a2d6182df
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 24 08:41:34 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 24 08:41:34 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d3fb5e15
set ACCEPT_LICENSE by project
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 08ab559..e5d0a25 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -224,8 +224,9 @@ class SetMakeConf(BuildStep):
for v in makeconf_variables_values_data:
if v['build_id'] == 0:
makeconf_variable_list.append(v['value'])
- if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list != []:
- makeconf_variable_list.append('ACCEPT_LICENSE="*"')
+ #NOTE: set it by project
+ #if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list != []:
+ # makeconf_variable_list.append('ACCEPT_LICENSE="*"')
if makeconf_variable_list != []:
makeconf_variable_string = k['variable'] + '="' + separator2.join(makeconf_variable_list) + '"'
makeconf_list.append(makeconf_variable_string)
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-01-24 0:42 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-01-24 0:42 UTC (permalink / raw
To: gentoo-commits
commit: 49658ae9a6842348fbad6d0674a7a7daf7aa0a37
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 24 00:38:42 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 24 00:38:42 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=49658ae9
add url for logs downloads
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 69 ++++++++++++++++++++++++------------
1 file changed, 46 insertions(+), 23 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 527c2b3..4ade33d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -17,6 +17,9 @@ from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
+#FIXME: should be set in config
+hosturl = 'http://77.110.8.67:8000'
+
def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output = {}
emerge_output['rc'] = rc
@@ -603,12 +606,14 @@ class CheckElogLogs(BuildStep):
super().__init__(**kwargs)
self.aftersteps_list = []
- def addFileUploade(self, sourcefile, destfile, name):
+ def addFileUploade(self, sourcefile, destfile, name, url, urlText):
self.aftersteps_list.append(steps.FileUpload(
name = name,
mode = 0o644,
- workersrc=sourcefile,
- masterdest=destfile
+ workersrc = sourcefile,
+ masterdest = destfile,
+ url = url,
+ urlText = urlText
))
@defer.inlineCallbacks
@@ -625,7 +630,9 @@ class CheckElogLogs(BuildStep):
destfile = yield os.path.join(self.getProperty('masterdest'), elogfile.replace('.log', '.elog'))
sourcefile = yield os.path.join(workdir, elogfile)
name = 'Upload Elogs'
- self.addFileUploade(sourcefile, destfile, name)
+ url = '/'.join([hosturl, self.getProperty('workername'), str(self.getProperty("buildnumber")), elogfile.replace('.log', '.elog')])
+ urlText = elogfile
+ self.addFileUploade(sourcefile, destfile, name, url, urlText)
if self.aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
@@ -669,7 +676,9 @@ class CheckBuildWorkDirs(BuildStep):
mode = 0o644,
workersrc = compressed_log_file,
masterdest = masterdest_file,
- workdir = cpv_build_dir
+ workdir = cpv_build_dir,
+ url = '/'.join([hosturl, self.getProperty('workername'), str(self.getProperty("buildnumber")), compressed_log_file]),
+ urlText = 'Compresed file for all finds'
))
if self.aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
@@ -710,24 +719,29 @@ class CheckEmergeLogs(BuildStep):
workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'))
self.aftersteps_list.append(steps.MasterShellCommand(
name = 'Make directory for Uploaded files',
- command=['mkdir', str(self.getProperty("buildnumber"))],
- workdir=workdir
+ command = ['mkdir', str(self.getProperty("buildnumber"))],
+ workdir = workdir
))
- def addFileUploade(self, sourcefile, destfile, name):
+ def addFileUploade(self, sourcefile, destfile, name, url, urlText):
self.aftersteps_list.append(steps.FileUpload(
name = name,
mode = 0o644,
- workersrc=sourcefile,
- masterdest=destfile
+ workersrc = sourcefile,
+ masterdest = destfile,
+ url=url,
+ urlText=urlText
))
@defer.inlineCallbacks
def getLogFile(self, cpv, log_dict):
- destfile = yield os.path.join(self.getProperty('masterdest'), log_dict[cpv]['full_logname'])
+ file = log_dict[cpv]['full_logname']
+ destfile = yield os.path.join(self.getProperty('masterdest'), file)
sourcefile = log_dict[cpv]['log_path']
name = 'Upload build log'
- self.addFileUploade(sourcefile, destfile, name)
+ url = '/'.join([hosturl, self.getProperty('workername'), str(self.getProperty("buildnumber")), file])
+ urlText = file
+ self.addFileUploade(sourcefile, destfile, name, url, urlText)
@defer.inlineCallbacks
def getElogFiles(self, cpv):
@@ -750,20 +764,29 @@ class CheckEmergeLogs(BuildStep):
@defer.inlineCallbacks
def getEmergeFiles(self, cpv):
# get emerge info
- destfile = yield os.path.join(self.getProperty('masterdest'), 'emerge_info.txt')
- sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
+ file = 'emerge_info.txt'
+ destfile = yield os.path.join(self.getProperty('masterdest'), file)
+ sourcefile = yield os.path.join('/', 'tmp', file)
name = 'Upload emerge info'
- self.addFileUploade(sourcefile, destfile, name)
+ url = '/'.join([hosturl, self.getProperty('workername'), str(self.getProperty("buildnumber")), file])
+ urlText = 'emerge --info and package info'
+ self.addFileUploade(sourcefile, destfile, name, url, urlText)
# get emerge.log
- destfile2 = yield os.path.join(self.getProperty('masterdest'), 'emerge.log')
- sourcefile2 = yield os.path.join('/', 'var', 'log', 'emerge.log')
- name2 = 'Upload emerge log'
- self.addFileUploade(sourcefile2, destfile2, name2)
+ file = 'emerge.log'
+ destfile = yield os.path.join(self.getProperty('masterdest'), file)
+ sourcefile = yield os.path.join('/', 'var', 'log', file)
+ name = 'Upload emerge log'
+ url = '/'.join([hosturl, self.getProperty('workername'), str(self.getProperty("buildnumber")), file])
+ urlText = 'emerge.log'
+ self.addFileUploade(sourcefile, destfile, name, url, urlText)
# world file
- destfile3 = yield os.path.join(self.getProperty('masterdest'), 'world')
- sourcefile3 = yield os.path.join('/', 'var', 'lib', 'portage', 'world')
- name3 = 'Upload world file'
- self.addFileUploade(sourcefile3, destfile3, name3)
+ file = 'world'
+ destfile = yield os.path.join(self.getProperty('masterdest'), file)
+ sourcefile = yield os.path.join('/', 'var', 'lib', 'portage', file)
+ name = 'Upload world file'
+ url = '/'.join([hosturl, self.getProperty('workername'), str(self.getProperty("buildnumber")), file])
+ urlText = 'world file'
+ self.addFileUploade(sourcefile, destfile, name, url, urlText)
# get elogs
self.getElogFiles(cpv)
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-01-22 20:50 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-01-22 20:50 UTC (permalink / raw
To: gentoo-commits
commit: 2d640d0aa09a6d662d30ee83914276d5d7d08556
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Jan 22 20:49:55 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Jan 22 20:49:55 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2d640d0a
Fix naming on alot of steps in run_build_request and warning for pre-build
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 70 ++++++++++++++++++++++++------------
1 file changed, 48 insertions(+), 22 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index f2fe16c..527c2b3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -289,10 +289,8 @@ class GetProjectRepositoryData(BuildStep):
return SUCCESS
class SetupPropertys(BuildStep):
-
- name = 'SetupPropertys'
+ name = 'Setup propertys for building'
description = 'Running'
- #descriptionDone = 'Ran'
descriptionSuffix = None
haltOnFailure = True
flunkOnFailure = True
@@ -305,7 +303,6 @@ class SetupPropertys(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
- self.descriptionDone = 'Building this %s' % self.getProperty("cpv")
self.setProperty('portage_repos_path', self.gentooci.config.project['project']['worker_portage_repos_path'], 'portage_repos_path')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
@@ -332,7 +329,7 @@ class SetupPropertys(BuildStep):
print(self.getProperty("project_build_data"))
self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
self.setProperty('masterdest', self.masterdest, 'masterdest')
-
+ self.descriptionDone = ' '.join([self.getProperty("cpv"), 'for project', self.getProperty('project_data')['name']])
return SUCCESS
class UpdateRepos(BuildStep):
@@ -359,6 +356,7 @@ class UpdateRepos(BuildStep):
repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
yield self.build.addStepsAfterCurrentStep([
steps.Git(repourl=repository_data['url'],
+ name = 'Git pull ' + repository_data['name'],
mode='full',
submodules=True,
alwaysUseLatest=True,
@@ -368,7 +366,6 @@ class UpdateRepos(BuildStep):
class RunEmerge(BuildStep):
- name = 'RunEmerge'
description = 'Running'
descriptionDone = 'Ran'
haltOnFailure = True
@@ -378,6 +375,7 @@ class RunEmerge(BuildStep):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
+ self.name = 'Setup emerge for ' + self.step + ' step'
self.build_env = {}
@defer.inlineCallbacks
@@ -385,6 +383,7 @@ class RunEmerge(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
+ self.stepname = 'Run emerge ' + 'step ' + self.step
shell_commad_list = [
'emerge',
'-v'
@@ -423,6 +422,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -442,6 +442,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -457,6 +458,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('@preserved-rebuild')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -471,6 +473,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--depclean')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfDepclean,
@@ -487,6 +490,7 @@ class RunEmerge(BuildStep):
pass
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfDepclean,
@@ -516,6 +520,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append(c + '/' + p)
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -540,6 +545,11 @@ class RunEmerge(BuildStep):
shell_commad_list.append('-p')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ warnOnWarnings = True,
+ warnOnFailure = True,
+ flunkOnFailure = False,
+ flunkOnWarnings = False,
+ name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -565,6 +575,7 @@ class RunEmerge(BuildStep):
shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name = self.stepname,
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfEmerge,
@@ -582,7 +593,7 @@ class RunEmerge(BuildStep):
class CheckElogLogs(BuildStep):
- name = 'CheckElogLogs'
+ name = 'Check elog logs'
description = 'Running'
descriptionDone = 'Ran'
haltOnFailure = True
@@ -592,8 +603,9 @@ class CheckElogLogs(BuildStep):
super().__init__(**kwargs)
self.aftersteps_list = []
- def addFileUploade(self, sourcefile, destfile):
+ def addFileUploade(self, sourcefile, destfile, name):
self.aftersteps_list.append(steps.FileUpload(
+ name = name,
mode = 0o644,
workersrc=sourcefile,
masterdest=destfile
@@ -612,14 +624,15 @@ class CheckElogLogs(BuildStep):
print(elogfile)
destfile = yield os.path.join(self.getProperty('masterdest'), elogfile.replace('.log', '.elog'))
sourcefile = yield os.path.join(workdir, elogfile)
- self.addFileUploade(sourcefile, destfile)
+ name = 'Upload Elogs'
+ self.addFileUploade(sourcefile, destfile, name)
if self.aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
class CheckBuildWorkDirs(BuildStep):
- name = 'CheckBuildWorkdir'
+ name = 'Setup tar for taring the logs'
description = 'Running'
descriptionDone = 'Ran'
haltOnFailure = True
@@ -647,12 +660,12 @@ class CheckBuildWorkDirs(BuildStep):
shell_commad_list.append(filename)
self.aftersteps_list.append(
steps.ShellCommand(
- name = 'Tarlogs',
+ name = 'Tar logs',
command = shell_commad_list,
workdir = cpv_build_dir
))
self.aftersteps_list.append(steps.FileUpload(
- name = 'UploadFindlogs',
+ name = 'Upload find logs',
mode = 0o644,
workersrc = compressed_log_file,
masterdest = masterdest_file,
@@ -664,7 +677,6 @@ class CheckBuildWorkDirs(BuildStep):
class CheckEmergeLogs(BuildStep):
- name = 'CheckEmergeLogs'
description = 'Running'
descriptionDone = 'Ran'
haltOnFailure = True
@@ -674,6 +686,7 @@ class CheckEmergeLogs(BuildStep):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
+ self.name = 'Check emerge logs for ' + self.step + ' step'
self.aftersteps_list = []
self.log_data = {}
self.faild_cpv = False
@@ -696,12 +709,14 @@ class CheckEmergeLogs(BuildStep):
def createDistDir(self):
workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'))
self.aftersteps_list.append(steps.MasterShellCommand(
+ name = 'Make directory for Uploaded files',
command=['mkdir', str(self.getProperty("buildnumber"))],
workdir=workdir
))
- def addFileUploade(self, sourcefile, destfile):
+ def addFileUploade(self, sourcefile, destfile, name):
self.aftersteps_list.append(steps.FileUpload(
+ name = name,
mode = 0o644,
workersrc=sourcefile,
masterdest=destfile
@@ -711,7 +726,8 @@ class CheckEmergeLogs(BuildStep):
def getLogFile(self, cpv, log_dict):
destfile = yield os.path.join(self.getProperty('masterdest'), log_dict[cpv]['full_logname'])
sourcefile = log_dict[cpv]['log_path']
- self.addFileUploade(sourcefile, destfile)
+ name = 'Upload build log'
+ self.addFileUploade(sourcefile, destfile, name)
@defer.inlineCallbacks
def getElogFiles(self, cpv):
@@ -722,6 +738,7 @@ class CheckEmergeLogs(BuildStep):
#shell_commad_list.append(elog_cpv + '*')
self.aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name = 'List elogs',
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfElogLs,
@@ -735,15 +752,18 @@ class CheckEmergeLogs(BuildStep):
# get emerge info
destfile = yield os.path.join(self.getProperty('masterdest'), 'emerge_info.txt')
sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
- self.addFileUploade(sourcefile, destfile)
+ name = 'Upload emerge info'
+ self.addFileUploade(sourcefile, destfile, name)
# get emerge.log
destfile2 = yield os.path.join(self.getProperty('masterdest'), 'emerge.log')
sourcefile2 = yield os.path.join('/', 'var', 'log', 'emerge.log')
- self.addFileUploade(sourcefile2, destfile2)
+ name2 = 'Upload emerge log'
+ self.addFileUploade(sourcefile2, destfile2, name2)
# world file
destfile3 = yield os.path.join(self.getProperty('masterdest'), 'world')
sourcefile3 = yield os.path.join('/', 'var', 'lib', 'portage', 'world')
- self.addFileUploade(sourcefile3, destfile3)
+ name3 = 'Upload world file'
+ self.addFileUploade(sourcefile3, destfile3, name3)
# get elogs
self.getElogFiles(cpv)
@@ -768,7 +788,7 @@ class CheckEmergeLogs(BuildStep):
shell_commad_list.append(find_pattern)
self.aftersteps_list.append(
steps.SetPropertyFromCommand(
- name = 'FindLogs',
+ name = 'Find logs',
command = shell_commad_list,
strip = True,
extract_fn = PersOutputOfBuildWorkdir,
@@ -828,6 +848,7 @@ class CheckEmergeLogs(BuildStep):
change_use_string = separator2.join(change_use_list)
self.aftersteps_list.append(
steps.StringDownload(change_use_string + separator,
+ name = 'Update package.use flags',
workerdest='zz_autouse' + str(self.getProperty('rerun')),
workdir='/etc/portage/package.use/'
)
@@ -938,6 +959,7 @@ class CheckEmergeLogs(BuildStep):
else:
self.getEmergeFiles(cpv)
self.aftersteps_list.append(steps.Trigger(
+ name = 'Setup properties for log parser and trigger it',
schedulerNames=['parse_build_log'],
waitForFinish=False,
updateSourceStamp=False,
@@ -969,6 +991,7 @@ class CheckDepcleanLogs(BuildStep):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
+ self.name = 'Check dep clean logs for ' + self.step + ' step'
@defer.inlineCallbacks
def run(self):
@@ -993,7 +1016,7 @@ class CheckDepcleanLogs(BuildStep):
class RunPkgCheck(BuildStep):
- name = 'RunPkgCheck'
+ name = 'Setup PkgCheck step'
description = 'Running'
descriptionDone = 'Ran'
descriptionSuffix = None
@@ -1029,6 +1052,7 @@ class RunPkgCheck(BuildStep):
pkgcheck_workdir = yield os.path.join(repository_path, c, p, '')
aftersteps_list.append(
steps.SetPropertyFromCommand(
+ name='Run pkgcheck step',
command=shell_commad_list,
strip=True,
extract_fn=PersOutputOfPkgCheck,
@@ -1040,7 +1064,7 @@ class RunPkgCheck(BuildStep):
class CheckPkgCheckLogs(BuildStep):
- name = 'CheckPkgCheckLogs'
+ name = 'Check pkgcheck logs'
description = 'Running'
descriptionDone = 'Ran'
descriptionSuffix = None
@@ -1094,6 +1118,7 @@ class RunEmergeInfo(BuildStep):
shell_commad_list.append('/tmp/emerge_info.txt')
aftersteps_list.append(
steps.ShellCommand(
+ name ='emerge --info',
# the list need to be joined to pipe to a file
command=' '.join(shell_commad_list),
workdir='/'
@@ -1115,6 +1140,7 @@ class RunEmergeInfo(BuildStep):
shell_commad_list.append('/tmp/emerge_info.txt')
aftersteps_list.append(
steps.ShellCommand(
+ name = 'Package info',
# the list need to be joined to pipe to a file
command=' '.join(shell_commad_list),
workdir='/'
@@ -1124,7 +1150,7 @@ class RunEmergeInfo(BuildStep):
class RunBuild(BuildStep):
- name = 'RunBuild'
+ name = 'Setup steps for building package'
description = 'Running'
descriptionDone = 'Ran'
descriptionSuffix = None
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-01-18 20:20 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-01-18 20:20 UTC (permalink / raw
To: gentoo-commits
commit: d3d07a68aa2c1bcba40c613f68867054bd71942d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 18 20:20:13 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 18 20:20:13 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d3d07a68
Use tee instead of >> on emerge -qpv0 cpv
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 5088c4a..f2fe16c 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1109,7 +1109,9 @@ class RunEmergeInfo(BuildStep):
shell_commad_list.append('=' + self.getProperty('cpv'))
shell_commad_list.append('--usepkg-exclude')
shell_commad_list.append(c + '/' + p)
- shell_commad_list.append('>>')
+ shell_commad_list.append('|')
+ shell_commad_list.append('tee')
+ shell_commad_list.append('-a')
shell_commad_list.append('/tmp/emerge_info.txt')
aftersteps_list.append(
steps.ShellCommand(
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-01-18 19:38 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-01-18 19:38 UTC (permalink / raw
To: gentoo-commits
commit: d319ffb16f4efcf62a7e3cee640f6af28f616021
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 18 19:38:31 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 18 19:38:31 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d319ffb1
Use tee instead of > on emerge info
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index dd2fbc2..5088c4a 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1089,7 +1089,8 @@ class RunEmergeInfo(BuildStep):
'emerge',
]
shell_commad_list.append('--info')
- shell_commad_list.append('>')
+ shell_commad_list.append('|')
+ shell_commad_list.append('tee')
shell_commad_list.append('/tmp/emerge_info.txt')
aftersteps_list.append(
steps.ShellCommand(
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-01-18 19:16 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-01-18 19:16 UTC (permalink / raw
To: gentoo-commits
commit: 1fc28f8dc77b5ef0c9d2d6eac054d91b455d9048
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 18 19:16:44 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 18 19:16:44 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1fc28f8d
Add support for find logfiles
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 110 ++++++++++++++++++++++++++++++-----
1 file changed, 97 insertions(+), 13 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index d8314e4..dd2fbc2 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -181,6 +181,19 @@ def PersOutputOfElogLs(rc, stdout, stderr):
'elog_ls_output' : elog_ls_output
}
+def PersOutputOfBuildWorkdir(rc, stdout, stderr):
+ build_workdir_find_output = {}
+ build_workdir_find_output['rc'] = rc
+ build_workdir_find_list = []
+ for line in stdout.split('\n'):
+ find_line = line.replace('./', '')
+ if find_line != '':
+ build_workdir_find_list.append(find_line)
+ build_workdir_find_output['build_workdir_find'] = build_workdir_find_list
+ return {
+ 'build_workdir_find_output' : build_workdir_find_output
+ }
+
class TriggerRunBuildRequest(BuildStep):
name = 'TriggerRunBuildRequest'
@@ -317,6 +330,9 @@ class SetupPropertys(BuildStep):
project_build_data['buildbot_build_id'])
self.setProperty('project_build_data', project_build_data, 'project_build_data')
print(self.getProperty("project_build_data"))
+ self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
+ self.setProperty('masterdest', self.masterdest, 'masterdest')
+
return SUCCESS
class UpdateRepos(BuildStep):
@@ -578,6 +594,7 @@ class CheckElogLogs(BuildStep):
def addFileUploade(self, sourcefile, destfile):
self.aftersteps_list.append(steps.FileUpload(
+ mode = 0o644,
workersrc=sourcefile,
masterdest=destfile
))
@@ -600,6 +617,51 @@ class CheckElogLogs(BuildStep):
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
+class CheckBuildWorkDirs(BuildStep):
+
+ name = 'CheckBuildWorkdir'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.aftersteps_list = []
+
+ @defer.inlineCallbacks
+ def run(self):
+ cpv = self.getProperty('faild_cpv')
+ cpv_build_dir = yield os.path.join('/', 'var', 'tmp', 'portage', self.getProperty('cpv_build_dir'))
+ compressed_log_file = cpv.replace('/', '_') + '.' + str(self.getProperty("buildnumber")) + '.logs.tar.bz2'
+ masterdest_file = yield os.path.join(self.getProperty('masterdest'), compressed_log_file)
+ # cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work')
+ if self.getProperty('build_workdir_find_output')['build_workdir_find'] != []:
+ shell_commad_list = []
+ shell_commad_list.append('tar')
+ shell_commad_list.append('-cjpf')
+ shell_commad_list.append(compressed_log_file)
+ for find_line in sorted(self.getProperty('build_workdir_find_output')['build_workdir_find']):
+ print(find_line)
+ filename = yield os.path.join('work', find_line)
+ shell_commad_list.append(filename)
+ self.aftersteps_list.append(
+ steps.ShellCommand(
+ name = 'Tarlogs',
+ command = shell_commad_list,
+ workdir = cpv_build_dir
+ ))
+ self.aftersteps_list.append(steps.FileUpload(
+ name = 'UploadFindlogs',
+ mode = 0o644,
+ workersrc = compressed_log_file,
+ masterdest = masterdest_file,
+ workdir = cpv_build_dir
+ ))
+ if self.aftersteps_list != []:
+ yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+ return SUCCESS
+
class CheckEmergeLogs(BuildStep):
name = 'CheckEmergeLogs'
@@ -640,13 +702,14 @@ class CheckEmergeLogs(BuildStep):
def addFileUploade(self, sourcefile, destfile):
self.aftersteps_list.append(steps.FileUpload(
+ mode = 0o644,
workersrc=sourcefile,
masterdest=destfile
))
@defer.inlineCallbacks
def getLogFile(self, cpv, log_dict):
- destfile = yield os.path.join(self.masterdest, log_dict[cpv]['full_logname'])
+ destfile = yield os.path.join(self.getProperty('masterdest'), log_dict[cpv]['full_logname'])
sourcefile = log_dict[cpv]['log_path']
self.addFileUploade(sourcefile, destfile)
@@ -670,42 +733,62 @@ class CheckEmergeLogs(BuildStep):
@defer.inlineCallbacks
def getEmergeFiles(self, cpv):
# get emerge info
- destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
+ destfile = yield os.path.join(self.getProperty('masterdest'), 'emerge_info.txt')
sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
self.addFileUploade(sourcefile, destfile)
# get emerge.log
- destfile2 = yield os.path.join(self.masterdest, 'emerge.log')
+ destfile2 = yield os.path.join(self.getProperty('masterdest'), 'emerge.log')
sourcefile2 = yield os.path.join('/', 'var', 'log', 'emerge.log')
self.addFileUploade(sourcefile2, destfile2)
# world file
- destfile3 = yield os.path.join(self.masterdest, 'world')
+ destfile3 = yield os.path.join(self.getProperty('masterdest'), 'world')
sourcefile3 = yield os.path.join('/', 'var', 'lib', 'portage', 'world')
self.addFileUploade(sourcefile3, destfile3)
# get elogs
self.getElogFiles(cpv)
- def getBuildWorkdirFiles(self):
+ @defer.inlineCallbacks
+ def getBuildWorkDirs(self, cpv):
#FIXME:
# get files from the build workdir
- pass
+ cpv_build_dir = yield os.path.join('/', 'var', 'tmp', 'portage', cpv)
+ print(cpv_build_dir)
+ self.setProperty('cpv_build_dir', cpv_build_dir, 'cpv_build_dir')
+ cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work')
+ #FIXME: take find pattern from db or config
+ find_pattern_list = ['meson-log.txt', 'CMakeCache.txt']
+ shell_commad_list = []
+ # we have *.log as default
+ shell_commad_list.append('find')
+ shell_commad_list.append('-name')
+ shell_commad_list.append('*.log')
+ for find_pattern in find_pattern_list:
+ shell_commad_list.append('-o')
+ shell_commad_list.append('-name')
+ shell_commad_list.append(find_pattern)
+ self.aftersteps_list.append(
+ steps.SetPropertyFromCommand(
+ name = 'FindLogs',
+ command = shell_commad_list,
+ strip = True,
+ extract_fn = PersOutputOfBuildWorkdir,
+ workdir = cpv_build_work_dir
+ ))
+ self.aftersteps_list.append(CheckBuildWorkDirs())
@defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
- emerge_output = self.getProperty('emerge_output')
- self.faild_cpv = emerge_output['failed']
- self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
shell_commad_list = [
'emerge',
'-v'
]
+ emerge_output = self.getProperty('emerge_output')
+ self.faild_cpv = emerge_output['failed']
package_dict = emerge_output['packages']
- self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
- self.setProperty('masterdest', self.masterdest, 'masterdest')
-
#FIXME: Prosees the logs and do stuff
# preserved-libs
if emerge_output['preserved_libs'] and projects_emerge_options['preserved_libs']:
@@ -849,8 +932,9 @@ class CheckEmergeLogs(BuildStep):
self.log_data[self.faild_cpv] = log_dict[self.faild_cpv]
yield self.getLogFile(self.faild_cpv, log_dict)
faild_version_data = yield self.getVersionData(self.faild_cpv)
+ self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
self.getEmergeFiles(self.faild_cpv)
- self.getBuildWorkdirFiles()
+ self.getBuildWorkDirs(self.faild_cpv)
else:
self.getEmergeFiles(cpv)
self.aftersteps_list.append(steps.Trigger(
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2022-01-08 17:25 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2022-01-08 17:25 UTC (permalink / raw
To: gentoo-commits
commit: 5e9aea3e883a20bc0adedceeeca2767230dd0ea3
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Jan 8 17:24:59 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Jan 8 17:24:59 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5e9aea3e
Upload emerge and elogs
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 126 +++++++++++++++++++++++++----------
1 file changed, 91 insertions(+), 35 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index e76fe3d..d8314e4 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -170,6 +170,17 @@ def PersOutputOfEmergeInfo(rc, stdout, stderr):
'emerge_info_output' : emerge_info_output
}
+def PersOutputOfElogLs(rc, stdout, stderr):
+ elog_ls_output = {}
+ elog_ls_output['rc'] = rc
+ elog_ls_list = []
+ for line in stdout.split('\n'):
+ elog_ls_list.append(line)
+ elog_ls_output['elog_ls'] = elog_ls_list
+ return {
+ 'elog_ls_output' : elog_ls_output
+ }
+
class TriggerRunBuildRequest(BuildStep):
name = 'TriggerRunBuildRequest'
@@ -553,6 +564,42 @@ class RunEmerge(BuildStep):
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+class CheckElogLogs(BuildStep):
+
+ name = 'CheckElogLogs'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.aftersteps_list = []
+
+ def addFileUploade(self, sourcefile, destfile):
+ self.aftersteps_list.append(steps.FileUpload(
+ workersrc=sourcefile,
+ masterdest=destfile
+ ))
+
+ @defer.inlineCallbacks
+ def run(self):
+ elog_ls_output = self.getProperty('elog_ls_output')
+ workdir = yield os.path.join('/', 'var', 'cache', 'portage', 'logs', 'elog')
+ for elogfile in elog_ls_output['elog_ls']:
+ if self.getProperty('faild_cpv'):
+ cpv = self.getProperty('faild_cpv')
+ else:
+ cpv = self.getProperty('cpv')
+ if elogfile.replace(':', '/').startswith(cpv):
+ print(elogfile)
+ destfile = yield os.path.join(self.getProperty('masterdest'), elogfile.replace('.log', '.elog'))
+ sourcefile = yield os.path.join(workdir, elogfile)
+ self.addFileUploade(sourcefile, destfile)
+ if self.aftersteps_list != []:
+ yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+ return SUCCESS
+
class CheckEmergeLogs(BuildStep):
name = 'CheckEmergeLogs'
@@ -567,6 +614,7 @@ class CheckEmergeLogs(BuildStep):
self.descriptionSuffix = self.step
self.aftersteps_list = []
self.log_data = {}
+ self.faild_cpv = False
@defer.inlineCallbacks
def getVersionData(self, cpv):
@@ -603,16 +651,38 @@ class CheckEmergeLogs(BuildStep):
self.addFileUploade(sourcefile, destfile)
@defer.inlineCallbacks
- def getEmergeFiles(self, faild_version_data):
+ def getElogFiles(self, cpv):
+ workdir = yield os.path.join('/', 'var', 'cache', 'portage', 'logs', 'elog')
+ elog_cpv = cpv.replace('/', ':')
+ shell_commad_list = []
+ shell_commad_list.append('ls')
+ #shell_commad_list.append(elog_cpv + '*')
+ self.aftersteps_list.append(
+ steps.SetPropertyFromCommand(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfElogLs,
+ workdir=workdir,
+ timeout=None
+ ))
+ self.aftersteps_list.append(CheckElogLogs())
+
+ @defer.inlineCallbacks
+ def getEmergeFiles(self, cpv):
# get emerge info
destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
self.addFileUploade(sourcefile, destfile)
- #FIXME:
- # if faild_version_data:
# get emerge.log
- # get elogs
+ destfile2 = yield os.path.join(self.masterdest, 'emerge.log')
+ sourcefile2 = yield os.path.join('/', 'var', 'log', 'emerge.log')
+ self.addFileUploade(sourcefile2, destfile2)
# world file
+ destfile3 = yield os.path.join(self.masterdest, 'world')
+ sourcefile3 = yield os.path.join('/', 'var', 'lib', 'portage', 'world')
+ self.addFileUploade(sourcefile3, destfile3)
+ # get elogs
+ self.getElogFiles(cpv)
def getBuildWorkdirFiles(self):
#FIXME:
@@ -625,6 +695,8 @@ class CheckEmergeLogs(BuildStep):
project_data = self.getProperty('project_data')
projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
emerge_output = self.getProperty('emerge_output')
+ self.faild_cpv = emerge_output['failed']
+ self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
shell_commad_list = [
'emerge',
'-v'
@@ -632,6 +704,7 @@ class CheckEmergeLogs(BuildStep):
package_dict = emerge_output['packages']
self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
+ self.setProperty('masterdest', self.masterdest, 'masterdest')
#FIXME: Prosees the logs and do stuff
# preserved-libs
@@ -730,8 +803,8 @@ class CheckEmergeLogs(BuildStep):
else:
# trigger parse_build_log with info about pre-build and it fail
pass
- # Check if extra build did work
- if self.step == 'extra-build':
+ # Make Logfile dict
+ if self.step == 'extra-build' or self.step == 'build':
print(emerge_output)
log_dict = {}
# get cpv, logname and log path
@@ -746,11 +819,8 @@ class CheckEmergeLogs(BuildStep):
full_logname = full_logname
)
print(log_dict)
- # Find log for cpv that was requested or did failed
- if not log_dict == {}:
- # requested cpv
- print(log_dict)
- faild_cpv = emerge_output['failed']
+ if self.step == 'extra-build':
+ #FIXME: Check if extra build did work
self.aftersteps_list.append(RunEmerge(step='pre-build'))
self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
@@ -760,43 +830,29 @@ class CheckEmergeLogs(BuildStep):
# local_log_path dir set in config
# format /var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz
if self.step == 'build':
- print(emerge_output)
- log_dict = {}
- # get cpv, logname and log path
- for log_path in emerge_output['log_paths']:
- c = log_path.split('/')[6]
- full_logname = log_path.split('/')[7]
- print(full_logname)
- pv = full_logname.split(':')[0]
- cpv = c + '/' + pv
- log_dict[cpv] = dict(
- log_path = log_path,
- full_logname = full_logname
- )
- print(log_dict)
# Find log for cpv that was requested or did failed
if not log_dict == {}:
# requested cpv
- print(log_dict)
cpv = self.getProperty('cpv')
- faild_cpv = emerge_output['failed']
faild_version_data = False
- if cpv in log_dict or faild_cpv in log_dict:
+ if cpv in log_dict or self.faild_cpv in log_dict:
yield self.createDistDir()
if cpv in log_dict:
self.log_data[cpv] = log_dict[cpv]
yield self.getLogFile(cpv, log_dict)
- if faild_cpv:
+ if self.faild_cpv:
# failed and build requested cpv
- if cpv == faild_cpv:
+ if cpv == self.faild_cpv:
faild_version_data = self.getProperty("version_data")
else:
# failed but not build requested cpv
- self.log_data[faild_cpv] = log_dict[faild_cpv]
- yield self.getLogFile(faild_cpv, log_dict)
- faild_version_data = yield self.getVersionData(faild_cpv)
+ self.log_data[self.faild_cpv] = log_dict[self.faild_cpv]
+ yield self.getLogFile(self.faild_cpv, log_dict)
+ faild_version_data = yield self.getVersionData(self.faild_cpv)
+ self.getEmergeFiles(self.faild_cpv)
self.getBuildWorkdirFiles()
- self.getEmergeFiles(faild_version_data)
+ else:
+ self.getEmergeFiles(cpv)
self.aftersteps_list.append(steps.Trigger(
schedulerNames=['parse_build_log'],
waitForFinish=False,
@@ -808,7 +864,7 @@ class CheckEmergeLogs(BuildStep):
'log_build_data' : self.log_data,
'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
'repository_data' : self.getProperty('repository_data'),
- 'faild_cpv' : faild_cpv,
+ 'faild_cpv' : self.faild_cpv,
'step' : self.step,
'build_workername' : self.getProperty('workername')
}
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-11-01 2:22 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-11-01 2:22 UTC (permalink / raw
To: gentoo-commits
commit: 1b1e91118c31d997289c3ed9b00638a71ae94588
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 1 02:23:12 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Nov 1 02:23:12 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1b1e9111
Add more logging and fix use test
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 3 ++
buildbot_gentoo_ci/steps/portage.py | 59 ++++++++++++++++++++++++++----------
2 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 79be77f..93e3529 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -607,6 +607,7 @@ class CheckEmergeLogs(BuildStep):
separator = '\n'
separator2 = ' '
change_use_list = []
+ log = yield self.addLog('change_use')
for cpv, v in emerge_output['change_use'].items():
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
@@ -623,6 +624,8 @@ class CheckEmergeLogs(BuildStep):
workdir='/etc/portage/package.use/'
)
)
+ yield log.addStdout('File: ' + 'zz_autouse' + str(self.getProperty('rerun')) + '\n')
+ yield log.addStdout(change_use_string + '\n')
# rerun
self.aftersteps_list.append(RunEmerge(step='pre-build'))
self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 408c1e9..08ab559 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -111,6 +111,8 @@ class SetMakeProfile(BuildStep):
workdir='/'
)
])
+ log = yield self.addLog('make.profile')
+ yield log.addStdout('Profile path: ' + makeprofile_path + '\n')
return SUCCESS
class SetReposConf(BuildStep):
@@ -135,6 +137,7 @@ class SetReposConf(BuildStep):
if repos_conf_data is None:
print('Default repo is not set in repos.conf')
return FAILURE
+ log = yield self.addLog('repos.conf')
# check if repos_conf_data['value'] is vaild repo name
separator = '\n'
default_conf = []
@@ -147,6 +150,10 @@ class SetReposConf(BuildStep):
workerdest="repos.conf/default.conf",
workdir='/etc/portage/')
])
+ # display the default.conf
+ yield log.addStdout('File: ' + 'default.conf' + '\n')
+ for line in default_conf:
+ yield log.addStdout(line + '\n')
# add all repos that project have in projects_repositorys to repos.conf/reponame.conf
projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
for project_repository_data in projects_repositorys_data:
@@ -159,11 +166,15 @@ class SetReposConf(BuildStep):
repository_conf.append('sync-type = git')
repository_conf.append('auto-sync = no')
repository_conf_string = separator.join(repository_conf)
+ filename = repository_data['name'] + '.conf'
yield self.build.addStepsAfterCurrentStep([
steps.StringDownload(repository_conf_string + separator,
- workerdest='repos.conf/' + repository_data['name'] + '.conf',
+ workerdest='repos.conf/' + filename,
workdir='/etc/portage/')
])
+ yield log.addStdout('File: ' + filename + '\n')
+ for line in repository_conf:
+ yield log.addStdout(line + '\n')
return SUCCESS
class SetMakeConf(BuildStep):
@@ -187,6 +198,7 @@ class SetMakeConf(BuildStep):
separator1 = '\n'
separator2 = ' '
makeconf_list = []
+ log = yield self.addLog('make.conf')
for k in makeconf_variables_data:
makeconf_variables_values_data = yield self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
makeconf_variable_list = []
@@ -236,6 +248,9 @@ class SetMakeConf(BuildStep):
workerdest="make.conf",
workdir='/etc/portage/')
])
+ # display the make.conf
+ for line in makeconf_list:
+ yield log.addStdout(line + '\n')
return SUCCESS
class SetPackageDefault(BuildStep):
@@ -255,18 +270,28 @@ class SetPackageDefault(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
separator1 = '\n'
separator2 = ' '
+ log = yield self.addLog('package.*')
self.aftersteps_list = []
- package_use_dir = False
- package_env_dir = False
+ self.aftersteps_list.append(steps.MakeDirectory(dir='package.use',
+ workdir='/etc/portage/'))
+ self.aftersteps_list.append(steps.MakeDirectory(dir='package.env',
+ workdir='/etc/portage/'))
#FIXME: accept_keywords
# add the needed package.* settings from db
+ # add package use
package_conf_use_list = []
package_settings = yield self.gentooci.db.projects.getProjectPortagePackageByUuid(self.getProperty('project_data')['uuid'])
for package_setting in package_settings:
if package_setting['directory'] == 'use':
package_conf_use_list.append(separator2.join(package_setting['package'],package_setting['value']))
+ if self.getProperty('use_data') is not None:
+ for k, v in self.getProperty('use_data').items():
+ for use, value in v.items():
+ if value:
+ package_conf_use_list.append(separator2.join([k, use]))
+ else:
+ package_conf_use_list.append(separator2.join([k, '-' + use]))
if package_conf_use_list != []:
- package_use_dir = True
package_conf_use_string = separator1.join(package_conf_use_list)
self.aftersteps_list.append(
steps.StringDownload(package_conf_use_string + separator1,
@@ -274,34 +299,32 @@ class SetPackageDefault(BuildStep):
workdir='/etc/portage/package.use/'
)
)
+ yield log.addStdout('File: ' + 'package.use/default.conf' + separator1)
+ for line in package_conf_use_list:
+ yield log.addStdout(line + separator1)
# for test we need to add env and use
#FIXME: check restrictions, test use mask and required use
if self.getProperty('projectrepository_data')['test']:
auxdb_iuses = yield self.gentooci.db.versions.getMetadataByUuidAndMatadata(self.getProperty("version_data")['uuid'], 'iuse')
for auxdb_iuse in auxdb_iuses:
- iuse, status = getIUseValue(auxdb_iuse)
+ iuse, status = getIUseValue(auxdb_iuse['value'])
if iuse == 'test':
- package_use_dir = True
self.aftersteps_list.append(
- steps.StringDownload(separator2.join('='+ self.getProperty("cpv"),'test') + separator1,
+ steps.StringDownload(separator2.join(['=' + self.getProperty("cpv"),'test']) + separator1,
workerdest='test.conf',
workdir='/etc/portage/package.use/'
)
)
- package_env_dir = True
+ yield log.addStdout('File: ' + 'package.use/test.conf' + separator1)
+ yield log.addStdout(separator2.join(['=' + self.getProperty("cpv"),'test']) + separator1)
self.aftersteps_list.append(
- steps.StringDownload(separator2.join('='+ self.getProperty("cpv"),'test.conf') + separator1,
+ steps.StringDownload(separator2.join(['=' + self.getProperty("cpv"),'test.conf']) + separator1,
workerdest='test.conf',
workdir='/etc/portage/package.env/'
)
)
- # add package.* dirs
- if package_use_dir:
- aftersteps_list.append(steps.MakeDirectory(dir='package.use',
- workdir='/etc/portage/'))
- if package_env_dir:
- aftersteps_list.append(steps.MakeDirectory(dir='package.env',
- workdir='/etc/portage/'))
+ yield log.addStdout('File: ' + 'package.env/test.conf' + separator1)
+ yield log.addStdout(separator2.join(['=' + self.getProperty("cpv"),'test.conf']) + separator1)
yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
@@ -336,6 +359,7 @@ class SetEnvDefault(BuildStep):
aftersteps_list = []
separator1 = '\n'
separator2 = ' '
+ log = yield self.addLog('env')
# create the dir
aftersteps_list.append(steps.MakeDirectory(dir='env',
workdir='/etc/portage/'))
@@ -357,6 +381,9 @@ class SetEnvDefault(BuildStep):
workerdest=k + '.conf',
workdir='/etc/portage/env/')
])
+ yield log.addStdout('File: ' + k + '.conf' + '\n')
+ for line in env_strings:
+ yield log.addStdout(line + '\n')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-10-23 22:26 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-10-23 22:26 UTC (permalink / raw
To: gentoo-commits
commit: 800a64ca521fef9d400b5bbe7fa4f9719934b2dd
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 23 22:27:09 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Oct 23 22:27:09 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=800a64ca
Use commmit id instead of time to check if repo need to be updated
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/repos.py | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/repos.py b/buildbot_gentoo_ci/steps/repos.py
index a2d46ce..f5c12d2 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -117,16 +117,25 @@ class CheckRepository(BuildStep):
repo = git.Repo(repository_path)
success = True
else:
+ print('repo commit id to check')
+ print(self.getProperty("revision"))
try:
- yield repo.git.pull()
+ commits = list(repo.iter_commits(rev=self.getProperty("revision"), max_count=2))
except:
- pass
+ try:
+ yield repo.git.pull()
+ except:
+ pass
+ else:
+ success = True
else:
- success = True
+ print('repo allready has commit id')
+ print(commits[0].hexsha)
+ return None
if success:
headcommit = repo.head.commit
+ print('repo updated to commit id')
print(headcommit.hexsha)
- print(headcommit.committed_date)
# chmod needed for ebuilds metadata portage.GetAuxMetadata step
# yield self.setchmod(repository_path)
return success
@@ -144,15 +153,11 @@ class CheckRepository(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(repository_uuid)
self.descriptionSuffix = repository_data['name']
- if repository_data['type'] == 'gitpuller':
- Poller_data = yield self.gentooci.db.repositorys.getGitPollerByUuid(repository_uuid)
- print(Poller_data['updated_at'])
- print(self.getProperty("commit_time"))
- if Poller_data['updated_at'] > self.getProperty("commit_time"):
- return SKIPPED
+ #self.Poller_data = yield self.gentooci.db.repositorys.getGitPollerByUuid(repository_uuid)
success = yield self.checkRepos(repository_data)
+ if success is None:
+ return SKIPPED
if not success:
return FAILURE
- if repository_data['type'] == 'gitpuller':
- yield self.gentooci.db.repositorys.updateGitPollerTime(repository_uuid)
+ #yield self.gentooci.db.repositorys.updateGitPollerTime(repository_uuid)
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-10-23 22:24 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-10-23 22:24 UTC (permalink / raw
To: gentoo-commits
commit: 0bff8de0077754ac80083c432210bdeb15ae4383
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 23 22:24:59 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Oct 23 22:24:59 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0bff8de0
Add ParserPkgCheckLog to parse_build_log
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 51 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d86e208..08ec6ce 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -16,6 +16,7 @@ from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
+from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import minio
@@ -305,6 +306,56 @@ class Upload(BuildStep):
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+class ParserPkgCheckLog(BuildStep):
+
+ name = 'ParserPkgCheckLog'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = False
+ flunkOnWarnings = False
+ flunkOnFailure = False
+ warnOnWarnings = False
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ if self.getProperty("pkg_check_log_data") is None:
+ return SKIPPED
+ returnstatus = SUCCESS
+ error = False
+ warning = False
+ log = yield self.addLog('Pkgcheck')
+ print(self.getProperty("pkg_check_log_data"))
+ for a in self.getProperty("pkg_check_log_data"):
+ status = ''
+ print(a)
+ if isinstance(a, dict):
+ for k, i in a.items():
+ if k.startswith('_'):
+ if k == '_info':
+ status = 'INFO: '
+ if k == '_error':
+ status = 'ERROR: '
+ error = True
+ if k == '_warning':
+ status = 'WARNING: '
+ warning = True
+ if k == '_style':
+ status = 'STYLE: '
+ if isinstance(i, dict):
+ for b, c in i.items():
+ yield log.addStdout(status + b + c + '\n')
+ else:
+ yield log.addStdout(i + '\n')
+ if error:
+ returnstatus = FAILURE
+ if warning and not error:
+ returnstatus = WARNINGS
+ return returnstatus
+
class setBuildStatus(BuildStep):
name = 'setBuildStatus'
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-10-06 1:35 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-10-06 1:35 UTC (permalink / raw
To: gentoo-commits
commit: 0e344fe1aa9e845d8bdaa704ee3d15064771ec49
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 6 01:36:22 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Oct 6 01:36:22 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0e344fe1
Use JsonReporter instead of XmlReporter for pkgcheck
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 39 ++++++++++++++++++++----------------
1 file changed, 22 insertions(+), 17 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 8f0aadb..80fe52f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -3,6 +3,7 @@
import os
import re
+import json
from portage.versions import catpkgsplit, cpv_getversion
from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
@@ -131,18 +132,13 @@ def PersOutputOfEmerge(rc, stdout, stderr):
def PersOutputOfPkgCheck(rc, stdout, stderr):
pkgcheck_output = {}
pkgcheck_output['rc'] = rc
- #FIXME: Handling of stdout output
- pkgcheck_xml_list = []
+ print(stdout)
+ pkgcheck_json_list = []
# split the lines
for line in stdout.split('\n'):
- # pkgcheck output list
- if line.startswith('<checks'):
- pkgcheck_xml_list.append(line)
- if line.startswith('<result'):
- pkgcheck_xml_list.append(line)
- if line.startswith('</checks'):
- pkgcheck_xml_list.append(line)
- pkgcheck_output['pkgcheck_xml'] = pkgcheck_xml_list
+ if line.startswith('{"'):
+ pkgcheck_json_list.append(json.loads(line))
+ pkgcheck_output['pkgcheck_json'] = pkgcheck_json_list
#FIXME: Handling of stderr output
return {
'pkgcheck_output' : pkgcheck_output
@@ -827,7 +823,7 @@ class RunPkgCheck(BuildStep):
'-v'
]
shell_commad_list.append('-R')
- shell_commad_list.append('XmlReporter')
+ shell_commad_list.append('JsonReporter')
aftersteps_list = []
if projectrepository_data['pkgcheck'] == 'full':
pkgcheck_workdir = yield os.path.join(repository_path, '')
@@ -856,15 +852,24 @@ class CheckPkgCheckLogs(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
- #@defer.inlineCallbacks
+ @defer.inlineCallbacks
def run(self):
- self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- project_data = self.getProperty('project_data')
+ #self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ #project_data = self.getProperty('project_data')
pkgcheck_output = self.getProperty('pkgcheck_output')
print(pkgcheck_output)
- #FIXME:
- # Perse the logs
- self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
+ if pkgcheck_output['pkgcheck_json'] == []:
+ return SKIPPED
+ pkg_check_log_data = []
+ c = yield catpkgsplit(self.getProperty("cpv"))[0]
+ p = yield catpkgsplit(self.getProperty("cpv"))[1]
+ v = yield cpv_getversion(self.getProperty("cpv"))
+ for json_dict in pkgcheck_output['pkgcheck_json']:
+ for k, i in json_dict[c][p].items():
+ if k == v or k == '_info' or k == '_style':
+ pkg_check_log_data.append(i)
+ if pkg_check_log_data != []:
+ self.setProperty('pkg_check_log_data', pkg_check_log_data, 'pkg_check_log_data')
return SUCCESS
class RunEmergeInfo(BuildStep):
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-10-02 20:52 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-10-02 20:52 UTC (permalink / raw
To: gentoo-commits
commit: 9a4cbee8916bebb51ad771a8e05e675b3b3a68b7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 2 20:53:19 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Oct 2 20:53:19 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9a4cbee8
Use gitpython instead of pygit2
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/repos.py | 37 +++++++++++++++++++++++--------------
1 file changed, 23 insertions(+), 14 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/repos.py b/buildbot_gentoo_ci/steps/repos.py
index 5b6b621..a2d46ce 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -2,7 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import os
-import pygit2
+import git
from twisted.internet import defer
@@ -104,20 +104,31 @@ class CheckRepository(BuildStep):
@defer.inlineCallbacks
def checkRepos(self, repository_data):
+ success = False
repository_path = yield os.path.join(self.getProperty("repository_basedir"), repository_data['name'])
- repo_path = yield pygit2.discover_repository(repository_path)
- print(repo_path)
- if repo_path is None:
- yield pygit2.clone_repository(repository_data['url'], repository_path)
- success = True
+ try:
+ repo = git.Repo(repository_path)
+ except:
+ try:
+ yield git.Repo.clone_from(repository_data['url'], repository_path)
+ except:
+ pass
+ else:
+ repo = git.Repo(repository_path)
+ success = True
else:
- repo = yield pygit2.Repository(repo_path)
- commit = repo.get(repo.head.target)
- success = yield self.gitPull(repo)
- print(commit.hex)
- print(commit.commit_time)
+ try:
+ yield repo.git.pull()
+ except:
+ pass
+ else:
+ success = True
+ if success:
+ headcommit = repo.head.commit
+ print(headcommit.hexsha)
+ print(headcommit.committed_date)
# chmod needed for ebuilds metadata portage.GetAuxMetadata step
- yield self.setchmod(repository_path)
+ # yield self.setchmod(repository_path)
return success
@defer.inlineCallbacks
@@ -140,8 +151,6 @@ class CheckRepository(BuildStep):
if Poller_data['updated_at'] > self.getProperty("commit_time"):
return SKIPPED
success = yield self.checkRepos(repository_data)
- if success is None:
- return SKIPPED
if not success:
return FAILURE
if repository_data['type'] == 'gitpuller':
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-09-28 22:16 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-09-28 22:16 UTC (permalink / raw
To: gentoo-commits
commit: c7aed08bcefcd3a3c4b4941f968416ba6d2a530e
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 28 22:16:42 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Sep 28 22:16:42 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c7aed08b
Fix package exclude code missing package_exclude['packages']
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 4f2b1b6..8f0aadb 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -458,12 +458,14 @@ class RunEmerge(BuildStep):
p = yield catpkgsplit(cpv)[1]
# Check if package is on the exclude list
if packages_excludes != []:
+ print(packages_excludes)
+ print(cpv)
for package_exclude in packages_excludes:
- if '/' not in package_exclude:
- if package_exclude == p:
+ if '/' in package_exclude['package']:
+ if package_exclude['package'] == c + '/' + p:
return SKIPPED
else:
- if package_exclude == c + '/' + p:
+ if package_exclude['package'] == p:
return SKIPPED
shell_commad_list.append('-pO')
# don't use bin for match
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-09-22 0:04 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-09-22 0:04 UTC (permalink / raw
To: gentoo-commits
commit: 00dbecbf9d20ff77a2dace764940341c6a9a9657
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Sep 22 00:04:56 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Sep 22 00:04:56 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=00dbecbf
Use umask in bb config intead of chmodin step
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 4073672..7732f39 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -334,10 +334,6 @@ class UpdateRepos(BuildStep):
for project_repository_data in projects_repositorys_data:
repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
- yield self.build.addStepsAfterCurrentStep([
- steps.ShellCommand(command=["chmod", "-R", "o+rX", repository_data['name']],
- workdir=portage_repos_path)
- ])
yield self.build.addStepsAfterCurrentStep([
steps.Git(repourl=repository_data['url'],
mode='full',
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-09-21 1:29 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-09-21 1:29 UTC (permalink / raw
To: gentoo-commits
commit: ee1e1227138eb6d1bff5c0685f03878240c77ac5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 21 01:28:32 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Sep 21 01:28:32 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ee1e1227
Chmod on the repository so portage can read it after the git step
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index ad66aec..4073672 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -335,7 +335,11 @@ class UpdateRepos(BuildStep):
repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
yield self.build.addStepsAfterCurrentStep([
- steps.Git(repourl=repository_data['url'],
+ steps.ShellCommand(command=["chmod", "-R", "o+rX", repository_data['name']],
+ workdir=portage_repos_path)
+ ])
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Git(repourl=repository_data['url'],
mode='full',
submodules=True,
alwaysUseLatest=True,
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-09-20 23:28 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-09-20 23:28 UTC (permalink / raw
To: gentoo-commits
commit: 5b97bafbb4228887c54bd835586ee6b8c006cc85
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Sep 20 23:27:24 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Sep 20 23:27:24 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5b97bafb
Support for circular deps media-libs/harfbuzz
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 110 ++++++++++++++++++++++++++++-------
1 file changed, 88 insertions(+), 22 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 9918938..ad66aec 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -5,6 +5,7 @@ import os
import re
from portage.versions import catpkgsplit, cpv_getversion
+from portage.dep import dep_getcpv, dep_getslot, dep_getrepo
from twisted.internet import defer
from twisted.python import log
@@ -19,15 +20,15 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['rc'] = rc
emerge_output['preserved_libs'] = False
emerge_output['change_use'] = False
+ emerge_output['circular_deps'] = False
emerge_output['failed'] = False
package_dict = {}
log_path_list = []
print(stderr)
# split the lines
for line in stdout.split('\n'):
- # package list
+ # package dict
subdict = {}
- subdict2 = {}
if line.startswith('[ebuild') or line.startswith('[binary') or line.startswith('[nomerge'):
# if binaries
if line.startswith('[ebuild') or line.startswith('[nomerge'):
@@ -36,21 +37,14 @@ def PersOutputOfEmerge(rc, stdout, stderr):
subdict['binary'] = True
# action [ N ] stuff
subdict['action'] = line[8:15].replace(' ', '')
- # cpv
# We my have more then one spece betvine ] and cpv
- cpv_line = re.sub(' +', ' ', line)
- # get cpv
- cpv_split = re.search('] (.+?) ', cpv_line).group(1).split(':')
- print(cpv_split)
- cpv = cpv_split[0]
+ pkg_line = re.sub(' +', ' ', line)
+ # get pkg
+ pkg = '=' + re.search('] (.+?) ', pkg_line).group(1)
# repository
+ subdict['repository'] = dep_getrepo(pkg)
# slot
- if cpv_split[1] == '':
- subdict['slot'] = None
- subdict['repository'] = cpv_split[2]
- else:
- subdict['slot'] = cpv_split[1]
- subdict['repository'] = cpv_split[3]
+ subdict['slot'] = dep_getslot(pkg)
# if action U version cpv
if 'U' in subdict['action']:
subdict['old_version'] = re.search(' \[(.+?)] ', line).group(1).split(':')
@@ -66,8 +60,8 @@ def PersOutputOfEmerge(rc, stdout, stderr):
subdict['python_targets'] = re.search('PYTHON_TARGETS="(.+?)" ', line).group(1).split(' ')
else:
subdict['python_targets'] = None
- # CPU_FLAGS_X86 list
- package_dict[cpv] = subdict
+ # FIXME: CPU_FLAGS_X86 list
+ package_dict[dep_getcpv(pkg)] = subdict
if line.startswith('>>>'):
if line.startswith('>>> Failed to'):
emerge_output['failed'] = line.split(' ')[4][:-1]
@@ -83,12 +77,11 @@ def PersOutputOfEmerge(rc, stdout, stderr):
if line.endswith('.log.gz'):
log_path_list.append(line.split(' ')[4])
#FIXME: Handling of depclean output dict of packages that get removed or saved
- emerge_output['package'] = package_dict
+ emerge_output['packages'] = package_dict
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
- emerge_output['change_use'] = False
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -123,6 +116,9 @@ def PersOutputOfEmerge(rc, stdout, stderr):
log_path = line.split(' ')[3]
if log_path not in inlog_path_list:
log_path_list.append(log_path)
+ #FIXME: make dict of cpv listed in the circular dependencies
+ if line.endswith('circular dependencies:'):
+ emerge_output['circular_deps'] = True
stderr_line_list.append(line)
emerge_output['stderr'] = stderr_line_list
emerge_output['log_paths'] = log_path_list
@@ -282,7 +278,6 @@ class SetupPropertys(BuildStep):
def __init__(self, **kwargs):
# set this in config
- self.portage_repos_path = '/var/db/repos/'
super().__init__(**kwargs)
@defer.inlineCallbacks
@@ -290,7 +285,7 @@ class SetupPropertys(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
self.descriptionDone = 'Building this %s' % self.getProperty("cpv")
- self.setProperty('portage_repos_path', self.portage_repos_path, 'portage_repos_path')
+ self.setProperty('portage_repos_path', self.gentooci.config.project['project']['worker_portage_repos_path'], 'portage_repos_path')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
@@ -356,7 +351,7 @@ class RunEmerge(BuildStep):
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, step=None,**kwargs):
+ def __init__(self, step=None, **kwargs):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
@@ -575,6 +570,7 @@ class CheckEmergeLogs(BuildStep):
'emerge',
'-v'
]
+ package_dict = emerge_output['packages']
#FIXME: Prosees the logs and do stuff
# preserved-libs
@@ -583,7 +579,7 @@ class CheckEmergeLogs(BuildStep):
# FIXME: check if cpv match
if self.step == 'match'and self.getProperty('projectrepository_data')['build']:
- if self.getProperty('cpv') in emerge_output['package']:
+ if self.getProperty('cpv') in package_dict:
self.setProperty('cpv_build', True, 'cpv_build')
print(self.getProperty('cpv_build'))
@@ -621,9 +617,79 @@ class CheckEmergeLogs(BuildStep):
# rerun
self.aftersteps_list.append(RunEmerge(step='pre-build'))
self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
+
+ # * Error: circular dependencies:
+ if emerge_output['circular_deps'] is True:
+ circular_dep = None
+ print('circular_deps')
+ for cpv, v in package_dict.items():
+ print(cpv)
+ print(catpkgsplit(cpv))
+ p = yield catpkgsplit(cpv)[1]
+ if p == 'harfbuzz':
+ circular_dep = 'harfbuzz'
+ # media-libs/harfbuzz
+ # https://wiki.gentoo.org/wiki/User:Sam/Portage_help/Circular_dependencies#Solution
+ if circular_dep == 'harfbuzz':
+ shell_commad_list = []
+ shell_commad_list.append('emerge')
+ shell_commad_list.append('-v')
+ # FIXME: cpv my get deleted in the tree
+ cpv = 'x11-libs/pango-1.48.5-r1'
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ shell_commad_list.append('-1')
+ shell_commad_list.append('=' + cpv)
+ # we don't use the bin for the requsted cpv
+ shell_commad_list.append('--usepkg-exclude')
+ shell_commad_list.append(c + '/' + p)
+ # rebuild this
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('freetype')
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('harfbuzz')
+ # don't build bin for virtual and acct-*
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('virtual')
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('acct-*')
+ self.aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfEmerge,
+ workdir='/',
+ env={'USE': "-harfbuzz"},
+ timeout=None
+ ))
+ self.aftersteps_list.append(CheckEmergeLogs('extra-build'))
else:
# trigger parse_build_log with info about pre-build and it fail
pass
+ # Check if extra build did work
+ if self.step == 'extra-build':
+ print(emerge_output)
+ log_dict = {}
+ # get cpv, logname and log path
+ for log_path in emerge_output['log_paths']:
+ c = log_path.split('/')[6]
+ full_logname = log_path.split('/')[7]
+ print(full_logname)
+ pv = full_logname.split(':')[0]
+ cpv = c + '/' + pv
+ log_dict[cpv] = dict(
+ log_path = log_path,
+ full_logname = full_logname
+ )
+ print(log_dict)
+ # Find log for cpv that was requested or did failed
+ if not log_dict == {}:
+ # requested cpv
+ print(log_dict)
+ faild_cpv = emerge_output['failed']
+ self.aftersteps_list.append(RunEmerge(step='pre-build'))
+ self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
+
#FIXME:
# Look for FAILURE and logname and download needed logfile and
# trigger a logparser
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-09-07 0:46 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-09-07 0:46 UTC (permalink / raw
To: gentoo-commits
commit: 1f2f01cb4febebe5d47b4ef6c9591b73cbdfc5a5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 7 00:46:53 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Sep 7 00:46:53 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1f2f01cb
Support more options in project dict in config file
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 2 +-
buildbot_gentoo_ci/steps/portage.py | 2 +-
buildbot_gentoo_ci/steps/update_db.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index d69c447..d86e208 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -37,7 +37,7 @@ class SetupPropertys(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
- default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+ default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
version_data = yield self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
self.setProperty("project_data", project_data, 'project_data')
self.setProperty("default_project_data", default_project_data, 'default_project_data')
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 37f7342..0a131d7 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -326,7 +326,7 @@ class SetEnvDefault(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
- default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+ default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
aftersteps_list = []
separator1 = '\n'
separator2 = ' '
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index 8e7f8ee..0ed4be1 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -19,7 +19,7 @@ class GetDataGentooCiProject(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- self.project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+ self.project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
if self.project_data is None:
log.err('No data for project in the database')
return FAILURE
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-07-17 8:14 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-07-17 8:14 UTC (permalink / raw
To: gentoo-commits
commit: 41b470c43fd84b66454576d1ec0d84433f0508b0
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 17 08:14:35 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Jul 17 08:14:35 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=41b470c4
Fix bug in PersOutputOfEmerge for cpv line with more then one spec
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 63a1ae9..5b53018 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -37,8 +37,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# action [ N ] stuff
subdict['action'] = line[8:15].replace(' ', '')
# cpv
- #FIXME: We my have more then one spece betvine ] and cpv
- cpv_split = re.search('] (.+?) ', line).group(1).split(':')
+ # We my have more then one spece betvine ] and cpv
+ cpv_line = re.sub(' +', ' ', line)
+ # get cpv
+ cpv_split = re.search('] (.+?) ', cpv_line).group(1).split(':')
print(cpv_split)
cpv = cpv_split[0]
# repository
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-05-18 8:55 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-05-18 8:55 UTC (permalink / raw
To: gentoo-commits
commit: 001f737e5d8cbc62272415580e34b53cee6d81b5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue May 18 08:55:41 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue May 18 08:55:41 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=001f737e
Make git use the latest revision
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 3a5de47..0bed647 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -327,8 +327,9 @@ class UpdateRepos(BuildStep):
repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
yield self.build.addStepsAfterCurrentStep([
steps.Git(repourl=repository_data['url'],
- mode='incremental',
+ mode='full',
submodules=True,
+ alwaysUseLatest=True,
workdir=os.path.join(repository_path, ''))
])
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-04-26 20:26 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-04-26 20:26 UTC (permalink / raw
To: gentoo-commits
commit: b2f1c115420e05c27fb7a57cf45734cb7f4a7731
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 26 20:26:04 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Apr 26 20:26:04 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b2f1c115
Add more checks and move * and >>> check to last
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 121 ++++++++++++++++++++++-----------------
1 file changed, 70 insertions(+), 51 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 178d71e..0960b9b 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -57,39 +57,46 @@ class ParserBuildLog(BuildStep):
self.summery_dict = {}
self.index = 1
self.log_search_pattern_list = []
- self.max_text_lines = self.index -1
+ self.max_text_lines = 0
super().__init__(**kwargs)
+ #FIXME: ansifilter
+ def ansiFilter(self, text):
+ return text
+
@defer.inlineCallbacks
def get_log_search_pattern(self):
# get pattern from the projects
# add that to log_search_pattern_list
for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
- self.log_search_pattern_list.append(project_pattern)
- # get the default profile pattern
+ # check if the search pattern is vaild
+ try:
+ re.compile(project_pattern['search'])
+ except re.error:
+ print("Non valid regex pattern")
+ print(project_pattern)
+ else:
+ self.log_search_pattern_list.append(project_pattern)
+ # get the default project pattern
# add if not pattern is in project ignore
+ self.project_pattern_ignore = yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('project_data')['uuid'])
for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])):
- match = True
- for project_pattern_ignore in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('default_project_data')['uuid'])):
- if project_pattern['search'] == project_pattern_ignore['search']:
- match = False
- if match:
- self.log_search_pattern_list.append(project_pattern)
+ if not project_pattern['search'] in self.project_pattern_ignore:
+ # check if the search pattern is vaild
+ try:
+ re.compile(project_pattern['search'])
+ except re.error:
+ print("Non valid regex pattern")
+ print(project_pattern)
+ else:
+ self.log_search_pattern_list.append(project_pattern)
def search_buildlog(self, tmp_index):
# get text line to search
- text_line = self.logfile_text_dict[tmp_index]
+ text_line = self.ansiFilter(self.logfile_text_dict[tmp_index])
# loop true the pattern list for match
for search_pattern in self.log_search_pattern_list:
search_hit = False
- # we add all line that start with ' * ' as info
- # we add all line that start with '>>>' but not '>>> /' as info
- if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
- self.summery_dict[tmp_index] = {}
- self.summery_dict[tmp_index]['text'] = text_line
- self.summery_dict[tmp_index]['type'] = 'info'
- self.summery_dict[tmp_index]['status'] = 'info'
- self.summery_dict[tmp_index]['search_pattern_id'] = 0
if search_pattern['search_type'] == 'in':
if search_pattern['search'] in text_line:
search_hit = True
@@ -100,63 +107,74 @@ class ParserBuildLog(BuildStep):
if text_line.endswith(search_pattern['search']):
search_hit = True
if search_pattern['search_type'] == 'search':
- if search_pattern['search'] in text_line:
+ if re.search(search_pattern['search'], text_line):
search_hit = True
+ # add the line if the pattern match
if search_hit:
print(text_line)
- print(search_pattern['search'])
+ print(search_pattern)
+ print(tmp_index)
self.summery_dict[tmp_index] = {}
self.summery_dict[tmp_index]['text'] = text_line
self.summery_dict[tmp_index]['type'] = search_pattern['type']
self.summery_dict[tmp_index]['status'] = search_pattern['status']
self.summery_dict[tmp_index]['search_pattern_id'] = search_pattern['id']
# add upper text lines if requested
- # max 10
- if search_pattern['start'] != 0 and search_hit:
- i = tmp_index
- i_start = i - search_pattern['start']
- match = True
- while match:
- i = i - 1
- if i < 0 or i < i_start:
- match = False
- else:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] = self.logfile_text_dict[i]
- self.summery_dict[i]['type'] = search_pattern['type']
- self.summery_dict[i]['status'] = 'info'
+ # max 5
+ if search_pattern['start'] != 0:
+ i = tmp_index - search_pattern['start'] - 1
+ match = True
+ while match:
+ i = i + 1
+ if i < (tmp_index - 9) or i == tmp_index:
+ match = False
+ else:
+ if not i in self.summery_dict:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i])
+ self.summery_dict[i]['type'] = 'info'
+ self.summery_dict[i]['status'] = 'info'
# add lower text lines if requested
- # max 10
- if search_pattern['end'] != 0 and search_hit:
- i = tmp_index
- i_end = i + search_pattern['end']
- match = True
- while match:
- i = i + 1
- if i > self.max_text_lines or i > i_end:
- match = False
- else:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] = self.logfile_text_dict[i]
- self.summery_dict[i]['type'] = search_pattern['type']
- self.summery_dict[i]['status'] = 'info'
+ # max 5
+ if search_pattern['end'] != 0:
+ i = tmp_index
+ end = tmp_index + search_pattern['end']
+ match = True
+ while match:
+ i = i + 1
+ if i > self.max_text_lines or i > end:
+ match = False
+ else:
+ if not i in self.summery_dict:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i])
+ self.summery_dict[i]['type'] = 'info'
+ self.summery_dict[i]['status'] = 'info'
+ else:
+ # we add all line that start with ' * ' as info
+ # we add all line that start with '>>>' but not '>>> /' as info
+ if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
+ if not tmp_index in self.summery_dict:
+ self.summery_dict[tmp_index] = {}
+ self.summery_dict[tmp_index]['text'] = text_line
+ self.summery_dict[tmp_index]['type'] = 'info'
+ self.summery_dict[tmp_index]['status'] = 'info'
@defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- #FIXME:
- # get the log parser pattern from db
yield self.get_log_search_pattern()
# open the log file
# read it to a buffer
# make a dict of the buffer
- # maby use mulitiprocces to speed up the search
+ # maybe use mulitiprocces to speed up the search
print(self.getProperty('log_build_data'))
if self.getProperty('faild_cpv'):
log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
else:
log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')]
file_path = yield os.path.join(self.master.basedir, 'cpv_logs', log_cpv['full_logname'])
+ #FIXME: decode it to utf-8
with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f:
for text_line in f:
self.logfile_text_dict[self.index] = text_line.strip('\n')
@@ -168,6 +186,7 @@ class ParserBuildLog(BuildStep):
if self.index >= 20:
del self.logfile_text_dict[self.index - 19]
self.index = self.index + 1
+ self.max_text_lines = self.index
f.close()
# check last 10 lines in logfile_text_dict
yield self.search_buildlog(self.index - 10)
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-04-20 7:05 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-04-20 7:05 UTC (permalink / raw
To: gentoo-commits
commit: c059e969ff0e106823e0bed4729d4c04db02eee8
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 20 07:05:33 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Apr 20 07:05:33 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c059e969
Support more then one Chaange USE
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 48 ++++++++++++++++++++++++------------
1 file changed, 32 insertions(+), 16 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index d9817d8..50f924d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -86,21 +86,35 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# split the lines
#FIXME: Handling of stderr output
stderr_line_list = []
+ emerge_output['change_use'] = False
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
change_use_list = []
# get cpv
cpv_split = line_list[1].split(':')
- change_use_list.append(cpv_split[0])
+ change_use = {}
# add use flags
if line_list[4].startswith('+') or line_list[4].startswith('-'):
- # we only support one for now
+ # we only support tre for now
if line_list[4].endswith(')'):
change_use_list.append(line_list[4].replace(')', ''))
+ elif line_list[5].endswith(')'):
+ change_use_list.append(line_list[4])
+ change_use_list.append(line_list[5].replace(')', ''))
+ elif line_list[6].endswith(')'):
+ change_use_list.append(line_list[4])
+ change_use_list.append(line_list[5])
+ change_use_list.append(line_list[6].replace(')', ''))
+ elif not line_list[6].endswith(')'):
+ change_use_list.append(line_list[4])
+ change_use_list.append(line_list[5])
+ change_use_list.append(line_list[6])
else:
change_use_list = False
- emerge_output['change_use'] = change_use_list
+ if change_use_list:
+ change_use[cpv_split[0]] = change_use_list
+ emerge_output['change_use'] = change_use
err_line_list = []
if line.startswith(' * '):
if line.endswith('.log.gz'):
@@ -226,7 +240,7 @@ class SetupPropertys(BuildStep):
name = 'SetupPropertys'
description = 'Running'
- descriptionDone = 'Ran'
+ #descriptionDone = 'Ran'
descriptionSuffix = None
haltOnFailure = True
flunkOnFailure = True
@@ -240,6 +254,7 @@ class SetupPropertys(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
+ self.descriptionDone = 'Building this %s' % self.getProperty("cpv")
self.setProperty('portage_repos_path', self.portage_repos_path, 'portage_repos_path')
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
@@ -550,6 +565,7 @@ class CheckEmergeLogs(BuildStep):
# update package.* if needed and rerun pre-build max 3 times
if self.step == 'pre-build':
print(emerge_output)
+ # this should be set in the config
if self.getProperty('rerun') <= 3:
# when we need to change use. we could rerun pre-build with
# --autounmask-use=y --autounmask-write=y --autounmask-only=y
@@ -560,21 +576,21 @@ class CheckEmergeLogs(BuildStep):
separator = '\n'
separator2 = ' '
change_use_list = []
- cpv = emerge_output['change_use'][0]
- c = yield catpkgsplit(cpv)[0]
- p = yield catpkgsplit(cpv)[1]
- change_use_list.append(c + '/' + p)
- # we only support one use
- use_flag = emerge_output['change_use'][1]
- if use_flag.startswith('+'):
- change_use_list.append(use_flag.replace('+', ''))
- else:
- change_use_list.append(use_flag)
+ for cpv, v in emerge_output['change_use'].items():
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ change_use_list.append(c + '/' + p)
+ for use_flag in v:
+ if use_flag.startswith('+'):
+ change_use_list.append(use_flag.replace('+', ''))
+ else:
+ change_use_list.append(use_flag)
change_use_string = separator2.join(change_use_list)
self.aftersteps_list.append(
steps.StringDownload(change_use_string + separator,
- workerdest='zz_autouse' + str(self.getProperty('rerun')),
- workdir='/etc/portage/package.use/')
+ workerdest='zz_autouse' + str(self.getProperty('rerun')),
+ workdir='/etc/portage/package.use/'
+ )
)
# rerun
self.aftersteps_list.append(RunEmerge(step='pre-build'))
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-04-13 19:03 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-04-13 19:03 UTC (permalink / raw
To: gentoo-commits
commit: 482017e7c399b4e97502617f0aa70ff50df54b7b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 13 19:03:04 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Apr 13 19:03:04 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=482017e7
check repository_linkname symalink in LocalWorkers
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index c1a4d90..c7e82ce 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -18,6 +18,7 @@ from twisted.python import log
from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot_gentoo_ci.steps import master as master_steps
@@ -367,6 +368,9 @@ class CheckPathLocal(BuildStep):
@defer.inlineCallbacks
def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ self.repository_linkname = self.gentooci.config.project['repository_basedir']
+ self.repository_basedir2 = '/home/repos2/'
self.portage_path = yield os.path.join('etc', 'portage')
self.profile_path = yield os.path.join(self.portage_path, 'make.profile')
self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
@@ -381,6 +385,8 @@ class CheckPathLocal(BuildStep):
]:
if not os.path.isdir(x):
os.makedirs(x)
+ if not os.path.islink(self.repository_linkname):
+ os.symlink(self.repository_basedir2, self.repository_linkname)
return SUCCESS
class SetMakeProfileLocal(BuildStep):
@@ -399,7 +405,7 @@ class SetMakeProfileLocal(BuildStep):
def run(self):
parent_path = yield os.path.join('etc','portage', 'make.profile', 'parent')
if os.path.isfile(parent_path):
- return SUCCESS
+ return SKIPPED
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir = self.gentooci.config.project['repository_basedir']
makeprofiles_paths = []
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-04-02 22:12 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-04-02 22:12 UTC (permalink / raw
To: gentoo-commits
commit: 489eb63a368e9bbfa73fcbda96cae8a1222e1287
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 2 22:06:40 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Apr 2 22:06:40 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=489eb63a
Don't run emerge build quiet
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index dce633f..b28d7e5 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -412,7 +412,6 @@ class RunEmerge(BuildStep):
cpv = self.getProperty("cpv")
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
- shell_commad_list.append('-q')
if projects_emerge_options['oneshot']:
shell_commad_list.append('-1')
shell_commad_list.append('=' + self.getProperty('cpv'))
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-04-02 22:12 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-04-02 22:12 UTC (permalink / raw
To: gentoo-commits
commit: 1560eecc8a0c32e219acfeda461dd6cb2d015f30
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 2 22:09:09 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Apr 2 22:09:09 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1560eecc
Check '>>>' add pattern type
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 30 ++++++++++++++++++++++--------
1 file changed, 22 insertions(+), 8 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index ae41fe5..5fe248f 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -62,7 +62,7 @@ class ParserBuildLog(BuildStep):
@defer.inlineCallbacks
def get_log_search_pattern(self):
- # get pattern from the profile
+ # get pattern from the projects
# add that to log_search_pattern_list
for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
self.log_search_pattern_list.append(project_pattern)
@@ -82,13 +82,21 @@ class ParserBuildLog(BuildStep):
text_line = self.logfile_text_dict[tmp_index]
# loop true the pattern list for match
for search_pattern in self.log_search_pattern_list:
- if re.search(search_pattern['search'], text_line) and not search_pattern['status'] == 'ignore':
+ # we add all line that start with ' * ' as info
+ # we add all line that start with '>>>' but not '>>> /' as info
+ if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
self.summery_dict[tmp_index] = {}
self.summery_dict[tmp_index]['text'] = text_line
+ self.summery_dict[tmp_index]['type'] = 'info'
+ self.summery_dict[tmp_index]['status'] = 'info'
+ if re.search(search_pattern['search'], text_line):
+ self.summery_dict[tmp_index] = {}
+ self.summery_dict[tmp_index]['text'] = text_line
+ self.summery_dict[tmp_index]['type'] = search_pattern['type']
self.summery_dict[tmp_index]['status'] = search_pattern['status']
# add upper text lines if requested
# max 10
- if search_pattern['start'] is not 0:
+ if search_pattern['start'] != 0:
i = tmp_index
i_start = i - search_pattern['start']
match = True
@@ -99,10 +107,11 @@ class ParserBuildLog(BuildStep):
else:
self.summery_dict[i] = {}
self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['type'] = search_pattern['type']
self.summery_dict[i]['status'] = 'info'
# add lower text lines if requested
# max 10
- if search_pattern['end'] is not 0:
+ if search_pattern['end'] != 0:
i = tmp_index
i_end = i + search_pattern['end']
match = True
@@ -113,6 +122,7 @@ class ParserBuildLog(BuildStep):
else:
self.summery_dict[i] = {}
self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['type'] = search_pattern['type']
self.summery_dict[i]['status'] = 'info'
# add text lines if requested that we need to search for the end
# max 10
@@ -127,13 +137,15 @@ class ParserBuildLog(BuildStep):
if not i + 1 > self.max_text_lines or not re.search(search_pattern['search_end'], self.logfile_text_dict[i + 1]):
self.summery_dict[i] = {}
self.summery_dict[i]['text'] = self.logfile_text_dict[i]
+ self.summery_dict[i]['type'] = search_pattern['type']
self.summery_dict[i]['status'] = 'info'
else:
match = False
else:
self.summery_dict[i] = {}
self.summery_dict[i]['text'] = self.logfile_text_dict[i]
- self.summery_dict[i]['status'] = 'info'
+ self.summery_dict[i]['type'] = search_pattern['type']
+ self.summery_dict[i]['status'] = 'info'
@defer.inlineCallbacks
def run(self):
@@ -148,9 +160,9 @@ class ParserBuildLog(BuildStep):
file_path = yield os.path.join(self.master.basedir, 'cpv_logs', self.getProperty('log_build_data')['full_logname'])
with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f:
for text_line in f:
- self.logfile_text_dict[self.index] = text_line
+ self.logfile_text_dict[self.index] = text_line.strip('\n')
# run the parse patten on the line
- # have a buffer on 5 before we run pattern check
+ # have a buffer on 10 before we run pattern check
if self.index >= 10:
yield self.search_buildlog(self.index - 9)
# remove text line that we don't need any more
@@ -158,8 +170,10 @@ class ParserBuildLog(BuildStep):
del self.logfile_text_dict[self.index - 19]
self.index = self.index + 1
f.close()
+ # check last 10 lines in logfile_text_dict
+ yield self.search_buildlog(self.index - 10)
print(self.summery_dict)
- # check last 5 lines in logfile_text_dict
+ # remove all lines with ignore in the dict
# setProperty summery_dict
self.setProperty("summery_log_dict", self.summery_dict, 'summery_log_dict')
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-04-02 22:12 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-04-02 22:12 UTC (permalink / raw
To: gentoo-commits
commit: 8d863f91968ff02e963ac59a75265e7e3af2cfcd
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 2 22:03:07 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Apr 2 22:03:07 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8d863f91
Check so keyword is not None
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index ff6dd94..dce633f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -171,11 +171,12 @@ class GetProjectRepositoryData(BuildStep):
if project_keyword_data['name'] != '*' or project_data['status'] == 'all':
self.setProperty('fullcheck', False, 'fullcheck')
# get status of the keyword on cpv
- if project_keyword_data['name'] in self.getProperty("version_keyword_dict"):
- version_keywords_data = self.getProperty("version_keyword_dict")[project_keyword_data['name']]
- # if match trigger BuildRequest on cpv
- if project_data['status'] == version_keywords_data['status']:
- yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
+ if self.getProperty("version_keyword_dict") is not None:
+ if project_keyword_data['name'] in self.getProperty("version_keyword_dict"):
+ version_keywords_data = self.getProperty("version_keyword_dict")[project_keyword_data['name']]
+ # if match trigger BuildRequest on cpv
+ if project_data['status'] == version_keywords_data['status']:
+ yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
return SUCCESS
class SetupPropertys(BuildStep):
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-04-02 22:12 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-04-02 22:12 UTC (permalink / raw
To: gentoo-commits
commit: 8bdcac12a5c3a41d8321a55b8fef36355dc0f6e7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 2 22:11:49 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Apr 2 22:11:49 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8bdcac12
Add GetAuxMetadata and use that to get aux metadata
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 63 ++++++++++++++++++++++++++++++++---
buildbot_gentoo_ci/steps/update_db.py | 17 +++++++++-
buildbot_gentoo_ci/steps/version.py | 12 ++++---
3 files changed, 83 insertions(+), 9 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 293b862..d66c952 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -10,6 +10,7 @@ from portage import _encodings
from portage import _unicode_encode
from portage import _parse_eapi_ebuild_head, eapi_is_supported
from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
+from portage import portdbapi
from twisted.internet import defer
from twisted.python import log
@@ -249,7 +250,7 @@ class SetMakeConf(BuildStep):
makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"')
makeconf_list.append('PORTAGE_ELOG_CLASSES="*"')
# We need echo:info to get the logfile name
- makeconf_list.append('PORTAGE_ELOG_SYSTEM="save echo:info"')
+ makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"')
# add ACCEPT_KEYWORDS from the project_data info
keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
if project_data['status'] == 'unstable':
@@ -395,7 +396,8 @@ class SetReposConfLocal(BuildStep):
repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf')
repos_conf_default_path = yield os.path.join(repos_conf_path, 'default.conf')
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- self.repository_basedir = self.gentooci.config.project['repository_basedir']
+ # the path should be set in the confg
+ self.repository_basedir2 = '/home/repos2/'
if not os.path.isfile(repos_conf_default_path):
# setup the default.conf
repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'repos.conf')
@@ -409,7 +411,7 @@ class SetReposConfLocal(BuildStep):
yield WriteTextToFile(repos_conf_default_path, default_conf)
repos_conf_repository_path = yield os.path.join(repos_conf_path, self.getProperty("repository_data")['name'] + '.conf')
if not os.path.isfile(repos_conf_repository_path):
- repository_path = yield os.path.join(self.getProperty("builddir"), self.repository_basedir, self.getProperty("repository_data")['name'])
+ repository_path = yield os.path.join(self.repository_basedir2, self.getProperty("repository_data")['name'])
repository_conf = []
repository_conf.append('[' + self.getProperty("repository_data")['name'] + ']')
repository_conf.append('location = ' + repository_path)
@@ -444,7 +446,7 @@ class SetMakeConfLocal(BuildStep):
makeconf_list.append('ACCEPT_KEYWORDS="~amd64 amd64"')
makeconf_list.append('EMERGE_DEFAULT_OPTS=""')
makeconf_list.append('ABI_X86="32 64"')
- makeconf_list.append('FEATURES=""')
+ makeconf_list.append('FEATURES="sandbox"')
yield WriteTextToFile(make_conf_path, makeconf_list)
return SUCCESS
@@ -530,3 +532,56 @@ class SetEnvForEbuildSH(BuildStep):
))
yield self.build.addStepsAfterCurrentStep(addStepEbuildSH)
return SUCCESS
+
+class GetAuxMetadata(BuildStep):
+
+ name = 'GetAuxMetadata'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ #@defer.inlineCallbacks
+ def thd_getAuxDbKeys(self):
+ auxdbs = self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys, myrepo=self.getProperty("repository_data")['name'])
+ return auxdbs
+
+ @defer.inlineCallbacks
+ def run(self):
+ config_root = yield os.path.join(self.getProperty("builddir"), '')
+ # setup mysettings and myportdb
+ mysettings = yield portage_config(config_root = config_root)
+ self.myportdb = yield portdbapi(mysettings=mysettings)
+ auxdbs = yield self.thd_getAuxDbKeys()
+ metadata = None
+ NoSplit = []
+ NoSplit.append('DESCRIPTION')
+ # should have 22 lines
+ if len(auxdbkeys) != len(auxdbs) or not isinstance(auxdbs, list):
+ # number of lines is incorrect or not a list.
+ print("Lines don't match or not a list")
+ yield self.myportdb.close_caches()
+ yield portdbapi.portdbapi_instances.remove(self.myportdb)
+ #self.setProperty('auxdb', metadata, 'auxdb')
+ return FAILURE
+ # split all keys to list instead of speces
+ metadata = {}
+ i = 0
+ for key in auxdbkeys:
+ if auxdbs[i] == '':
+ metadata[key] = None
+ else:
+ if ' ' in auxdbs[i] and key not in NoSplit:
+ metadata[key] = auxdbs[i].split(' ')
+ else:
+ metadata[key] = []
+ metadata[key].append(auxdbs[i])
+ i = i + 1
+ self.setProperty('auxdb', metadata, 'auxdb')
+ yield self.myportdb.close_caches()
+ yield portdbapi.portdbapi_instances.remove(self.myportdb)
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index b84e256..3594c1b 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -140,11 +140,20 @@ class UpdateRepos(BuildStep):
else:
raise AssertionError('Unknown merge analysis result')
+ @defer.inlineCallbacks
+ def setchmod(self, path):
+ for root, dirs, files in os.walk(path):
+ for d in dirs:
+ yield os.chmod(os.path.join(root, d), 0o0755)
+ for f in files:
+ yield os.chmod(os.path.join(root, f), 0o0644)
+
@defer.inlineCallbacks
def run(self):
#FIXME check HEAD agenst local and worker local tree so we don't gitpull evrytime
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- self.repository_basedir = self.gentooci.config.project['repository_basedir']
+ #self.repository_basedir = self.gentooci.config.project['repository_basedir']
+ self.repository_basedir = yield os.path.join('/home', 'repos2')
self.profile_repository_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name'])
repo_path = yield pygit2.discover_repository(self.profile_repository_path)
print(repo_path)
@@ -153,6 +162,9 @@ class UpdateRepos(BuildStep):
else:
repo = yield pygit2.Repository(repo_path)
yield self.gitPull(repo)
+ # chmod and chown
+ yield self.setchmod(self.profile_repository_path)
+ #yield os.chown(self.profile_repository_path, 'buildbot', 'portage')
if self.getProperty("profile_repository_data")['name'] != self.getProperty("repository_data")['name']:
self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'])
repo_path = yield pygit2.discover_repository(self.repository_path)
@@ -161,6 +173,9 @@ class UpdateRepos(BuildStep):
else:
repo = yield pygit2.Repository(repo_path)
yield self.gitPull(repo)
+ # chmod and chown
+ yield self.setchmod(self.profile_repository_path)
+ #yield os.chown(self.repository_path, 'buildbot', 'portage')
return SUCCESS
class TriggerCheckForCPV(BuildStep):
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index 4c35914..426ddd4 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -131,7 +131,6 @@ class AddVersionKeyword(BuildStep):
return SUCCESS
print(auxdb)
for keyword in auxdb:
- print(keyword)
status = 'stable'
if keyword[0] in ["~"]:
keyword = keyword[1:]
@@ -170,7 +169,8 @@ class CheckPathHash(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir = self.gentooci.config.project['repository_basedir']
- self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'])
+ self.repository_path = yield os.path.join('/home', 'repos2', self.getProperty("repository_data")['name'])
+ #self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'])
self.cp_path = yield pkgsplit(self.getProperty("cpv"))[0]
self.file_name = yield self.getProperty("package_data")['name'] + '-' + self.getProperty("version") + '.ebuild'
self.ebuild_file = yield os.path.join(self.repository_path, self.cp_path, self.file_name)
@@ -256,7 +256,9 @@ class CheckV(BuildStep):
if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None:
if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']:
addStepVData.append(GetCommitdata())
- addStepVData.append(portage_steps.SetEnvForEbuildSH())
+ #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH
+ #addStepVData.append(portage_steps.SetEnvForEbuildSH())
+ addStepVData.append(portage_steps.GetAuxMetadata())
addStepVData.append(AddVersion())
addStepVData.append(AddVersionKeyword())
addStepVData.append(TriggerBuildCheck())
@@ -265,7 +267,9 @@ class CheckV(BuildStep):
return SUCCESS
if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None:
addStepVData.append(GetCommitdata())
- addStepVData.append(portage_steps.SetEnvForEbuildSH())
+ #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH
+ #addStepVData.append(portage_steps.SetEnvForEbuildSH())
+ addStepVData.append(portage_steps.GetAuxMetadata())
addStepVData.append(AddVersion())
addStepVData.append(AddVersionKeyword())
addStepVData.append(TriggerBuildCheck())
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-03-29 0:34 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-03-29 0:34 UTC (permalink / raw
To: gentoo-commits
commit: f4ee7748cdd77534ace36b6bd957f27229fbd748
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 29 00:24:46 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Mar 29 00:24:46 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f4ee7748
Change code for the PersOutputOfEbuildSH so we may fix *deps to long lines
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 32 +++++++++++++++++++-------------
buildbot_gentoo_ci/steps/version.py | 10 ++++++----
2 files changed, 25 insertions(+), 17 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index e2315cb..293b862 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -32,27 +32,33 @@ def WriteTextToFile(path, text_list):
def PersOutputOfEbuildSH(rc, stdout, stderr):
metadata = None
- metadata_lines = stdout.splitlines()
- metadata_valid = True
NoSplit = []
NoSplit.append('DESCRIPTION')
- if len(auxdbkeys) != len(metadata_lines):
- # Don't trust bash's returncode if the
+ #make dict of the stout
+ index = 1
+ metadata_line_dict = {}
+ for text_line in stdout.splitlines():
+ metadata_line_dict[index] = text_line
+ index = index + 1
+ # should have 22 lines
+ if len(auxdbkeys) != index -1:
# number of lines is incorrect.
return {
'auxdb' : metadata
}
- else:
- metadata_tmp = dict(zip(auxdbkeys, metadata_lines))
+ # split all keys to list instead of speces
metadata = {}
- for k, v in metadata_tmp.items():
- if v == '':
- metadata[k] = None
+ i = 1
+ for key in auxdbkeys:
+ if metadata_line_dict[i] == '':
+ metadata[key] = None
else:
- if ' ' in v and k not in NoSplit:
- metadata[k] = v.split(' ')
+ if ' ' in metadata_line_dict[i] and key not in NoSplit:
+ metadata[key] = metadata_line_dict[i].split(' ')
else:
- metadata[k] = v
+ metadata[key] = []
+ metadata[key].append(metadata_line_dict[i])
+ i = i + 1
return {
'auxdb' : metadata
}
@@ -519,7 +525,7 @@ class SetEnvForEbuildSH(BuildStep):
command=ebuild_commands,
env=ebuild_env,
workdir=self.getProperty("builddir"),
- strip=True,
+ strip=False,
extract_fn=PersOutputOfEbuildSH
))
yield self.build.addStepsAfterCurrentStep(addStepEbuildSH)
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index 143a758..4c35914 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -125,9 +125,13 @@ class AddVersionKeyword(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.version_keyword_dict = {}
auxdb = self.getProperty("auxdb")['KEYWORDS']
- if auxdb is None:
- auxdb = []
+ if auxdb is None or not isinstance(auxdb, list):
+ self.version_keyword_dict = None
+ self.setProperty('version_keyword_dict', self.version_keyword_dict, 'version_keyword_dict')
+ return SUCCESS
+ print(auxdb)
for keyword in auxdb:
+ print(keyword)
status = 'stable'
if keyword[0] in ["~"]:
keyword = keyword[1:]
@@ -147,8 +151,6 @@ class AddVersionKeyword(BuildStep):
version_keyword_data['keyword_id'],
version_keyword_data['status'])
self.version_keyword_dict[keyword] = version_keyword_data
- if self.version_keyword_dict == {}:
- self.version_keyword_dict = None
self.setProperty('version_keyword_dict', self.version_keyword_dict, 'version_keyword_dict')
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-03-29 0:34 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-03-29 0:34 UTC (permalink / raw
To: gentoo-commits
commit: e01a8423d8cc9371320ea0016107d13c3a7dd991
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 29 00:33:07 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Mar 29 00:33:07 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e01a8423
Change code in TriggerCheckForCPV for trigging
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/update_db.py | 50 +++++++++++++++++------------------
1 file changed, 25 insertions(+), 25 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index d35a967..b84e256 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -181,40 +181,40 @@ class TriggerCheckForCPV(BuildStep):
# check if git_change is a string or a list
if not isinstance(self.git_changes, list):
return FAILURE
- self.success = True
addStepUpdateCPVData = []
for change_data in self.git_changes:
# make a trigger for all cpv in the list
for cpv in change_data['cpvs']:
+ self.success = True
+ if change_data['repository'] != self.getProperty("repository_data")['name']:
+ log.msg("%s don't match" % change_data['repository'])
+ self.success = False
+ # Trigger cpv builds and update db if we are working with ebuilds
# check that cpv is valied
if catpkgsplit(cpv) is None:
log.msg("%s is not vaild package name" % cpv)
self.success = False
- else:
- if change_data['repository'] != self.getProperty("repository_data")['name']:
- log.msg("%s don't match" % change_data['repository'])
- self.success = False
- else:
- revision_data = {}
- revision_data['author'] = change_data['author']
- revision_data['committer'] = change_data['committer']
- revision_data['comments'] = change_data['comments']
- revision_data['revision'] = change_data['revision']
- # call update_cpv_data
- addStepUpdateCPVData.append(
- steps.Trigger(
- schedulerNames=['update_cpv_data'],
- waitForFinish=False,
- updateSourceStamp=False,
- set_properties={
- 'cpv' : cpv,
- 'project_data' : self.getProperty("project_data"),
- 'repository_data' : self.getProperty("repository_data"),
- 'revision_data' : revision_data,
- }
- )
+ if self.success:
+ revision_data = {}
+ revision_data['author'] = change_data['author']
+ revision_data['committer'] = change_data['committer']
+ revision_data['comments'] = change_data['comments']
+ revision_data['revision'] = change_data['revision']
+ addStepUpdateCPVData.append(
+ steps.Trigger(
+ schedulerNames=['update_cpv_data'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : cpv,
+ 'project_data' : self.getProperty("project_data"),
+ 'repository_data' : self.getProperty("repository_data"),
+ 'revision_data' : revision_data,
+ }
)
- yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
+ )
+ print(addStepUpdateCPVData)
+ yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
if self.success is False:
return FAILURE
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-03-20 13:42 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-03-20 13:42 UTC (permalink / raw
To: gentoo-commits
commit: f2a3a0fdedff108964c0ffee79cd2757edcb6f32
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 20 13:40:11 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Mar 20 13:40:11 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f2a3a0fd
Check that keyword is in version_keyword_dict
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index d09c4b6..22608e3 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -163,10 +163,11 @@ class GetProjectRepositoryData(BuildStep):
if project_keyword_data['name'] != '*' or project_data['status'] == 'all':
self.setProperty('fullcheck', False, 'fullcheck')
# get status of the keyword on cpv
- version_keywords_data = self.getProperty("version_keyword_dict")[project_keyword_data['name']]
- # if unstable trigger BuildRequest on cpv
- if project_data['status'] == version_keywords_data['status']:
- yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
+ if project_keyword_data['name'] in self.getProperty("version_keyword_dict"):
+ version_keywords_data = self.getProperty("version_keyword_dict")[project_keyword_data['name']]
+ # if match trigger BuildRequest on cpv
+ if project_data['status'] == version_keywords_data['status']:
+ yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
return SUCCESS
class SetupPropertys(BuildStep):
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-03-20 13:42 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-03-20 13:42 UTC (permalink / raw
To: gentoo-commits
commit: ee9e8401620340982d6e922ca982f50fee1ec6fa
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 20 13:34:03 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Mar 20 13:34:03 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ee9e8401
Change GetCommitdata to use revision_data. Use SetEnvForEbuildSH to get auxdb data.
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/master.py | 235 ++++++++++++++++++++++++++++++++++++
buildbot_gentoo_ci/steps/portage.py | 130 +++++++++++++++++++-
buildbot_gentoo_ci/steps/version.py | 66 +++-------
3 files changed, 380 insertions(+), 51 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/master.py b/buildbot_gentoo_ci/steps/master.py
new file mode 100644
index 0000000..dac3830
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/master.py
@@ -0,0 +1,235 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.steps.master.py
+# buildbot.steps.shell.py
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import os
+import pprint
+import re
+
+from twisted.internet import defer
+from twisted.internet import error
+from twisted.internet import reactor
+from twisted.internet.protocol import ProcessProtocol
+from twisted.python import runtime
+from twisted.python.versions import Version
+
+from buildbot.process import buildstep
+from buildbot.process import logobserver
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SUCCESS
+from buildbot.util import deferwaiter
+
+
+class MasterSetPropertyFromCommand(buildstep.ShellMixin, buildstep.BuildStep):
+
+ """
+ Run a shell command locally - on the buildmaster. The shell command
+ COMMAND is specified just as for a RemoteShellCommand. Note that extra
+ logfiles are not supported.
+ """
+ name = "Mastersetproperty"
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ renderables = ['command', 'env', 'property']
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, command, property=None, extract_fn=None, strip=True,
+ includeStdout=True, includeStderr=False, **kwargs):
+ self.env = kwargs.pop('env', None)
+ self.usePTY = kwargs.pop('usePTY', 0)
+ self.interruptSignal = kwargs.pop('interruptSignal', 'KILL')
+ self.logEnviron = kwargs.pop('logEnviron', True)
+
+ self.property = property
+ self.extract_fn = extract_fn
+ self.strip = strip
+ self.includeStdout = includeStdout
+ self.includeStderr = includeStderr
+
+ if not ((property is not None) ^ (extract_fn is not None)):
+ config.error(
+ "Exactly one of property and extract_fn must be set")
+
+ super().__init__(**kwargs)
+
+ self.command = command
+ self.masterWorkdir = self.workdir
+ self._deferwaiter = deferwaiter.DeferWaiter()
+ self._status_object = None
+ self.success = True
+
+ if self.extract_fn:
+ self.includeStderr = True
+
+ self.observer = logobserver.BufferLogObserver(
+ wantStdout=self.includeStdout,
+ wantStderr=self.includeStderr)
+ self.addLogObserver('stdio', self.observer)
+
+ class LocalPP(ProcessProtocol):
+
+ def __init__(self, step):
+ self.step = step
+ self._finish_d = defer.Deferred()
+ self.step._deferwaiter.add(self._finish_d)
+
+ def outReceived(self, data):
+ self.step._deferwaiter.add(self.step.stdio_log.addStdout(data))
+
+ def errReceived(self, data):
+ self.step._deferwaiter.add(self.step.stdio_log.addStderr(data))
+
+ def processEnded(self, status_object):
+ if status_object.value.exitCode is not None:
+ msg = "exit status {}\n".format(status_object.value.exitCode)
+ self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg))
+
+ if status_object.value.signal is not None:
+ msg = "signal {}\n".format(status_object.value.signal)
+ self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg))
+
+ self.step._status_object = status_object
+ self._finish_d.callback(None)
+
+ @defer.inlineCallbacks
+ def run(self):
+ # render properties
+ command = self.command
+ # set up argv
+ if isinstance(command, (str, bytes)):
+ if runtime.platformType == 'win32':
+ # allow %COMSPEC% to have args
+ argv = os.environ['COMSPEC'].split()
+ if '/c' not in argv:
+ argv += ['/c']
+ argv += [command]
+ else:
+ # for posix, use /bin/sh. for other non-posix, well, doesn't
+ # hurt to try
+ argv = ['/bin/sh', '-c', command]
+ else:
+ if runtime.platformType == 'win32':
+ # allow %COMSPEC% to have args
+ argv = os.environ['COMSPEC'].split()
+ if '/c' not in argv:
+ argv += ['/c']
+ argv += list(command)
+ else:
+ argv = command
+
+ self.stdio_log = yield self.addLog("stdio")
+
+ if isinstance(command, (str, bytes)):
+ yield self.stdio_log.addHeader(command.strip() + "\n\n")
+ else:
+ yield self.stdio_log.addHeader(" ".join(command) + "\n\n")
+ yield self.stdio_log.addHeader("** RUNNING ON BUILDMASTER **\n")
+ yield self.stdio_log.addHeader(" in dir {}\n".format(os.getcwd()))
+ yield self.stdio_log.addHeader(" argv: {}\n".format(argv))
+
+ if self.env is None:
+ env = os.environ
+ else:
+ assert isinstance(self.env, dict)
+ env = self.env
+ for key, v in self.env.items():
+ if isinstance(v, list):
+ # Need to do os.pathsep translation. We could either do that
+ # by replacing all incoming ':'s with os.pathsep, or by
+ # accepting lists. I like lists better.
+ # If it's not a string, treat it as a sequence to be
+ # turned in to a string.
+ self.env[key] = os.pathsep.join(self.env[key])
+
+ # do substitution on variable values matching pattern: ${name}
+ p = re.compile(r'\${([0-9a-zA-Z_]*)}')
+
+ def subst(match):
+ return os.environ.get(match.group(1), "")
+ newenv = {}
+ for key, v in env.items():
+ if v is not None:
+ if not isinstance(v, (str, bytes)):
+ raise RuntimeError(("'env' values must be strings or "
+ "lists; key '{}' is incorrect").format(key))
+ newenv[key] = p.sub(subst, env[key])
+ env = newenv
+
+ if self.logEnviron:
+ yield self.stdio_log.addHeader(" env: %r\n" % (env,))
+
+ # TODO add a timeout?
+ self.process = reactor.spawnProcess(self.LocalPP(self), argv[0], argv,
+ path=self.masterWorkdir, usePTY=self.usePTY, env=env)
+
+ # self._deferwaiter will yield only after LocalPP finishes
+
+ yield self._deferwaiter.wait()
+ status_value = self._status_object.value
+ if status_value.signal is not None:
+ self.descriptionDone = ["killed ({})".format(status_value.signal)]
+ self.success = False
+ elif status_value.exitCode != 0:
+ self.descriptionDone = ["failed ({})".format(status_value.exitCode)]
+ self.success = False
+
+
+ yield self.stdio_log.finish()
+
+ property_changes = {}
+
+ if self.property:
+ if not self.success:
+ return FAILURE
+ result = self.observer.getStdout()
+ if self.strip:
+ result = result.strip()
+ propname = self.property
+ self.setProperty(propname, result, "MastertSetPropertyFromCommand Step")
+ property_changes[propname] = result
+ else:
+ new_props = self.extract_fn(self._status_object.value.exitCode,
+ self.observer.getStdout(),
+ self.observer.getStderr())
+ for k, v in new_props.items():
+ self.setProperty(k, v, "MasterSetPropertyFromCommand Step")
+ property_changes = new_props
+
+ props_set = ["{}: {}".format(k, repr(v))
+ for k, v in sorted(property_changes.items())]
+ yield self.addCompleteLog('property changes', "\n".join(props_set))
+
+ if len(property_changes) > 1:
+ self.descriptionDone = '{} properties set'.format(len(property_changes))
+ elif len(property_changes) == 1:
+ self.descriptionDone = 'property \'{}\' set'.format(list(property_changes)[0])
+ if not self.success:
+ return FAILURE
+ return SUCCESS
+
+ def interrupt(self, reason):
+ try:
+ self.process.signalProcess(self.interruptSignal)
+ except KeyError: # Process not started yet
+ pass
+ except error.ProcessExitedAlready:
+ pass
+ super().interrupt(reason)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 4fbe141..6d8388a 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -2,6 +2,14 @@
# Distributed under the terms of the GNU General Public License v2
import os
+import io
+
+from portage import config as portage_config
+from portage import auxdbkeys
+from portage import _encodings
+from portage import _unicode_encode
+from portage import _parse_eapi_ebuild_head, eapi_is_supported
+from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
from twisted.internet import defer
from twisted.python import log
@@ -11,6 +19,8 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
+from buildbot_gentoo_ci.steps import master as master_steps
+
@defer.inlineCallbacks
def WriteTextToFile(path, text_list):
separator = '\n'
@@ -20,6 +30,33 @@ def WriteTextToFile(path, text_list):
yield f.write(separator)
yield f.close
+def PersOutputOfEbuildSH(rc, stdout, stderr):
+ metadata = None
+ metadata_lines = stdout.splitlines()
+ metadata_valid = True
+ NoSplit = []
+ NoSplit.append('DESCRIPTION')
+ if len(auxdbkeys) != len(metadata_lines):
+ # Don't trust bash's returncode if the
+ # number of lines is incorrect.
+ return {
+ 'auxdb' : metadata
+ }
+ else:
+ metadata_tmp = dict(zip(auxdbkeys, metadata_lines))
+ metadata = {}
+ for k, v in metadata_tmp.items():
+ if v == '':
+ metadata[k] = None
+ else:
+ if ' ' in v and k not in NoSplit:
+ metadata[k] = v.split(' ')
+ else:
+ metadata[k] = v
+ return {
+ 'auxdb' : metadata
+ }
+
class SetMakeProfile(BuildStep):
name = 'SetMakeProfile'
@@ -316,7 +353,7 @@ class SetMakeProfileLocal(BuildStep):
@defer.inlineCallbacks
def run(self):
- parent_path = yield os.path.join('portage', 'make.profile', 'parent')
+ parent_path = yield os.path.join('etc','portage', 'make.profile', 'parent')
if os.path.isfile(parent_path):
return SUCCESS
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
@@ -325,7 +362,7 @@ class SetMakeProfileLocal(BuildStep):
makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'make.profile')
for makeprofile in makeprofiles_data:
makeprofile_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name'], 'profiles', makeprofile['value'], '')
- makeprofiles_paths.append('../../' + makeprofile_path)
+ makeprofiles_paths.append('../../../' + makeprofile_path)
yield WriteTextToFile(parent_path, makeprofiles_paths)
return SUCCESS
@@ -343,7 +380,7 @@ class SetReposConfLocal(BuildStep):
@defer.inlineCallbacks
def run(self):
- repos_conf_path = yield os.path.join('portage', 'repos.conf')
+ repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf')
repos_conf_default_path = yield os.path.join(repos_conf_path, 'default.conf')
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir = self.gentooci.config.project['repository_basedir']
@@ -360,7 +397,7 @@ class SetReposConfLocal(BuildStep):
yield WriteTextToFile(repos_conf_default_path, default_conf)
repos_conf_repository_path = yield os.path.join(repos_conf_path, self.getProperty("repository_data")['name'] + '.conf')
if not os.path.isfile(repos_conf_repository_path):
- repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'])
+ repository_path = yield os.path.join(self.getProperty("builddir"), self.repository_basedir, self.getProperty("repository_data")['name'])
repository_conf = []
repository_conf.append('[' + self.getProperty("repository_data")['name'] + ']')
repository_conf.append('location = ' + repository_path)
@@ -384,7 +421,7 @@ class SetMakeConfLocal(BuildStep):
@defer.inlineCallbacks
def run(self):
- make_conf_path = yield os.path.join('portage', 'make.conf')
+ make_conf_path = yield os.path.join('etc', 'portage', 'make.conf')
if os.path.isfile(make_conf_path):
return SUCCESS
makeconf_list = []
@@ -398,3 +435,86 @@ class SetMakeConfLocal(BuildStep):
makeconf_list.append('FEATURES=""')
yield WriteTextToFile(make_conf_path, makeconf_list)
return SUCCESS
+
+class SetEnvForEbuildSH(BuildStep):
+
+ name = 'SetEnvForEbuildSH'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ def getEapiFromFile(self):
+ with io.open(_unicode_encode(self.getProperty("ebuild_file"),
+ encoding=_encodings['fs'], errors='strict'),
+ mode='r', encoding=_encodings['repo.content'],
+ errors='replace') as f:
+ _eapi, _eapi_lineno = _parse_eapi_ebuild_head(f)
+
+ return _eapi
+
+ @defer.inlineCallbacks
+ def run(self):
+ addStepEbuildSH = []
+ ebuild_commands = []
+ ebuild_env = {}
+ config_root = yield os.path.join(self.getProperty("builddir"), '')
+ mysettings = yield portage_config(config_root = config_root)
+
+ #Get EAPI from file and add it to env
+ eapi = yield self.getEapiFromFile()
+ print(eapi)
+ if eapi is None or not eapi_is_supported(eapi):
+ print('invalid eapi')
+ eapi = '0'
+ print(eapi_is_supported(eapi))
+ ebuild_env['EAPI'] = eapi
+
+ #FIXME: check manifest on ebuild_file
+
+ #Setup ENV
+ category = yield catpkgsplit(self.getProperty("cpv"))[0]
+ package = yield catpkgsplit(self.getProperty("cpv"))[1]
+ version = yield catpkgsplit(self.getProperty("cpv"))[2]
+ revision = yield catpkgsplit(self.getProperty("cpv"))[3]
+ portage_bin_path = mysettings["PORTAGE_BIN_PATH"]
+ ebuild_sh_path = yield os.path.join(portage_bin_path, 'ebuild.sh')
+ #ebuild_env['PORTAGE_DEBUG'] = '1'
+ ebuild_env['EBUILD_PHASE'] = 'depend'
+ ebuild_env['CATEGORY'] = category
+ ebuild_env['P'] = package + '-' + version
+ ebuild_env['PN'] = package
+ ebuild_env['PR'] = revision
+ ebuild_env['PV'] = version
+ if revision == 'r0':
+ ebuild_env['PF'] = ebuild_env['P']
+ ebuild_env['PVR'] = version
+ else:
+ ebuild_env['PF'] = ebuild_env['P'] + '-' + revision
+ ebuild_env['PVR'] = version + '-' + revision
+ ebuild_env['PORTAGE_BIN_PATH'] = portage_bin_path
+ ebuild_env['EBUILD'] = self.getProperty("ebuild_file")
+ ebuild_env['PORTAGE_PIPE_FD'] = '1'
+ ebuild_env['WORKDIR'] = yield os.path.join(mysettings["PORTAGE_TMPDIR"], 'portage', category, ebuild_env['PF'], 'work')
+ ebuild_env['PORTAGE_ECLASS_LOCATIONS'] = self.getProperty("repository_path")
+
+ #FIXME: use sandbox if in FEATURES
+ ebuild_commands.append(ebuild_sh_path)
+ ebuild_commands.append('depend')
+
+ addStepEbuildSH.append(master_steps.MasterSetPropertyFromCommand(
+ name = 'RunEbuildSH',
+ haltOnFailure = True,
+ flunkOnFailure = True,
+ command=ebuild_commands,
+ env=ebuild_env,
+ workdir=self.getProperty("builddir"),
+ strip=True,
+ extract_fn=PersOutputOfEbuildSH
+ ))
+ yield self.build.addStepsAfterCurrentStep(addStepEbuildSH)
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index d47a253..143a758 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -7,10 +7,7 @@ import git
from portage.xml.metadata import MetaDataXML
from portage.checksum import perform_checksum
-from portage.versions import cpv_getversion, pkgsplit
-from portage import auxdbkeys
-from portage import config as portage_config
-from portage import portdbapi
+from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
from twisted.internet import defer
from twisted.python import log
@@ -20,6 +17,8 @@ from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.plugins import steps
+from buildbot_gentoo_ci.steps import portage as portage_steps
+
class GetVData(BuildStep):
name = 'GetVData'
@@ -34,6 +33,8 @@ class GetVData(BuildStep):
@defer.inlineCallbacks
def run(self):
+ # set cwd to builddir
+ yield os.chdir(self.getProperty("builddir"))
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.version = yield cpv_getversion(self.getProperty("cpv"))
print(self.version)
@@ -73,33 +74,7 @@ class AddVersion(BuildStep):
self.setProperty("version_data", self.version_data, 'version_data')
return SUCCESS
-class GetAuxMetadata(BuildStep):
-
- name = 'GetAuxMetadata'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- @defer.inlineCallbacks
- def run(self):
- self.mysettings = yield portage_config(config_root = self.getProperty("config_root"))
- self.myportdb = yield portdbapi(mysettings=self.mysettings)
- try:
- auxdb_list = yield self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys, myrepo=self.getProperty("repository_data")['name'])
- except:
- print("Failed to get aux data for %s" % self.getProperty("cpv"))
- yield self.myportdb.close_caches()
- yield portdbapi.portdbapi_instances.remove(self.myportdb)
- return FAILURE
- self.setProperty('aux_metadata', auxdb_list, 'aux_metadata')
- yield self.myportdb.close_caches()
- yield portdbapi.portdbapi_instances.remove(self.myportdb)
- return SUCCESS
class GetCommitdata(BuildStep):
@@ -113,19 +88,10 @@ class GetCommitdata(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
- @defer.inlineCallbacks
+ #@defer.inlineCallbacks
def run(self):
- #FIXME: Could be a better way to get the log
- git_log_ebuild = ''
- g = git.Git(self.getProperty("repository_path"))
- index = 1
- git_log_dict = {}
- git_log = yield g.log('-n 1', self.getProperty("ebuild_file"))
- print(git_log)
- for line in git_log.splitlines():
- git_log_dict[index] = line
- index = index + 1
- self.setProperty('commit_id', re.sub('commit ', '', git_log_dict[1]), 'commit_id')
+ print(self.getProperty("revision_data"))
+ self.setProperty('commit_id', self.getProperty("revision_data")['revision'], 'commit_id')
return SUCCESS
class AddVersionKeyword(BuildStep):
@@ -158,7 +124,10 @@ class AddVersionKeyword(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.version_keyword_dict = {}
- for keyword in self.getProperty("aux_metadata")[8].split():
+ auxdb = self.getProperty("auxdb")['KEYWORDS']
+ if auxdb is None:
+ auxdb = []
+ for keyword in auxdb:
status = 'stable'
if keyword[0] in ["~"]:
keyword = keyword[1:]
@@ -178,6 +147,8 @@ class AddVersionKeyword(BuildStep):
version_keyword_data['keyword_id'],
version_keyword_data['status'])
self.version_keyword_dict[keyword] = version_keyword_data
+ if self.version_keyword_dict == {}:
+ self.version_keyword_dict = None
self.setProperty('version_keyword_dict', self.version_keyword_dict, 'version_keyword_dict')
return SUCCESS
@@ -197,7 +168,7 @@ class CheckPathHash(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir = self.gentooci.config.project['repository_basedir']
- self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'] + '.git')
+ self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'])
self.cp_path = yield pkgsplit(self.getProperty("cpv"))[0]
self.file_name = yield self.getProperty("package_data")['name'] + '-' + self.getProperty("version") + '.ebuild'
self.ebuild_file = yield os.path.join(self.repository_path, self.cp_path, self.file_name)
@@ -274,14 +245,17 @@ class CheckV(BuildStep):
self.old_version_data = self.getProperty("old_version_data")
self.ebuild_file = self.getProperty("ebuild_file")
addStepVData = []
+ print(self.ebuild_file)
+ print(self.old_version_data)
+ print(self.getProperty("ebuild_file_hash"))
if self.getProperty("ebuild_file") is None and self.getProperty("old_version_data") is not None:
addStepVData.append(TriggerBuildCheck())
addStepVData.append(DeleteOldVersion())
if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None:
if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']:
addStepVData.append(GetCommitdata())
+ addStepVData.append(portage_steps.SetEnvForEbuildSH())
addStepVData.append(AddVersion())
- addStepVData.append(GetAuxMetadata())
addStepVData.append(AddVersionKeyword())
addStepVData.append(TriggerBuildCheck())
addStepVData.append(DeleteOldVersion())
@@ -289,8 +263,8 @@ class CheckV(BuildStep):
return SUCCESS
if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None:
addStepVData.append(GetCommitdata())
+ addStepVData.append(portage_steps.SetEnvForEbuildSH())
addStepVData.append(AddVersion())
- addStepVData.append(GetAuxMetadata())
addStepVData.append(AddVersionKeyword())
addStepVData.append(TriggerBuildCheck())
yield self.build.addStepsAfterCurrentStep(addStepVData)
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-03-20 13:42 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-03-20 13:42 UTC (permalink / raw
To: gentoo-commits
commit: c852e4cb0089da63776d0b5939eadd1180da7080
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 20 13:41:42 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Mar 20 13:41:42 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c852e4cb
Fix gitpull back to origin
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/update_db.py | 45 +++++++++++++++++++----------------
1 file changed, 25 insertions(+), 20 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index bfcd2ef..d35a967 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -65,7 +65,7 @@ class CheckPath(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir = self.gentooci.config.project['repository_basedir']
- self.portage_path = 'portage'
+ self.portage_path = yield os.path.join('etc', 'portage')
self.profile_path = yield os.path.join(self.portage_path, 'make.profile')
self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
print(os.getcwd())
@@ -74,10 +74,11 @@ class CheckPath(BuildStep):
success = True
print(os.getcwd())
for x in [
- self.profile_path,
- self.repos_path,
- self.repository_basedir
- ]:
+ self.portage_path,
+ self.profile_path,
+ self.repos_path,
+ self.repository_basedir
+ ]:
if not os.path.isdir(x):
os.makedirs(x)
return SUCCESS
@@ -96,48 +97,52 @@ class UpdateRepos(BuildStep):
# Origin: https://github.com/MichaelBoselowitz/pygit2-examples/blob/master/examples.py#L54
# Modifyed by Gentoo Authors.
- @defer.inlineCallbacks
def gitPull(self, repo, remote_name='origin', branch='master'):
for remote in repo.remotes:
if remote.name == remote_name:
- yield remote.fetch()
- remote_master_id = yield repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
- merge_result, _ = yield repo.merge_analysis(remote_master_id)
+ remote.fetch()
+ remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
+ print(remote_master_id)
+ merge_result, _ = repo.merge_analysis(remote_master_id)
+ print(merge_result)
# Up to date, do nothing
if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
+ print('UP_TO_DATE')
return
# We can just fastforward
elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
- yield repo.checkout_tree(repo.get(remote_master_id))
+ print('FASTFORWARD')
+ repo.checkout_tree(repo.get(remote_master_id))
try:
- master_ref = yield repo.lookup_reference('refs/heads/%s' % (branch))
- yield master_ref.set_target(remote_master_id)
+ master_ref = repo.lookup_reference('refs/heads/%s' % (branch))
+ master_ref.set_target(remote_master_id)
except KeyError:
- yield repo.create_branch(branch, repo.get(remote_master_id))
- yield repo.head.set_target(remote_master_id)
+ repo.create_branch(branch, repo.get(remote_master_id))
+ repo.head.set_target(remote_master_id)
elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
- yield repo.merge(remote_master_id)
-
+ print('NORMAL')
+ repo.merge(remote_master_id)
if repo.index.conflicts is not None:
for conflict in repo.index.conflicts:
print('Conflicts found in:', conflict[0].path)
raise AssertionError('Conflicts, ahhhhh!!')
- user = yield repo.default_signature
- tree = yield repo.index.write_tree()
- commit = yield repo.create_commit('HEAD',
+ user = repo.default_signature
+ tree = repo.index.write_tree()
+ commit = repo.create_commit('HEAD',
user,
user,
'Merge!',
tree,
[repo.head.target, remote_master_id])
# We need to do this or git CLI will think we are still merging.
- yield repo.state_cleanup()
+ repo.state_cleanup()
else:
raise AssertionError('Unknown merge analysis result')
@defer.inlineCallbacks
def run(self):
+ #FIXME check HEAD agenst local and worker local tree so we don't gitpull evrytime
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir = self.gentooci.config.project['repository_basedir']
self.profile_repository_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name'])
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-02-28 18:48 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-02-28 18:48 UTC (permalink / raw
To: gentoo-commits
commit: 95436406f0010275ce5fdb6142b916ca66f91af0
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Feb 28 18:47:19 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Feb 28 18:47:19 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=95436406
Don't use binary or build some packages
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 32 ++++++++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 81d6e95..21472c7 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -255,6 +255,11 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--changed-use')
shell_commad_list.append('--pretend')
shell_commad_list.append('@world')
+ # don't build bin for virtual and acct-*
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('virtual')
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
@@ -269,6 +274,11 @@ class RunEmerge(BuildStep):
shell_commad_list.append('--changed-deps')
shell_commad_list.append('--changed-use')
shell_commad_list.append('@world')
+ # don't build bin for virtual and acct-*
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('virtual')
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
@@ -338,8 +348,19 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckEmergeLogs('match'))
if self.step == 'pre-build':
+ cpv = self.getProperty("cpv")
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
shell_commad_list.append('-p')
shell_commad_list.append('=' + self.getProperty('cpv'))
+ # we don't use the bin for the requsted cpv
+ shell_commad_list.append('--usepkg-exclude')
+ shell_commad_list.append(c + '/' + p)
+ # don't build bin for virtual and acct-*
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('virtual')
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
@@ -351,10 +372,21 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckEmergeLogs('pre-build'))
if self.step == 'build':
+ cpv = self.getProperty("cpv")
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
shell_commad_list.append('-q')
if projects_emerge_options['oneshot']:
shell_commad_list.append('-1')
shell_commad_list.append('=' + self.getProperty('cpv'))
+ # we don't use the bin for the requsted cpv
+ shell_commad_list.append('--usepkg-exclude')
+ shell_commad_list.append(c + '/' + p)
+ # don't build bin for virtual and acct-*
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('virtual')
+ shell_commad_list.append('--buildpkg-exclude')
+ shell_commad_list.append('acct-*')
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
command=shell_commad_list,
^ permalink raw reply related [flat|nested] 72+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/
@ 2021-02-24 21:27 Magnus Granberg
0 siblings, 0 replies; 72+ messages in thread
From: Magnus Granberg @ 2021-02-24 21:27 UTC (permalink / raw
To: gentoo-commits
commit: 2da4defbcae32889b60c852179d1d718bcccbf57
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 24 21:26:06 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Feb 24 21:26:06 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2da4defb
Don't use binary in the match and fix a bug in the match logic
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/builders.py | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 3ecb78b..3c19345 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -475,6 +475,8 @@ class RunEmerge(BuildStep):
timeout=None
))
aftersteps_list.append(CheckEmergeLogs('update'))
+ if projects_emerge_options['preserved_libs']:
+ self.setProperty('preserved_libs', True, 'preserved-libs')
if self.step == 'preserved-libs' and self.getProperty('preserved_libs'):
shell_commad_list.append('-q')
@@ -501,9 +503,8 @@ class RunEmerge(BuildStep):
workdir='/'
))
aftersteps_list.append(CheckEmergeLogs('depclean'))
- self.setProperty('depclean', False, 'depclean')
- if self.step == 'depclean' and self.getProperty('depclean') and projects_emerge_options['depclean']:
+ if self.step == 'depclean' and self.getProperty('depclean'):
shell_commad_list.append('-q')
shell_commad_list.append('--depclean')
aftersteps_list.append(
@@ -520,6 +521,8 @@ class RunEmerge(BuildStep):
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
shell_commad_list.append('-pO')
+ # don't use bin for match
+ shell_commad_list.append('--usepkg=n')
shell_commad_list.append(c + '/' + p)
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
@@ -546,7 +549,8 @@ class RunEmerge(BuildStep):
if self.step == 'build':
shell_commad_list.append('-q')
- shell_commad_list.append('-1')
+ if projects_emerge_options['oneshot']:
+ shell_commad_list.append('-1')
shell_commad_list.append('=' + self.getProperty('cpv'))
aftersteps_list.append(
steps.SetPropertyFromCommandNewStyle(
@@ -557,6 +561,8 @@ class RunEmerge(BuildStep):
timeout=None
))
aftersteps_list.append(CheckEmergeLogs('build'))
+ if projects_emerge_options['preserved_libs']:
+ self.setProperty('preserved_libs', True, 'preserved-libs')
if not self.step is None and aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
@@ -598,7 +604,7 @@ class CheckEmergeLogs(BuildStep):
# FIXME: check if cpv match
if self.step == 'match'and self.getProperty('projectrepository_data')['build']:
- if emerge_output['package'][self.getProperty('cpv')]:
+ if self.getProperty('cpv') in emerge_output['package']:
self.setProperty('cpv_build', True, 'cpv_build')
print(self.getProperty('cpv_build'))
@@ -701,5 +707,7 @@ class RunBuild(BuildStep):
aftersteps_list = []
aftersteps_list.append(RunEmerge(step='pre-build'))
aftersteps_list.append(RunEmerge(step='build'))
+ self.setProperty('depclean', False, 'depclean')
+ self.setProperty('preserved_libs', False, 'preserved-libs')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
^ permalink raw reply related [flat|nested] 72+ messages in thread
end of thread, other threads:[~2024-10-16 20:50 UTC | newest]
Thread overview: 72+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-07-27 11:02 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2024-10-16 20:50 Magnus Granberg
2024-03-31 19:40 Magnus Granberg
2024-03-31 19:40 Magnus Granberg
2024-03-23 14:13 Magnus Granberg
2024-03-21 16:56 Magnus Granberg
2024-03-19 17:19 Magnus Granberg
2024-03-17 20:40 Magnus Granberg
2024-03-16 8:49 Magnus Granberg
2024-02-05 18:54 Magnus Granberg
2024-02-04 11:32 Magnus Granberg
2024-02-04 9:49 Magnus Granberg
2024-02-03 10:02 Magnus Granberg
2023-07-26 9:23 Magnus Granberg
2023-06-10 10:08 Magnus Granberg
2023-06-05 8:09 Magnus Granberg
2023-06-05 8:09 Magnus Granberg
2023-05-08 20:19 Magnus Granberg
2023-03-10 22:36 Magnus Granberg
2023-03-10 22:36 Magnus Granberg
2023-03-10 1:47 Magnus Granberg
2023-02-28 21:41 Magnus Granberg
2023-02-28 21:41 Magnus Granberg
2022-09-25 11:46 Magnus Granberg
2022-08-10 21:51 Magnus Granberg
2022-08-05 18:32 Magnus Granberg
2022-07-30 22:40 Magnus Granberg
2022-07-27 11:02 Magnus Granberg
2022-07-27 11:02 Magnus Granberg
2022-07-27 11:02 Magnus Granberg
2022-07-26 11:54 Magnus Granberg
2022-07-13 20:01 Magnus Granberg
2022-07-08 21:54 Magnus Granberg
2022-07-06 18:44 Magnus Granberg
2022-07-04 17:44 Magnus Granberg
2022-06-28 23:26 Magnus Granberg
2022-06-01 0:19 Magnus Granberg
2022-04-22 12:34 Magnus Granberg
2022-01-28 10:09 Magnus Granberg
2022-01-24 8:41 Magnus Granberg
2022-01-24 0:42 Magnus Granberg
2022-01-22 20:50 Magnus Granberg
2022-01-18 20:20 Magnus Granberg
2022-01-18 19:38 Magnus Granberg
2022-01-18 19:16 Magnus Granberg
2022-01-08 17:25 Magnus Granberg
2021-11-01 2:22 Magnus Granberg
2021-10-23 22:26 Magnus Granberg
2021-10-23 22:24 Magnus Granberg
2021-10-06 1:35 Magnus Granberg
2021-10-02 20:52 Magnus Granberg
2021-09-28 22:16 Magnus Granberg
2021-09-22 0:04 Magnus Granberg
2021-09-21 1:29 Magnus Granberg
2021-09-20 23:28 Magnus Granberg
2021-09-07 0:46 Magnus Granberg
2021-07-17 8:14 Magnus Granberg
2021-05-18 8:55 Magnus Granberg
2021-04-26 20:26 Magnus Granberg
2021-04-20 7:05 Magnus Granberg
2021-04-13 19:03 Magnus Granberg
2021-04-02 22:12 Magnus Granberg
2021-04-02 22:12 Magnus Granberg
2021-04-02 22:12 Magnus Granberg
2021-04-02 22:12 Magnus Granberg
2021-03-29 0:34 Magnus Granberg
2021-03-29 0:34 Magnus Granberg
2021-03-20 13:42 Magnus Granberg
2021-03-20 13:42 Magnus Granberg
2021-03-20 13:42 Magnus Granberg
2021-02-28 18:48 Magnus Granberg
2021-02-24 21:27 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox