public inbox for gentoo-commits@lists.gentoo.org
 help / color / mirror / Atom feed
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-03-18 23:18 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2021-03-18 23:18 UTC (permalink / raw
  To: gentoo-commits

commit:     04836c8c78369e1a5acbc460343b003c3a337f20
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 18 23:16:08 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Mar 18 23:16:08 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=04836c8c

Clean up Package and Category code

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  8 +++---
 buildbot_gentoo_ci/steps/category.py       | 25 ++++++++++-------
 buildbot_gentoo_ci/steps/package.py        | 44 +++++++++++++++++-------------
 3 files changed, 44 insertions(+), 33 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index f5a3b35..db48130 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -41,17 +41,17 @@ def update_db_cp():
     # else
     #   add category to db step
     #   return category_data
-    f.addStep(category.CheckCGentooCiProject())
+    f.addStep(category.CheckC())
     # if package in db
     #   return package_data
     #   add check package path step at end
     # else
     #   add package to db step
     #   return package_data
-    f.addStep(package.CheckPGentooCiProject())
+    f.addStep(package.CheckP())
     # Trigger new builders with v from cpv
-    #   return package_data, cpv, repository_data, project_data, config_root
-    f.addStep(package.TriggerCheckVGentooCiProject())
+    #   return package_data, cpv, repository_data, project_data
+    f.addStep(package.TriggerCheckForV())
     return f
 
 def update_db_v():

diff --git a/buildbot_gentoo_ci/steps/category.py b/buildbot_gentoo_ci/steps/category.py
index 1aca99a..81061e6 100644
--- a/buildbot_gentoo_ci/steps/category.py
+++ b/buildbot_gentoo_ci/steps/category.py
@@ -18,6 +18,13 @@ class AddCategory(BuildStep):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+    name = 'AddCategory'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
@@ -26,27 +33,25 @@ class AddCategory(BuildStep):
         self.category_data['uuid'] = yield self.gentooci.db.categorys.addCategory(self.category_data['name'])
         print(self.category_data)
         self.setProperty("category_data", self.category_data, 'category_data')
-        self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
-        self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
-        self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
-        self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
         return SUCCESS
 
-class CheckCGentooCiProject(BuildStep):
+class CheckC(BuildStep):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+    name = 'CheckC'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
         self.category = yield self.getProperty("cpv").split('/')[0]
         print(self.category)
-        print(self.getProperty("repository"))
         self.category_data = yield self.gentooci.db.categorys.getCategoryByName(self.category)
-        self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
-        self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
-        self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
-        self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
         print(self.category_data)
         if self.category_data is None:
             self.setProperty("category", self.category, 'category')

diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
index 3a28da2..ea5ab23 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -20,6 +20,13 @@ class AddPackage(BuildStep):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+    name = 'AddPackage'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
@@ -34,28 +41,25 @@ class AddPackage(BuildStep):
                                             )
         print(self.package_data)
         self.setProperty("package_data", self.package_data, 'package_data')
-        self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
-        self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
-        self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
-        self.setProperty("category_data", self.getProperty("category_data"), 'category_data')
-        self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
         return SUCCESS
 
-class CheckPGentooCiProject(BuildStep):
+class CheckP(BuildStep):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+    name = 'CheckP'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
         self.package = yield catpkgsplit(self.getProperty("cpv"))[1]
         print(self.package)
         self.package_data = yield self.gentooci.db.packages.getPackageByName(self.package)
-        self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
-        self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
-        self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
-        self.setProperty("category_data", self.getProperty("category_data"), 'category_data')
-        self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
         print(self.package_data)
         if self.package_data is None:
             self.setProperty("package", self.package, 'package')
@@ -66,10 +70,17 @@ class CheckPGentooCiProject(BuildStep):
         #yield self.build.addStepsAfterLastStep([CheckPathPackage()])
         return SUCCESS
 
-class TriggerCheckVGentooCiProject(BuildStep):
+class TriggerCheckForV(BuildStep):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+    name = 'TriggerCheckForV'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
     @defer.inlineCallbacks
     def run(self):
         addStepUpdateVData = []
@@ -81,17 +92,12 @@ class TriggerCheckVGentooCiProject(BuildStep):
                         set_properties={
                             'cpv' : self.getProperty("cpv"),
                             'package_data' : self.getProperty("package_data"),
-                            'config_root' : self.getProperty("config_root"),
                             'repository_data' : self.getProperty("repository_data"),
                             'category_data' : self.getProperty("category_data"),
+                            'revision_data' : self.getProperty("revision_data"),
+                            'project_data' : self.getProperty("project_data"),
                         }
                     )
                 )
         yield self.build.addStepsAfterCurrentStep(addStepUpdateVData)
-        self.setProperty("config_root", self.getProperty("config_root"), 'config_root')
-        self.setProperty("project_data", self.getProperty("project_data"), 'project_data')
-        self.setProperty("repository_data", self.getProperty("repository_data"), 'repository_data')
-        self.setProperty("category_data", self.getProperty("category_data"), 'category_data')
-        self.setProperty("package_data", self.getProperty("package_data"), 'package_data')
-        self.setProperty("cpv", self.getProperty("cpv"), 'cpv')
         return SUCCESS


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-03-24 23:15 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2021-03-24 23:15 UTC (permalink / raw
  To: gentoo-commits

commit:     b720c6bbcb79211a276d06a1eaf1543b89644767
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 24 23:14:47 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Mar 24 23:14:47 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b720c6bb

Add the log parser BuildFactory steps

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/builders.py      |  8 +++
 buildbot_gentoo_ci/config/buildfactorys.py | 22 +++++++-
 buildbot_gentoo_ci/config/schedulers.py    |  3 +
 buildbot_gentoo_ci/steps/builders.py       | 90 ++++++++++++++++++++----------
 buildbot_gentoo_ci/steps/portage.py        | 10 +++-
 5 files changed, 101 insertions(+), 32 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index c7b4469..0235753 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -49,4 +49,12 @@ def gentoo_builders(b=[]):
         factory=buildfactorys.run_build_request()
         )
     )
+    # FIXME: get workers from db
+    # Use multiplay workers
+    b.append(util.BuilderConfig(
+        name='parse_build_log',
+        workername='updatedb_1',
+        factory=buildfactorys.parse_build_log()
+        )
+    )
     return b

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index db48130..df7baef 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -10,6 +10,7 @@ from buildbot_gentoo_ci.steps import package
 from buildbot_gentoo_ci.steps import version
 from buildbot_gentoo_ci.steps import builders
 from buildbot_gentoo_ci.steps import portage
+from buildbot_gentoo_ci.steps import logs
 
 def update_db_check():
     f = util.BuildFactory()
@@ -159,5 +160,24 @@ def run_build_request():
     f.addStep(builders.RunEmerge(step='depclean'))
     f.addStep(builders.RunEmerge(step='preserved-libs'))
     f.addStep(builders.RunEmerge(step='depclean'))
-    f.addStep(builders.setBuildStatus())
+    return f
+
+def parse_build_log():
+    f = util.BuildFactory()
+    # FIXME: 6
+    # set needed Propertys
+    f.addStep(logs.SetupPropertys())
+    # pers the build log for info qa errors
+    #f.addStep(logs.ParserBuildLog())
+    # pers the log from pkg check
+    #f.addStep(logs.ParserPkgCheckLog())
+    # Upload the log to the cloud and remove the log
+    #f.addStep(logs.Upload())
+    # check the sum log if we need to make a issue/bug/pr report
+    # set it SUCCESS/FAILURE/WARNINGS
+    #f.addStep(logs.MakeIssue())
+    # set BuildStatus
+    #f.addStep(logs.setBuildStatus())
+    # setup things for the irc bot
+    #f.addStep(logs.SetIrcInfo())
     return f

diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index 0cbec96..efdac75 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -74,6 +74,8 @@ def gentoo_schedulers():
                                builderNames=["build_request_data"])
     run_build_request = schedulers.Triggerable(name="run_build_request",
                                builderNames=["run_build_request"])
+    pers_build_log = schedulers.Triggerable(name="parse_build_log",
+                               builderNames=["parse_build_log"])
     s = []
     s.append(test_updatedb)
     s.append(scheduler_update_db)
@@ -81,4 +83,5 @@ def gentoo_schedulers():
     s.append(update_v_data)
     s.append(build_request_data)
     s.append(run_build_request)
+    s.append(parse_build_log)
     return s

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 22608e3..ff6dd94 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -20,22 +20,26 @@ def PersOutputOfEmerge(rc, stdout, stderr):
     emerge_output['preserved_libs'] = False
     emerge_output['depclean'] = False
     package_dict = {}
+    log_path_list = []
     print(stderr)
     emerge_output['stderr'] = stderr
     # split the lines
     for line in stdout.split('\n'):
         # package list
         subdict = {}
-        if line.startswith('[ebuild') or line.startswith('[binary'):
+        subdict2 = {}
+        if line.startswith('[ebuild') or line.startswith('[binary') or line.startswith('[nomerge'):
             # if binaries
-            if line.startswith('[ebuild'):
+            if line.startswith('[ebuild') or line.startswith('[nomerge'):
                 subdict['binary'] = False
             else:
                 subdict['binary'] = True
             # action [ N ] stuff
             subdict['action'] = line[8:15].replace(' ', '')
             # cpv
+            #FIXME: We my have more then one spece betvine ] and cpv
             cpv_split = re.search('] (.+?) ', line).group(1).split(':')
+            print(cpv_split)
             cpv = cpv_split[0]
             # repository
             # slot
@@ -69,8 +73,12 @@ def PersOutputOfEmerge(rc, stdout, stderr):
             #FIXME: Handling of !!! output
             if line.startswith('!!! existing preserved libs'):
                 pass
+        if line.startswith(' * '):
+            if line.endswith('.log.gz'):
+                log_path_list.append(line.split(' ')[4])
         #FIXME: Handling of depclean output dict of packages that get removed or saved
     emerge_output['package'] = package_dict
+    emerge_output['log_paths'] = log_path_list
     # split the lines
     #FIXME: Handling of stderr output
     for line in stderr.split('\n'):
@@ -192,10 +200,14 @@ class SetupPropertys(BuildStep):
         projectrepository_data = self.getProperty('projectrepository_data')
         print(projectrepository_data)
         project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
+        repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
         self.setProperty('project_data', project_data, 'project_data')
+        self.setProperty('repository_data', repository_data, 'repository_data')
         self.setProperty('preserved_libs', False, 'preserved-libs')
         self.setProperty('depclean', False, 'depclean')
         self.setProperty('cpv_build', False, 'cpv_build')
+        self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
+        self.setProperty('faild_version_data', None, 'faild_version_data')
         print(self.getProperty("buildnumber"))
         if self.getProperty('project_build_data') is None:
             project_build_data = {}
@@ -472,6 +484,49 @@ class CheckEmergeLogs(BuildStep):
         if self.step == 'pre-build':
             print(emerge_output)
 
+        #FIXME:
+        # Look for FAILURE and logname and download needed logfile and
+        # trigger a logparser
+        # local_log_path dir set in config
+        # format /var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz
+        if self.step == 'build':
+            print(emerge_output)
+            log_dict = {}
+            # get cpv, logname and log path
+            for log_path in emerge_output['log_paths']:
+                c = log_path.split('/')[6]
+                full_logname = log_path.split('/')[7]
+                print(full_logname)
+                pv = full_logname.split(':')[0]
+                cpv = c + '/' + pv
+                log_dict[cpv] = dict(
+                                log_path = log_path,
+                                full_logname = full_logname
+                                )
+            print(log_dict)
+            # Find log for cpv that was requested or did faild
+            if not log_dict == {}:
+                # requested cpv
+                if self.getProperty('cpv') in log_dict:
+                    log_data = log_dict[self.getProperty('cpv')]
+                    masterdest = yield os.path.join(self.master.basedir, 'cpv_logs', log_data['full_logname'])
+                    aftersteps_list.append(steps.FileUpload(
+                        workersrc=log_data['log_path'],
+                        masterdest=masterdest
+                        ))
+                    aftersteps_list.append(steps.Trigger(
+                        schedulerNames=['parse_build_log'],
+                        waitForFinish=False,
+                        updateSourceStamp=False,
+                        set_properties={
+                            'cpv' : self.getProperty("cpv"),
+                            'faild_version_data' : self.getProperty('faild_version_data'),
+                            'project_build_data' : self.getProperty('project_build_data'),
+                            'log_build_data' : log_data,
+                            'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
+                            'repository_data' : self.getProperty('repository_data')
+                        }
+                    ))
         if not self.step is None and aftersteps_list != []:
             yield self.build.addStepsAfterCurrentStep(aftersteps_list)
         return SUCCESS
@@ -496,8 +551,7 @@ class RunPkgCheck(BuildStep):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
         project_data = self.getProperty('project_data')
         portage_repos_path = self.getProperty('portage_repos_path')
-        repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
-        repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
+        repository_path = yield os.path.join(portage_repos_path, self.getProperty('repository_data')['name'])
         cpv = self.getProperty("cpv")
         c = yield catpkgsplit(cpv)[0]
         p = yield catpkgsplit(cpv)[1]
@@ -544,7 +598,7 @@ class CheckPkgCheckLogs(BuildStep):
         print(pkgcheck_output)
         #FIXME:
         # Perse the logs
-        # tripp irc request with pkgcheck info
+        self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
         return SUCCESS
 
 class RunBuild(BuildStep):
@@ -562,6 +616,8 @@ class RunBuild(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         if not self.getProperty('cpv_build'):
+            #FIXME:
+            # trigger pars_build_log if we have any logs to check
             return SUCCESS
         aftersteps_list = []
         aftersteps_list.append(RunEmerge(step='pre-build'))
@@ -570,27 +626,3 @@ class RunBuild(BuildStep):
         self.setProperty('preserved_libs', False, 'preserved-libs')
         yield self.build.addStepsAfterCurrentStep(aftersteps_list)
         return SUCCESS
-
-class setBuildStatus(BuildStep):
-
-    name = 'setBuildStatus'
-    description = 'Running'
-    descriptionDone = 'Ran'
-    descriptionSuffix = None
-    haltOnFailure = True
-    flunkOnFailure = True
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    @defer.inlineCallbacks
-    def run(self):
-        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        project_build_data = self.getProperty('project_build_data')
-        if project_build_data['status'] == 'in-progress':
-            yield self.gentooci.db.builds.setSatusBuilds(
-                                                    project_build_data['build_id'],
-                                                    project_build_data['project_uuid'],
-                                                    'completed')
-            self.setProperty('project_build_data', project_build_data, 'project_build_data')
-        return SUCCESS

diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 6d8388a..e2315cb 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -190,6 +190,8 @@ class SetMakeConf(BuildStep):
                 makeconf_variable_list.append('cgroup')
                 makeconf_variable_list.append('-news')
                 makeconf_variable_list.append('-collision-protect')
+                makeconf_variable_list.append('split-log')
+                makeconf_variable_list.append('compress-build-logs')
             # EMERGE_DEFAULT_OPTS
             if k['variable'] == 'EMERGE_DEFAULT_OPTS':
                 makeconf_variable_list.append('--buildpkg=y')
@@ -201,6 +203,8 @@ class SetMakeConf(BuildStep):
                 makeconf_variable_list.append('--nospinner')
                 makeconf_variable_list.append('--color=n')
                 makeconf_variable_list.append('--ask=n')
+                makeconf_variable_list.append('--quiet-build=y')
+                makeconf_variable_list.append('--quiet-fail=y')
             # CFLAGS
             if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
                 makeconf_variable_list.append('-O2')
@@ -236,8 +240,10 @@ class SetMakeConf(BuildStep):
         makeconf_list.append('NOCOLOR=true')
         makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
         makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
-        makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
-        makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
+        makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"')
+        makeconf_list.append('PORTAGE_ELOG_CLASSES="*"')
+        # We need echo:info to get the logfile name
+        makeconf_list.append('PORTAGE_ELOG_SYSTEM="save echo:info"')
         # add ACCEPT_KEYWORDS from the project_data info
         keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
         if project_data['status'] == 'unstable':


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-04-04 20:30 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2021-04-04 20:30 UTC (permalink / raw
  To: gentoo-commits

commit:     915abba59f59767d341bb190de5bba5ae03097d0
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Apr  4 20:26:01 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Apr  4 20:26:01 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=915abba5

Add build log sammary

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 +-
 buildbot_gentoo_ci/steps/logs.py           | 35 +++++++++++++++++++++++++-----
 2 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index d8db8f8..6e8baf6 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -177,7 +177,7 @@ def parse_build_log():
     # set it SUCCESS/FAILURE/WARNINGS
     f.addStep(logs.MakeIssue())
     # add sum log to buildbot log
-    #f.addStep(logs.setBuildbotLog)
+    f.addStep(logs.setBuildbotLog())
     # set BuildStatus
     f.addStep(logs.setBuildStatus())
     # setup things for the irc bot

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 5fe248f..767fdd4 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -175,7 +175,7 @@ class ParserBuildLog(BuildStep):
         print(self.summery_dict)
         # remove all lines with ignore in the dict
         # setProperty summery_dict
-        self.setProperty("summery_log_dict", self.summery_dict, 'summery_log_dict')
+        self.setProperty("summary_log_dict", self.summery_dict, 'summary_log_dict')
         return SUCCESS
 
 class MakeIssue(BuildStep):
@@ -193,27 +193,50 @@ class MakeIssue(BuildStep):
     #@defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        summery_log_dict = self.getProperty('summery_log_dict')
+        summary_log_dict = self.getProperty('summary_log_dict')
         error = False
         warning = False
-        self.summery_log_list = []
+        self.summary_log_list = []
         log_hash = hashlib.sha256()
-        for k, v in sorted(summery_log_dict.items()):
+        for k, v in sorted(summary_log_dict.items()):
             if v['status'] == 'error':
                 error = True
             if v['status'] == 'warning':
                 warning = True
-            self.summery_log_list.append(v['text'])
+            self.summary_log_list.append(v['text'])
             log_hash.update(v['text'].encode('utf-8'))
         # add build log
         # add issue/bug/pr report
-        self.setProperty("summery_log_list", self.summery_log_list, 'summery_log_list')
+        self.setProperty("summary_log_list", self.summary_log_list, 'summary_log_list')
         if error:
             self.setProperty("status", 'failed', 'status')
         if warning:
             self.setProperty("status", 'warning', 'status')
         return SUCCESS
 
+class setBuildbotLog(BuildStep):
+
+    name = 'setBuildbotLog'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        #setup the log
+        log = yield self.addLog('summary')
+        # add line for line
+        for line in self.getProperty('summary_log_list'):
+            yield log.addStdout(line + '\n')
+        # add emerge info log
+        return SUCCESS
+
 class setBuildStatus(BuildStep):
 
     name = 'setBuildStatus'


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-04-04 20:30 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2021-04-04 20:30 UTC (permalink / raw
  To: gentoo-commits

commit:     81e7f0bdd56bf11fdbc0990191273ad59c1a3b01
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Apr  4 20:29:38 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Apr  4 20:29:38 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=81e7f0bd

Add support for Change USE, one use only

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 +-
 buildbot_gentoo_ci/steps/builders.py       | 58 ++++++++++++++++++++++++++----
 buildbot_gentoo_ci/steps/portage.py        |  1 +
 3 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 6e8baf6..fddfa2b 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -120,7 +120,7 @@ def run_build_request():
     # setup env
     f.addStep(portage.SetEnvDefault())
     # setup package.*
-    #f.addStep(portage.SetPackageDefault())
+    f.addStep(portage.SetPackageDefault())
     # setup files in /etc if needed
     # run --regen if needed on repo
     # update packages before any tests

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index b28d7e5..96572c0 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -19,6 +19,7 @@ def PersOutputOfEmerge(rc, stdout, stderr):
     emerge_output['rc'] = rc
     emerge_output['preserved_libs'] = False
     emerge_output['depclean'] = False
+    emerge_output['change_use'] = False
     package_dict = {}
     log_path_list = []
     print(stderr)
@@ -82,7 +83,20 @@ def PersOutputOfEmerge(rc, stdout, stderr):
     # split the lines
     #FIXME: Handling of stderr output
     for line in stderr.split('\n'):
-        pass
+        if 'Change USE:' in line:
+            line_list = line.split(' ')
+            change_use_list = []
+            # get cpv
+            cpv_split = line_list[1].split(':')
+            change_use_list.append(cpv_split[0])
+            # add use flags
+            if line_list[4].startswith('+') or line_list[4].startswith('-'):
+                # we only support one for now
+                if line_list[4].endswith(')'):
+                    change_use_list.append(line_list[4].replace(')', ''))
+                else:
+                    change_use_list = False
+            emerge_output['change_use'] = change_use_list
     return {
         'emerge_output' : emerge_output
         }
@@ -209,6 +223,7 @@ class SetupPropertys(BuildStep):
         self.setProperty('cpv_build', False, 'cpv_build')
         self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
         self.setProperty('faild_version_data', None, 'faild_version_data')
+        self.setProperty('rerun', 0, 'rerun')
         print(self.getProperty("buildnumber"))
         if self.getProperty('project_build_data') is None:
             project_build_data = {}
@@ -268,13 +283,13 @@ class RunEmerge(BuildStep):
     name = 'RunEmerge'
     description = 'Running'
     descriptionDone = 'Ran'
-    descriptionSuffix = None
     haltOnFailure = True
     flunkOnFailure = True
 
     def __init__(self, step=None,**kwargs):
         self.step = step
         super().__init__(**kwargs)
+        self.descriptionSuffix = self.step
 
     @defer.inlineCallbacks
     def run(self):
@@ -388,7 +403,6 @@ class RunEmerge(BuildStep):
             cpv = self.getProperty("cpv")
             c = yield catpkgsplit(cpv)[0]
             p = yield catpkgsplit(cpv)[1]
-            shell_commad_list.append('-p')
             shell_commad_list.append('=' + self.getProperty('cpv'))
             # we don't use the bin for the requsted cpv
             shell_commad_list.append('--usepkg-exclude')
@@ -398,6 +412,7 @@ class RunEmerge(BuildStep):
             shell_commad_list.append('virtual')
             shell_commad_list.append('--buildpkg-exclude')
             shell_commad_list.append('acct-*')
+            shell_commad_list.append('-p')
             aftersteps_list.append(
                 steps.SetPropertyFromCommandNewStyle(
                         command=shell_commad_list,
@@ -444,13 +459,13 @@ class CheckEmergeLogs(BuildStep):
     name = 'CheckLogs'
     description = 'Running'
     descriptionDone = 'Ran'
-    descriptionSuffix = None
     haltOnFailure = True
     flunkOnFailure = True
 
     def __init__(self, step=None,**kwargs):
         self.step = step
         super().__init__(**kwargs)
+        self.descriptionSuffix = self.step
 
     @defer.inlineCallbacks
     def run(self):
@@ -480,10 +495,41 @@ class CheckEmergeLogs(BuildStep):
             print(self.getProperty('cpv_build'))
 
         #FIXME:
-        # update package.* if needed and rerun pre-build max X times
+        # update package.* if needed and rerun pre-build max 3 times
         if self.step == 'pre-build':
             print(emerge_output)
-
+            if self.getProperty('rerun') <= 3:
+                # when we need to change use. we could rerun pre-build with
+                # --autounmask-use=y --autounmask-write=y --autounmask-only=y
+                # but we use --binpkg--respect-use=y in EMERGE_DEFAULT_OPTS
+                # read man emerge for more
+                if emerge_output['change_use']:
+                    # add a zz file for autounmask use
+                    separator = '\n'
+                    separator2 = ' '
+                    change_use_list = []
+                    cpv = emerge_output['change_use'][0]
+                    c = yield catpkgsplit(cpv)[0]
+                    p = yield catpkgsplit(cpv)[1]
+                    change_use_list.append(c + '/' + p)
+                    # we only support one use
+                    use_flag = emerge_output['change_use'][1]
+                    if use_flag.startswith('+'):
+                        change_use_list.append(use_flag.replace('+', ''))
+                    else:
+                        change_use_list.append(use_flag)
+                    change_use_string = separator2.join(change_use_list)
+                    aftersteps_list.append(
+                        steps.StringDownload(change_use_string + separator,
+                                workerdest='zz_autouse' + str(self.getProperty('rerun')),
+                                workdir='/etc/portage/package.use/')
+                        )
+                    # rerun
+                    aftersteps_list.append(RunEmerge(step='pre-build'))
+                    self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
+            else:
+                # trigger parse_build_log with info about pre-build and it fail
+                pass
         #FIXME:
         # Look for FAILURE and logname and download needed logfile and
         # trigger a logparser

diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index d66c952..7822af1 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -286,6 +286,7 @@ class SetPackageDefault(BuildStep):
         aftersteps_list = []
         packagedir_list = []
         packagedir_list.append('env')
+        packagedir_list.append('use')
         #FIXME:
         # get list what dir we need to make from db
         # create the dirs


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-05-02 15:24 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2021-05-02 15:24 UTC (permalink / raw
  To: gentoo-commits

commit:     c5da0d10e98b62390d47b5b13d66a135706177ed
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun May  2 15:24:05 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun May  2 15:24:05 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c5da0d10

Add support for emerge --info

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 ++
 buildbot_gentoo_ci/steps/builders.py       | 44 +++++++++++++++++++++++++++++-
 buildbot_gentoo_ci/steps/logs.py           | 24 +++++++++++++++-
 3 files changed, 68 insertions(+), 2 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index e08127b..417e385 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -177,6 +177,8 @@ def parse_build_log():
     f.addStep(logs.MakeIssue())
     # add sum log to buildbot log
     f.addStep(logs.setBuildbotLog())
+    # add emerge info to log and db
+    f.addStep(logs.setEmergeInfoLog())
     # set BuildStatus
     f.addStep(logs.setBuildStatus())
     # setup things for the irc bot

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 50f924d..3a5de47 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -164,6 +164,17 @@ def PersOutputOfDepclean(rc, stdout, stderr):
         'depclean_output' : depclean_output
         }
 
+def PersOutputOfEmergeInfo(rc, stdout, stderr):
+    emerge_info_output = {}
+    emerge_info_output['rc'] = rc
+    emerge_info_list = []
+    for line in stdout.split('\n'):
+        emerge_info_list.append(line)
+    emerge_info_output['emerge_info'] = emerge_info_list
+    return {
+        'emerge_info_output' : emerge_info_output
+        }
+
 class TriggerRunBuildRequest(BuildStep):
     
     name = 'TriggerRunBuildRequest'
@@ -650,7 +661,8 @@ class CheckEmergeLogs(BuildStep):
                             'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
                             'repository_data' : self.getProperty('repository_data'),
                             'faild_cpv' : faild_cpv,
-                            'step' : self.step
+                            'step' : self.step,
+                            'emerge_info' : self.getProperty('emerge_info_output')['emerge_info']
                         }
                     ))
         if not self.step is None and self.aftersteps_list != []:
@@ -761,6 +773,35 @@ class CheckPkgCheckLogs(BuildStep):
         self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
         return SUCCESS
 
+class RunEmergeInfo(BuildStep):
+
+    name = 'RunEmergeInfo'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        aftersteps_list = []
+        shell_commad_list = [
+                    'emerge',
+                    ]
+        shell_commad_list.append('--info')
+        aftersteps_list.append(
+                steps.SetPropertyFromCommandNewStyle(
+                        command=shell_commad_list,
+                        strip=True,
+                        extract_fn=PersOutputOfEmergeInfo,
+                        workdir='/',
+                        timeout=None
+                ))
+        yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+        return SUCCESS
+
 class RunBuild(BuildStep):
 
     name = 'RunBuild'
@@ -780,6 +821,7 @@ class RunBuild(BuildStep):
             # trigger pars_build_log if we have any logs to check
             return SUCCESS
         aftersteps_list = []
+        aftersteps_list.append(RunEmergeInfo())
         aftersteps_list.append(RunEmerge(step='pre-build'))
         aftersteps_list.append(RunEmerge(step='build'))
         self.setProperty('depclean', False, 'depclean')

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 0960b9b..3c0ef8d 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -252,7 +252,29 @@ class setBuildbotLog(BuildStep):
         # add line for line
         for line in self.getProperty('summary_log_list'):
             yield log.addStdout(line + '\n')
-        # add emerge info log
+        return SUCCESS
+
+class setEmergeInfoLog(BuildStep):
+
+    name = 'setEmergeInfoLog'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        #setup the log
+        log = yield self.addLog('emerge_info')
+        #FIXME: add emerge info to db
+        # add line for line
+        for line in self.getProperty('emerge_info'):
+            yield log.addStdout(line + '\n')
         return SUCCESS
 
 class setBuildStatus(BuildStep):


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-10-07  9:17 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2021-10-07  9:17 UTC (permalink / raw
  To: gentoo-commits

commit:     66b8e3dc0f0d56d3a154353b4363efa65850e4d1
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Oct  7 09:18:11 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Oct  7 09:18:11 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=66b8e3dc

Fix support for bb 3.3.0

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 +-
 buildbot_gentoo_ci/steps/builders.py       | 22 +++++++++++-----------
 buildbot_gentoo_ci/steps/portage.py        |  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 41c3171..45f8f48 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -101,7 +101,7 @@ def run_build_request():
     f.addStep(builders.SetupPropertys())
     # Clean and add new /etc/portage
     #NOTE: remove the symlink befor the dir
-    f.addStep(buildbot_steps.ShellCommandNewStyle(
+    f.addStep(buildbot_steps.ShellCommand(
                         command=['rm', 'make.profile'],
                         workdir='/etc/portage/'
                         ))

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 80fe52f..79be77f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -375,7 +375,7 @@ class RunEmerge(BuildStep):
             shell_commad_list.append('--buildpkg-exclude')
             shell_commad_list.append('acct-*')
             aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
@@ -394,7 +394,7 @@ class RunEmerge(BuildStep):
             shell_commad_list.append('--buildpkg-exclude')
             shell_commad_list.append('acct-*')
             aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
@@ -409,7 +409,7 @@ class RunEmerge(BuildStep):
             shell_commad_list.append('-q')
             shell_commad_list.append('@preserved-rebuild')
             aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
@@ -423,7 +423,7 @@ class RunEmerge(BuildStep):
             shell_commad_list.append('--pretend')
             shell_commad_list.append('--depclean')
             aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfDepclean,
@@ -439,7 +439,7 @@ class RunEmerge(BuildStep):
             if self.getProperty('depclean'):
                 pass
             aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfDepclean,
@@ -468,7 +468,7 @@ class RunEmerge(BuildStep):
             shell_commad_list.append('--usepkg=n')
             shell_commad_list.append(c + '/' + p)
             aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
@@ -492,7 +492,7 @@ class RunEmerge(BuildStep):
             shell_commad_list.append('acct-*')
             shell_commad_list.append('-p')
             aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
@@ -517,7 +517,7 @@ class RunEmerge(BuildStep):
             shell_commad_list.append('--buildpkg-exclude')
             shell_commad_list.append('acct-*')
             aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
@@ -663,7 +663,7 @@ class CheckEmergeLogs(BuildStep):
                         shell_commad_list.append('--buildpkg-exclude')
                         shell_commad_list.append('acct-*')
                         self.aftersteps_list.append(
-                            steps.SetPropertyFromCommandNewStyle(
+                            steps.SetPropertyFromCommand(
                                 command=shell_commad_list,
                                 strip=True,
                                 extract_fn=PersOutputOfEmerge,
@@ -830,7 +830,7 @@ class RunPkgCheck(BuildStep):
         else:
             pkgcheck_workdir = yield os.path.join(repository_path, c, p, '')
         aftersteps_list.append(
-            steps.SetPropertyFromCommandNewStyle(
+            steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfPkgCheck,
@@ -891,7 +891,7 @@ class RunEmergeInfo(BuildStep):
                     ]
         shell_commad_list.append('--info')
         aftersteps_list.append(
-                steps.SetPropertyFromCommandNewStyle(
+                steps.SetPropertyFromCommand(
                         command=shell_commad_list,
                         strip=True,
                         extract_fn=PersOutputOfEmergeInfo,

diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 94323ba..dcbf599 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -105,7 +105,7 @@ class SetMakeProfile(BuildStep):
         shell_commad_list.append(makeprofile_path)
         shell_commad_list.append('/etc/portage/make.profile')
         yield self.build.addStepsAfterCurrentStep([
-            steps.ShellCommandNewStyle(
+            steps.ShellCommand(
                         command=shell_commad_list,
                         workdir='/'
                 )


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-12-24  0:33 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2021-12-24  0:33 UTC (permalink / raw
  To: gentoo-commits

commit:     b3edf64aaff42659b3593dfe24a7e0a68e1881a0
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 24 00:33:19 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Dec 24 00:33:19 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b3edf64a

put emerge and package info to a file insted

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  4 ++
 buildbot_gentoo_ci/steps/builders.py       | 74 +++++++++++++++++++++++------
 buildbot_gentoo_ci/steps/logs.py           | 75 +++++++++++++++++++++++++++++-
 3 files changed, 137 insertions(+), 16 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 27f0076..6203a1f 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -172,8 +172,12 @@ def parse_build_log():
     f.addStep(logs.MakeIssue())
     # add sum log to buildbot log
     f.addStep(logs.setBuildbotLog())
+    # pers the emerge info
+    f.addStep(logs.SetupParserEmergeInfoLog())
     # add emerge info to log and db
     f.addStep(logs.setEmergeInfoLog())
+    # add package info to log and db
+    f.addStep(logs.setPackageInfoLog())
     # set BuildStatus
     f.addStep(logs.setBuildStatus())
     # setup things for the irc bot

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 7247abd..e76fe3d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -590,14 +590,35 @@ class CheckEmergeLogs(BuildStep):
             workdir=workdir
         ))
 
-    @defer.inlineCallbacks
-    def getLogFile(self, cpv, log_dict):
-        masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")) ,log_dict[cpv]['full_logname'])
+    def addFileUploade(self, sourcefile, destfile):
         self.aftersteps_list.append(steps.FileUpload(
-            workersrc=log_dict[cpv]['log_path'],
-            masterdest=masterdest
+            workersrc=sourcefile,
+            masterdest=destfile
         ))
 
+    @defer.inlineCallbacks
+    def getLogFile(self, cpv, log_dict):
+        destfile = yield os.path.join(self.masterdest, log_dict[cpv]['full_logname'])
+        sourcefile = log_dict[cpv]['log_path']
+        self.addFileUploade(sourcefile, destfile)
+
+    @defer.inlineCallbacks
+    def getEmergeFiles(self, faild_version_data):
+        # get emerge info
+        destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
+        sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
+        self.addFileUploade(sourcefile, destfile)
+        #FIXME:
+        # if faild_version_data:
+        # get emerge.log
+        # get elogs
+        # world file
+
+    def getBuildWorkdirFiles(self):
+        #FIXME:
+        # get files from the build workdir
+        pass
+
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
@@ -610,6 +631,8 @@ class CheckEmergeLogs(BuildStep):
                     ]
         package_dict = emerge_output['packages']
 
+        self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
+
         #FIXME: Prosees the logs and do stuff
         # preserved-libs
         if emerge_output['preserved_libs'] and projects_emerge_options['preserved_libs']:
@@ -757,12 +780,12 @@ class CheckEmergeLogs(BuildStep):
                 print(log_dict)
                 cpv = self.getProperty('cpv')
                 faild_cpv = emerge_output['failed']
+                faild_version_data = False
                 if cpv in log_dict or faild_cpv in log_dict:
+                    yield self.createDistDir()
                     if cpv in log_dict:
                         self.log_data[cpv] = log_dict[cpv]
-                        yield self.createDistDir()
                         yield self.getLogFile(cpv, log_dict)
-                        faild_version_data = False
                     if faild_cpv:
                         # failed and build requested cpv
                         if cpv == faild_cpv:
@@ -772,6 +795,8 @@ class CheckEmergeLogs(BuildStep):
                             self.log_data[faild_cpv] = log_dict[faild_cpv]
                             yield self.getLogFile(faild_cpv, log_dict)
                             faild_version_data = yield self.getVersionData(faild_cpv)
+                        self.getBuildWorkdirFiles()
+                    self.getEmergeFiles(faild_version_data)
                     self.aftersteps_list.append(steps.Trigger(
                         schedulerNames=['parse_build_log'],
                         waitForFinish=False,
@@ -785,7 +810,6 @@ class CheckEmergeLogs(BuildStep):
                             'repository_data' : self.getProperty('repository_data'),
                             'faild_cpv' : faild_cpv,
                             'step' : self.step,
-                            'emerge_info' : self.getProperty('emerge_info_output')['emerge_info'],
                             'build_workername' : self.getProperty('workername')
                         }
                     ))
@@ -920,17 +944,37 @@ class RunEmergeInfo(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         aftersteps_list = []
+        # add emerge info
         shell_commad_list = [
                     'emerge',
                     ]
         shell_commad_list.append('--info')
+        shell_commad_list.append('>')
+        shell_commad_list.append('/tmp/emerge_info.txt')
         aftersteps_list.append(
-                steps.SetPropertyFromCommand(
-                        command=shell_commad_list,
-                        strip=True,
-                        extract_fn=PersOutputOfEmergeInfo,
-                        workdir='/',
-                        timeout=None
+                steps.ShellCommand(
+                        # the list need to be joined to pipe to a file
+                        command=' '.join(shell_commad_list),
+                        workdir='/'
+                ))
+        # add package info
+        cpv = self.getProperty("cpv")
+        c = yield catpkgsplit(cpv)[0]
+        p = yield catpkgsplit(cpv)[1]
+        shell_commad_list = [
+                    'emerge',
+                    ]
+        shell_commad_list.append('-qpvO')
+        shell_commad_list.append('=' + self.getProperty('cpv'))
+        shell_commad_list.append('--usepkg-exclude')
+        shell_commad_list.append(c + '/' + p)
+        shell_commad_list.append('>>')
+        shell_commad_list.append('/tmp/emerge_info.txt')
+        aftersteps_list.append(
+                steps.ShellCommand(
+                        # the list need to be joined to pipe to a file
+                        command=' '.join(shell_commad_list),
+                        workdir='/'
                 ))
         yield self.build.addStepsAfterCurrentStep(aftersteps_list)
         return SUCCESS
@@ -954,8 +998,8 @@ class RunBuild(BuildStep):
             # trigger pars_build_log if we have any logs to check
             return SKIPPED
         aftersteps_list = []
-        aftersteps_list.append(RunEmergeInfo())
         aftersteps_list.append(RunEmerge(step='pre-build'))
+        aftersteps_list.append(RunEmergeInfo())
         aftersteps_list.append(RunEmerge(step='build'))
         aftersteps_list.append(RunEmerge(step='pre-depclean'))
         aftersteps_list.append(RunEmerge(step='preserved-libs'))

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index dc5d3a1..d6ee705 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -45,6 +45,23 @@ def PersOutputOfLogParser(rc, stdout, stderr):
         'build_summery_output' : build_summery_output
         }
 
+def PersOutputOfEmergeInfo(rc, stdout, stderr):
+    #FIXME: line for package info
+    emerge_info_output = {}
+    emerge_info_output['rc'] = rc
+    emerge_info_list = []
+    emerge_package_info = []
+    for line in stdout.split('\n'):
+        if line.startswith('['):
+            emerge_package_info.append(line)
+        else:
+            emerge_info_list.append(line)
+    emerge_info_output['emerge_info'] = emerge_info_list
+    emerge_info_output['emerge_package_info'] = emerge_package_info
+    return {
+        'emerge_info_output' : emerge_info_output
+        }
+
 class SetupPropertys(BuildStep):
     
     name = 'SetupPropertys'
@@ -387,6 +404,38 @@ class setBuildbotLog(BuildStep):
             yield log.addStdout(line + '\n')
         return SUCCESS
 
+class SetupParserEmergeInfoLog(BuildStep):
+
+    name = 'SetupParserEmergeInfoLog'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
+        command = []
+        command.append('cat')
+        command.append('emerge_info.txt')
+        self.aftersteps_list = []
+        self.aftersteps_list.append(master_steps.MasterSetPropertyFromCommand(
+                                                            name = 'RunEmergeInfoLogParser',
+                                                            haltOnFailure = True,
+                                                            flunkOnFailure = True,
+                                                            command=command,
+                                                            workdir=workdir,
+                                                            strip=False,
+                                                            extract_fn=PersOutputOfEmergeInfo
+                                                            ))
+        yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+        return SUCCESS
+
 class setEmergeInfoLog(BuildStep):
 
     name = 'setEmergeInfoLog'
@@ -406,10 +455,34 @@ class setEmergeInfoLog(BuildStep):
         log = yield self.addLog('emerge_info')
         #FIXME: add emerge info to db
         # add line for line
-        for line in self.getProperty('emerge_info'):
+        for line in self.getProperty('emerge_info_output')['emerge_info']:
             yield log.addStdout(line + '\n')
         return SUCCESS
 
+class setPackageInfoLog(BuildStep):
+
+    name = 'setPackageInfoLog'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        #setup the log
+        log = yield self.addLog('package_info')
+        #FIXME: add package info to db
+        # add line for line
+        for line in self.getProperty('emerge_info_output')['emerge_package_info']:
+            yield log.addStdout(line + '\n')
+        return SUCCESS
+
+
 class Upload(BuildStep):
 
     name = 'Upload'


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2022-05-28 14:55 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2022-05-28 14:55 UTC (permalink / raw
  To: gentoo-commits

commit:     1f57c498694cbdce80ad43b2b5a3c0c44cdee588
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat May 28 14:54:50 2022 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat May 28 14:54:50 2022 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1f57c498

Add support for docker latent workers

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/workers.py | 39 ++++++++++++++++++++++++++++++++++--
 buildbot_gentoo_ci/steps/builders.py | 15 ++++++++------
 2 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py
index fbdc2a9..d4c20f7 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -1,7 +1,7 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2022 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-from buildbot.plugins import worker
+from buildbot.plugins import worker, util
 
 class gentoo_ci_workers():
     def __init__(self, worker_data, **kwargs):
@@ -47,17 +47,52 @@ class gentoo_ci_workers():
         print(node_worker)
         return node_worker
 
+@util.renderer
+def docker_images(props):
+    return 'bb-worker-' + props.getProperty('project_uuid') + ':latest'
+
+@util.renderer
+def docker_volumes(props):
+    volumes_list = []
+    #FIXME: set in master.cfg
+    src_dir = '/srv/gentoo/portage/' + props.getProperty('project_uuid')
+    dest_dir = '/var/cache/portage'
+    #add distdir
+    volumes_list.append(src_dir + '/distfiles' + ':' + dest_dir + '/distfiles')
+    #add bindir
+    volumes_list.append(src_dir + '/packages' + ':' + dest_dir + '/packages')
+    return volumes_list
+
 def gentoo_workers(worker_data):
     w = []
     g_ci_w = gentoo_ci_workers(worker_data)
     LocalWorkers = g_ci_w.getLocalWorkersUuid()
     BuildWorkers = g_ci_w.getBuildWorkersAllData()
     NodeWorkers = g_ci_w.getNodedWorkersAllData()
+    docker_hostconfig = {}
+    # For use of sandbox stuff
+    # FEATURES="ipc-sandbox network-sandbox pid-sandbox"
+    docker_hostconfig['cap_add'] = ['SYS_ADMIN', 'NET_ADMIN', 'SYS_PTRACE']
+    # libseccomp overhead
+    # https://github.com/seccomp/libseccomp/issues/153
+    docker_hostconfig['security_opt'] = ['seccomp=unconfined']
     for local_worker in LocalWorkers:
         w.append(worker.LocalWorker(local_worker))
     for build_worker in BuildWorkers:
         if build_worker['type'] == 'default':
             w.append(worker.Worker(build_worker['uuid'], build_worker['password']))
+        #FIXME: set settings in master.cfg
+        if build_worker['type'] == 'docker':
+            w.append(worker.DockerLatentWorker(build_worker['uuid'],
+                            build_worker['password'],
+                            docker_host='tcp://192.168.1.3:2375',
+                            image=docker_images,
+                            volumes=docker_volumes,
+                            hostconfig=docker_hostconfig,
+                            followStartupLogs=True,
+                            masterFQDN='192.168.1.5',
+                            build_wait_timeout=3600
+                            ))
     for node_worker in NodeWorkers:
         if node_worker['type'] == 'node':
             w.append(worker.Worker(node_worker['uuid'], node_worker['password']))

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 8ce6545..1d4f52d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -236,7 +236,8 @@ class TriggerRunBuildRequest(BuildStep):
                             'projectrepository_data' : self.projectrepository_data,
                             'use_data' : self.use_data,
                             'fullcheck' : self.getProperty("fullcheck"),
-                            'project_build_data' : project_build_data
+                            'project_build_data' : project_build_data,
+                            'project_uuid' : self.project_data['uuid']
                         }
                 )])
         return SUCCESS
@@ -384,6 +385,8 @@ class RunEmerge(BuildStep):
         self.descriptionSuffix = self.step
         self.name = 'Setup emerge for ' + self.step + ' step'
         self.build_env = {}
+        #FIXME: Set build timeout in config
+        self.build_timeout = 1800
 
     @defer.inlineCallbacks
     def run(self):
@@ -454,7 +457,7 @@ class RunEmerge(BuildStep):
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
                         workdir='/',
-                        timeout=None
+                        timeout=self.build_timeout
                 ))
             aftersteps_list.append(CheckEmergeLogs('update'))
             if projects_emerge_options['preserved_libs']:
@@ -470,7 +473,7 @@ class RunEmerge(BuildStep):
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
                         workdir='/',
-                        timeout=None
+                        timeout=self.build_timeout
                 ))
             aftersteps_list.append(CheckEmergeLogs('preserved-libs'))
             self.setProperty('preserved_libs', False, 'preserved-libs')
@@ -532,7 +535,7 @@ class RunEmerge(BuildStep):
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
                         workdir='/',
-                        timeout=None
+                        timeout=self.build_timeout
                 ))
             aftersteps_list.append(CheckEmergeLogs('match'))
 
@@ -561,7 +564,7 @@ class RunEmerge(BuildStep):
                         strip=True,
                         extract_fn=PersOutputOfEmerge,
                         workdir='/',
-                        timeout=None
+                        timeout=self.build_timeout
                 ))
             aftersteps_list.append(CheckEmergeLogs('pre-build'))
 
@@ -588,7 +591,7 @@ class RunEmerge(BuildStep):
                         extract_fn=PersOutputOfEmerge,
                         workdir='/',
                         env=self.build_env,
-                        timeout=None
+                        timeout=self.build_timeout
                 ))
             aftersteps_list.append(CheckEmergeLogs('build'))
             if projects_emerge_options['preserved_libs']:


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2022-09-01 14:57 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2022-09-01 14:57 UTC (permalink / raw
  To: gentoo-commits

commit:     0d19b4627eba83ffccadc753105b5cbdf53a68e7
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Sep  1 14:56:42 2022 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Sep  1 14:56:42 2022 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0d19b462

Add Bug ID for Irc reporter and nice titel

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/reporters.py |  7 ++++---
 buildbot_gentoo_ci/steps/bugs.py       | 24 ++++++++++++++++--------
 buildbot_gentoo_ci/steps/logs.py       | 16 ++++++++++++++++
 3 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/buildbot_gentoo_ci/config/reporters.py b/buildbot_gentoo_ci/config/reporters.py
index 9832305..bd55037 100644
--- a/buildbot_gentoo_ci/config/reporters.py
+++ b/buildbot_gentoo_ci/config/reporters.py
@@ -8,10 +8,11 @@ from buildbot.reporters.message import MessageFormatter
 
 from buildbot_gentoo_ci.reporters import irc
 irc_template = '''{% set resultsList = ["\x0303SUCCESS", "\x0308WARNINGS", "\x0304FAILURE"] %}\
-{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303" }}repo/{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
+{{ "\x02" }}{{ build['properties']['cpv'][0] }}{{ "\x02" }} {{ "\x0303" }}Repo:{{ projects }}:{{ build['properties']['branch'][0] }}{{ "\x03" }} \
 {{ build['properties']['revision'][0]|truncate(10, True) }} {{ "\x0302" }}{{ build['properties']['owners'][0][0] }}{{ "\x03" }} \
-{{ build['properties']['event'][0] }} {{ projects }}:{{ build['properties']['project_data'][0]['name'] }} \
-{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{ "\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }}\
+{{ "\x0306" }}{{ build['properties']['event'][0] }}{{ "\x03" }} {{ projects }}:{{ build['properties']['project_data'][0]['name'] }} \
+{{ "\x02" }}{{ "Build: "}}{{ resultsList[build['results']] }}{{ "\x03" }}{{ "\x02" }} {{ "\x0312" }}{{ build_url }}{{ "\x03" }} \
+{% if build['properties']['bgo'][0]['match'] is true %}{{ "\x0311" }}Bugid: {{build['properties']['bgo'][0]['id']}}{{ "\x03" }}{% endif %}\
 '''
 
 def ircGenerators():

diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 27b1388..801fc98 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -87,20 +87,28 @@ class GetBugs(BuildStep):
         yield log.addStdout('Open Bugs\n')
         match = False
         for bug in buglist:
-            yield log.addStdout('Bug: ' + str(bug['id']) + ' Summary: ' + bug['summary'] +'\n')
-            if re.search(self.getProperty('error_dict')['title_issue'][:20], bug['summary']):
-                print('Bug found')
-                print(bug)
+            yield log.addStdout(f"Bug: {str(bug['id'])} Summary: {bug['summary']}\n")
+            # we splite the lines to lists and try to match the words
+            matches = 0
+            match_search_text = list(self.getProperty('error_dict')['title_issue'].split())
+            match_bug_text = list(bug['summary'].split())
+            #FIXME: add check for cp
+            for match_word in match_search_text:
+                if match_word in match_bug_text:
+                    matches = matches + 1
+            if matches >= 10:
+                print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}")
                 match = {}
+                match['match'] = True
                 match['id'] = bug['id']
                 match['summary'] = bug['summary']
+        yield log.addStdout(f"Line to match: {self.getProperty('error_dict')['title_issue']}\n")
         if match:
-            yield log.addStdout('Match bug found\n')
-            yield log.addStdout('Bug: ' + str(match['id']) + ' Summary: ' + match['summary'] +'\n')
+            yield log.addStdout('Match bug: YES\n')
+            yield log.addStdout(f"Bug: {str(match['id'])} Summary: {match['summary']}\n")
             self.setProperty("bgo", match, 'bgo')
             return
-        yield log.addStdout('NO Match bug found\n')
-        self.setProperty("bgo", False, 'bgo')
+        yield log.addStdout('Match bug: NO\n')
 
     @defer.inlineCallbacks
     def run(self):

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 164bb24..57f9394 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -91,6 +91,7 @@ class SetupPropertys(BuildStep):
         else:
             log_cpv = self.getProperty('cpv')
         self.setProperty("log_cpv", log_cpv, 'log_cpv')
+        self.setProperty("bgo", dict( match=False), 'bgo')
         self.descriptionDone = 'Runing log checker on ' + log_cpv
         return SUCCESS
 
@@ -333,6 +334,19 @@ class MakeIssue(BuildStep):
         yield log.addStdout(log_cpv['full_logname'] + '\n')
         yield log.addStdout('world.log' + '\n')
 
+    def getNiceErrorLine(self, line):
+        # strip away hex addresses, loong path names, line and time numbers and other stuff
+        # https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L467
+        # FIXME: Add the needed line when needed
+        if re.search(': line', line):
+            line = re.sub(r"\d", "<snip>", line)
+        # Shorten the path
+        if line.startswith('/usr/'):
+            line = line.replace(os.path.split(line.split(' ', 1)[0])[0], '/...')
+        if re.search(': \d:\d: ', line):
+            line = re.sub(r":\d:\d: ", ": ", line)
+        return line
+
     def ClassifyIssue(self):
         # get the title for the issue
         text_issue_list = []
@@ -344,8 +358,10 @@ class MakeIssue(BuildStep):
         # add the issue error
         if text_issue_list != []:
             self.error_dict['title_issue'] = text_issue_list[0].replace('*', '').strip()
+            self.error_dict['title_issue_nice'] = self.getNiceErrorLine(text_issue_list[0].replace('*', '').strip())
         else:
             self.error_dict['title_issue'] = 'title_issue : None'
+            self.error_dict['title_nice'] = 'title_issue : None'
         self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
         #set the error title
         self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' + self.error_dict['title_issue']


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2022-11-09 21:05 Magnus Granberg
  0 siblings, 0 replies; 10+ messages in thread
From: Magnus Granberg @ 2022-11-09 21:05 UTC (permalink / raw
  To: gentoo-commits

commit:     ea8cb4a47f01964a09a7bd6f863d871a01c8b76d
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Nov  9 21:05:15 2022 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Nov  9 21:05:15 2022 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ea8cb4a4

Rename setBuildbotLog to Summarylog

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 2 +-
 buildbot_gentoo_ci/steps/logs.py           | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 7f2dff9..d07a143 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -180,7 +180,7 @@ def parse_build_log():
     # set it SUCCESS/FAILURE/WARNINGS
     f.addStep(logs.MakeIssue())
     # add sum log to buildbot log
-    f.addStep(logs.setBuildbotLog())
+    f.addStep(logs.Summarylog())
     # pers the emerge info
     f.addStep(logs.ReadEmergeInfoLog())
     # add emerge info to log and db

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 67845e7..2a52308 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -404,9 +404,9 @@ class MakeIssue(BuildStep):
             yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
         return SUCCESS
 
-class setBuildbotLog(BuildStep):
+class Summarylog(BuildStep):
 
-    name = 'setBuildbotLog'
+    name = 'Summarylog'
     description = 'Running'
     descriptionDone = 'Ran'
     descriptionSuffix = None


^ permalink raw reply related	[flat|nested] 10+ messages in thread

end of thread, other threads:[~2022-11-09 21:05 UTC | newest]

Thread overview: 10+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-05-02 15:24 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/ Magnus Granberg
  -- strict thread matches above, loose matches on Subject: below --
2022-11-09 21:05 Magnus Granberg
2022-09-01 14:57 Magnus Granberg
2022-05-28 14:55 Magnus Granberg
2021-12-24  0:33 Magnus Granberg
2021-10-07  9:17 Magnus Granberg
2021-04-04 20:30 Magnus Granberg
2021-04-04 20:30 Magnus Granberg
2021-03-24 23:15 Magnus Granberg
2021-03-18 23:18 Magnus Granberg

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox