* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2015-04-18 20:27 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2015-04-18 20:27 UTC (permalink / raw
To: gentoo-commits
commit: 0cdd91079514fc4e4da4a625b106d5759e867716
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 18 19:27:28 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Apr 18 19:27:28 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0cdd9107
add the portage patch
patches/portage.patch | 287 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 287 insertions(+)
diff --git a/patches/portage.patch b/patches/portage.patch
new file mode 100644
index 0000000..eefe0f3
--- /dev/null
+++ b/patches/portage.patch
@@ -0,0 +1,287 @@
+2015-04-18 Magnus Granberg <zorry@gentoo.org>
+
+ * tbc/pym/actions.py
+ Use the patched Scheduler and add build_dict so it can be ust.
+ We use or own mydepgraph (build_mydepgraph) that call backtrack_depgraph.
+ Return the output_buffer for emerge info.
+ And pass unresolvable in action_depclean so we can use it later.
+ * tbc/pym/main.py
+ Use or own patched actions.
+ We pass build_dict to some functions.
+ * tbc/pym/Scheduler.py
+ We copy Scheduler.py from portage and patch it.
+ Fix so we can use add_buildlog_main()
+ We use add_buildlog_main() for loging.
+
+--- a/tbc/pym/actions.py 2013-03-22 17:57:23.000000000 +0100
++++ b/tbc/pym/actions.py 2013-03-22 19:00:43.265582143 +0100
+@@ -72,7 +72,7 @@ from _emerge.MetadataRegen import Metada
+ from _emerge.Package import Package
+ from _emerge.ProgressHandler import ProgressHandler
+ from _emerge.RootConfig import RootConfig
+-from _emerge.Scheduler import Scheduler
++from zobcs.Scheduler import Scheduler
+ from _emerge.search import search
+ from _emerge.SetArg import SetArg
+ from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice
+@@ -83,6 +83,8 @@ from _emerge.UnmergeDepPriority import U
+ from _emerge.UseFlagDisplay import pkg_use_display
+ from _emerge.userquery import userquery
+
++from zobcs.build_depgraph import build_mydepgraph
++
+ if sys.hexversion >= 0x3000000:
+ long = int
+ _unicode = str
+@@ -90,7 +92,7 @@ else:
+ _unicode = unicode
+
+ def action_build(settings, trees, mtimedb,
+- myopts, myaction, myfiles, spinner):
++ myopts, myaction, myfiles, spinner, build_dict, session):
+
+ if '--usepkgonly' not in myopts:
+ old_tree_timestamp_warn(settings['PORTDIR'], settings)
+@@ -312,16 +314,10 @@ def action_build(settings, trees, mtimed
+ print(darkgreen("emerge: It seems we have nothing to resume..."))
+ return os.EX_OK
+
+- try:
+- success, mydepgraph, favorites = backtrack_depgraph(
+- settings, trees, myopts, myparams, myaction, myfiles, spinner)
+- except portage.exception.PackageSetNotFound as e:
+- root_config = trees[settings['EROOT']]['root_config']
+- display_missing_pkg_set(root_config, e.value)
+- return 1
++ success, settings, trees, mtimedb, mydepgraph = build_mydepgraph(settings,
++ trees, mtimedb, myopts, myparams, myaction, myfiles, spinner, build_dict, session)
+
+ if not success:
+- mydepgraph.display_problems()
+ return 1
+
+ mergecount = None
+@@ -613,7 +609,7 @@ def action_depclean(settings, trees, ldp
+ # The calculation is done in a separate function so that depgraph
+ # references go out of scope and the corresponding memory
+ # is freed before we call unmerge().
+- rval, cleanlist, ordered, req_pkg_count = \
++ rval, cleanlist, ordered, req_pkg_count, unresolvable = \
+ calc_depclean(settings, trees, ldpath_mtimes,
+ myopts, action, args_set, spinner)
+
+@@ -816,7 +812,7 @@ def calc_depclean(settings, trees, ldpat
+ resolver.display_problems()
+
+ if not success:
+- return 1, [], False, 0
++ return 1, [], False, 0, []
+
+ def unresolved_deps():
+
+@@ -827,7 +823,7 @@ def calc_depclean(settings, trees, ldpat
+ unresolvable.add((dep.atom, dep.parent.cpv))
+
+ if not unresolvable:
+- return False
++ return None
+
+ if unresolvable and not allow_missing_deps:
+
+@@ -877,11 +873,12 @@ def calc_depclean(settings, trees, ldpat
+ "dependencies then use %s." % good("--nodeps"))
+ writemsg_level("".join("%s%s\n" % (prefix, line) for line in msg),
+ level=logging.ERROR, noiselevel=-1)
+- return True
+- return False
++ return unresolvable
++ return None
+
+- if unresolved_deps():
+- return 1, [], False, 0
++ unresolvable = unresolved_deps()
++ if not unresolvable is None:
++ return 1, [], False, 0, unresolvable
+
+ graph = resolver._dynamic_config.digraph.copy()
+ required_pkgs_total = 0
+@@ -1160,7 +1157,7 @@ def calc_depclean(settings, trees, ldpat
+ priority=UnmergeDepPriority(runtime=True),
+ root=pkg.root)):
+ resolver.display_problems()
+- return 1, [], False, 0
++ return 1, [], False, 0, []
+
+ writemsg_level("\nCalculating dependencies ")
+ success = resolver._complete_graph(
+@@ -1168,9 +1165,10 @@ def calc_depclean(settings, trees, ldpat
+ writemsg_level("\b\b... done!\n")
+ resolver.display_problems()
+ if not success:
+- return 1, [], False, 0
+- if unresolved_deps():
+- return 1, [], False, 0
++ return 1, [], False, 0, []
++ unresolvable = unresolved_deps()
++ if not unresolvable is None:
++ return 1, [], False, 0, unresolvable
+
+ graph = resolver._dynamic_config.digraph.copy()
+ required_pkgs_total = 0
+@@ -1179,7 +1177,7 @@ def calc_depclean(settings, trees, ldpat
+ required_pkgs_total += 1
+ cleanlist = create_cleanlist()
+ if not cleanlist:
+- return 0, [], False, required_pkgs_total
++ return 0, [], False, required_pkgs_total, []
+ clean_set = set(cleanlist)
+
+ if clean_set:
+@@ -1289,8 +1287,8 @@ def calc_depclean(settings, trees, ldpat
+ graph.remove(node)
+ cleanlist.append(node.cpv)
+
+- return 0, cleanlist, ordered, required_pkgs_total
+- return 0, [], False, required_pkgs_total
++ return 0, cleanlist, ordered, required_pkgs_total, []
++ return 0, [], False, required_pkgs_total, []
+
+ def action_deselect(settings, trees, opts, atoms):
+ enter_invalid = '--ask-enter-invalid' in opts
+@@ -1692,11 +1692,8 @@ def action_info(settings, trees, myopts,
+ unset_vars.append(k)
+ if unset_vars:
+ append("Unset: "+", ".join(unset_vars))
+- append("")
+- append("")
+- writemsg_stdout("\n".join(output_buffer),
+- noiselevel=-1)
+- del output_buffer[:]
++
++ return False, output_buffer
+
+ # If some packages were found...
+ if mypkgs:
+@@ -3607,7 +3607,7 @@ def repo_name_duplicate_check(trees):
+
+ return bool(ignored_repos)
+
+-def run_action(emerge_config):
++def run_action(emerge_config, build_dict, session):
+
+ # skip global updates prior to sync, since it's called after sync
+ if emerge_config.action not in ('help', 'info', 'sync', 'version') and \
+@@ -4059,7 +4059,7 @@ def run_action(emerge_config):
+ retval = action_build(emerge_config.target_config.settings,
+ emerge_config.trees, emerge_config.target_config.mtimedb,
+ emerge_config.opts, emerge_config.action,
+- emerge_config.args, spinner)
++ emerge_config.args, spinner, build_dict, session)
+ post_emerge(emerge_config.action, emerge_config.opts,
+ emerge_config.args, emerge_config.target_config.root,
+ emerge_config.trees, emerge_config.target_config.mtimedb, retval)
+--- a/tbc/pym/main.py 2013-03-22 17:57:23.000000000 +0100
++++ b/tbc/pym/main.py 2012-12-06 03:32:56.104889716 +0100
+@@ -11,7 +11,7 @@ portage.proxy.lazyimport.lazyimport(glob
+ 'logging',
+ 'portage.util:writemsg_level',
+ 'textwrap',
+- '_emerge.actions:load_emerge_config,run_action,' + \
++ 'zobcs.actions:load_emerge_config,run_action,' + \
+ 'validate_ebuild_environment',
+ '_emerge.help:help@emerge_help',
+ )
+@@ -968,15 +968,20 @@ def profile_check(trees, myaction):
+ return 1
+ return os.EX_OK
+
+-def emerge_main(args=None):
++def emerge_main(args=None, build_dict=None, session=None):
+ """
+ @param args: command arguments (default: sys.argv[1:])
+ @type args: list
++ @param build_dict: info of the build_job
++ @type build_dict: dict
+ """
+ if args is None:
+ args = sys.argv[1:]
+
+ args = portage._decode_argv(args)
++
++ if build_dict is None:
++ build_dict = {}
+
+ # Disable color until we're sure that it should be enabled (after
+ # EMERGE_DEFAULT_OPTS has been parsed).
+@@ -1028,7 +1028,7 @@ def emerge_main(args=None):
+ parse_opts(tmpcmdline)
+
+ try:
+- return run_action(emerge_config)
++ return run_action(emerge_config, build_dict, session)
+ finally:
+ # Call destructors for our portdbapi instances.
+ for x in emerge_config.trees.values():
+--- a/tbc/pym/Scheduler.py 2013-03-22 17:57:23.000000000 +0100
++++ b/tbc/pym/Scheduler.py 2012-12-21 02:09:28.082301168 +0100
+@@ -62,6 +62,8 @@ from _emerge.PackageMerge import Package
+ from _emerge.PollScheduler import PollScheduler
+ from _emerge.SequentialTaskQueue import SequentialTaskQueue
+
++from zobcs.build_log import add_buildlog_main
++
+ if sys.hexversion >= 0x3000000:
+ basestring = str
+
+@@ -1254,8 +1251,9 @@ class Scheduler(PollScheduler):
+
+ def _do_merge_exit(self, merge):
+ pkg = merge.merge.pkg
++ settings = merge.merge.settings
++ trees = self.trees
+ if merge.returncode != os.EX_OK:
+- settings = merge.merge.settings
+ build_dir = settings.get("PORTAGE_BUILDDIR")
+ build_log = settings.get("PORTAGE_LOG_FILE")
+
+@@ -1266,6 +1264,7 @@ class Scheduler(PollScheduler):
+ if not self._terminated_tasks:
+ self._failed_pkg_msg(self._failed_pkgs[-1], "install", "to")
+ self._status_display.failed = len(self._failed_pkgs)
++ add_buildlog_main(settings, pkg, trees)
+ return
+
+ self._task_complete(pkg)
+@@ -1284,6 +1283,7 @@ class Scheduler(PollScheduler):
+ self._pkg_cache.pop(pkg_to_replace, None)
+
+ if pkg.installed:
++ add_buildlog_main(settings, pkg, trees)
+ return
+
+ # Call mtimedb.commit() after each merge so that
+@@ -1294,6 +1294,7 @@ class Scheduler(PollScheduler):
+ if not mtimedb["resume"]["mergelist"]:
+ del mtimedb["resume"]
+ mtimedb.commit()
++ add_buildlog_main(settings, pkg, trees)
+
+ def _build_exit(self, build):
+ self._running_tasks.pop(id(build), None)
+@@ -1318,6 +1319,8 @@ class Scheduler(PollScheduler):
+ self._status_display.merges = len(self._task_queues.merge)
+ else:
+ settings = build.settings
++ trees = self.trees
++ pkg = build.pkg
+ build_dir = settings.get("PORTAGE_BUILDDIR")
+ build_log = settings.get("PORTAGE_LOG_FILE")
+
+@@ -1329,6 +1332,7 @@ class Scheduler(PollScheduler):
+ self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for")
+ self._status_display.failed = len(self._failed_pkgs)
+ self._deallocate_config(build.settings)
++ add_buildlog_main(settings, pkg, trees)
+ self._jobs -= 1
+ self._status_display.running = self._jobs
+ self._schedule()
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2015-10-04 20:06 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2015-10-04 20:06 UTC (permalink / raw
To: gentoo-commits
commit: 583dee12c3f1363fa4593c7fc2532673546a74ce
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 4 20:06:34 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Oct 4 20:06:34 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=583dee12
fix a typo in the portage.patch
patches/portage.patch | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/patches/portage.patch b/patches/portage.patch
index fa8e6ca..ae45666 100644
--- a/patches/portage.patch
+++ b/patches/portage.patch
@@ -287,8 +287,8 @@
self._jobs -= 1
self._status_display.running = self._jobs
self._schedule()
---- a/pym/repoman/main.py 2015-09-22 02:20:41.000000000 +0200
-+++ b/pym/repoman/main.py 2015-10-04 20:21:57.586494104 +0200
+--- a/pym/tbc/repoman/main.py 2015-09-22 02:20:41.000000000 +0200
++++ b/pym//tbc/repoman/main.py 2015-10-04 20:21:57.586494104 +0200
@@ -45,8 +45,9 @@ bad = create_color_func("BAD")
os.umask(0o22)
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2015-10-04 20:10 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2015-10-04 20:10 UTC (permalink / raw
To: gentoo-commits
commit: aa9cb43df0b14b0e45fa1749b5c89ab769abd618
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 4 20:10:07 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Oct 4 20:10:07 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aa9cb43d
fix a typo in the portage.patch
patches/portage.patch | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/patches/portage.patch b/patches/portage.patch
index ae45666..1faf3c0 100644
--- a/patches/portage.patch
+++ b/patches/portage.patch
@@ -288,7 +288,7 @@
self._status_display.running = self._jobs
self._schedule()
--- a/pym/tbc/repoman/main.py 2015-09-22 02:20:41.000000000 +0200
-+++ b/pym//tbc/repoman/main.py 2015-10-04 20:21:57.586494104 +0200
++++ b/pym/tbc/repoman/main.py 2015-10-04 20:21:57.586494104 +0200
@@ -45,8 +45,9 @@ bad = create_color_func("BAD")
os.umask(0o22)
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2015-12-19 11:18 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2015-12-19 11:18 UTC (permalink / raw
To: gentoo-commits
commit: f1ff3a446ad5babea21f85eae321972cb5917044
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 19 11:18:15 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Dec 19 11:18:15 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f1ff3a44
fix typo in the portage.patch for repoman
patches/portage.patch | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/patches/portage.patch b/patches/portage.patch
index 3728501..bbf8720 100644
--- a/patches/portage.patch
+++ b/patches/portage.patch
@@ -295,7 +295,7 @@
-def repoman_main(argv):
- config_root = os.environ.get("PORTAGE_CONFIGROOT")
-+def repoman_main:(argv, config_root=None, pkgdir=None):
++def repoman_main(argv, config_root=None, pkgdir=None):
+ if config_root is None:
+ config_root = os.environ.get("PORTAGE_CONFIGROOT")
repoman_settings = portage.config(config_root=config_root, local_config=False)
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2016-01-19 17:39 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2016-01-19 17:39 UTC (permalink / raw
To: gentoo-commits
commit: 9a68fea06890542d4c82a8ddf2e22ca44cd50a2e
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 19 17:40:44 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 19 17:40:44 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9a68fea0
support portage 2.2.27
patches/portage.patch | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/patches/portage.patch b/patches/portage.patch
index bbf8720..6f2d17f 100644
--- a/patches/portage.patch
+++ b/patches/portage.patch
@@ -44,7 +44,7 @@
if '--usepkgonly' not in myopts:
old_tree_timestamp_warn(settings['PORTDIR'], settings)
-@@ -312,16 +314,10 @@ def action_build(settings, trees, mtimed
+@@ -319,20 +319,14 @@ def action_build(settings, trees, mtimed
print(darkgreen("emerge: It seems we have nothing to resume..."))
return os.EX_OK
@@ -58,6 +58,10 @@
+ success, settings, trees, mtimedb, mydepgraph = build_mydepgraph(settings,
+ trees, mtimedb, myopts, myparams, myaction, myfiles, spinner, build_dict, session)
+ if "--autounmask-only" in myopts:
+ mydepgraph.display_problems()
+ return 0
+
if not success:
- mydepgraph.display_problems()
return 1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2022-04-03 21:46 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2022-04-03 21:46 UTC (permalink / raw
To: gentoo-commits
commit: 4a7e8d870f80b9d2f8a33a8c65d8c8c7db480637
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 3 21:46:38 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Apr 3 21:46:38 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4a7e8d87
Add secret string remote shell command patch
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
patches/secretstring.patch | 74 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 74 insertions(+)
diff --git a/patches/secretstring.patch b/patches/secretstring.patch
new file mode 100644
index 0000000..4e0833d
--- /dev/null
+++ b/patches/secretstring.patch
@@ -0,0 +1,74 @@
+diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py
+index 7ac18b086..82ff7ce07 100644
+--- a/master/buildbot/process/buildstep.py
++++ b/master/buildbot/process/buildstep.py
+@@ -850,6 +850,7 @@ class ShellMixin:
+ 'sigtermTime',
+ 'initialStdin',
+ 'decodeRC',
++ 'SecretString',
+ ]
+ renderables = _shellMixinArgs
+
+diff --git a/master/buildbot/process/remotecommand.py b/master/buildbot/process/remotecommand.py
+index 90ec4c44d..fb3099b28 100644
+--- a/master/buildbot/process/remotecommand.py
++++ b/master/buildbot/process/remotecommand.py
+@@ -44,7 +44,7 @@ class RemoteCommand(base.RemoteCommandImpl):
+
+ def __init__(self, remote_command, args, ignore_updates=False,
+ collectStdout=False, collectStderr=False, decodeRC=None,
+- stdioLogName='stdio'):
++ stdioLogName='stdio', SecretString=False):
+ if decodeRC is None:
+ decodeRC = {0: SUCCESS}
+ self.logs = {}
+@@ -70,6 +70,7 @@ class RemoteCommand(base.RemoteCommandImpl):
+ self.commandID = None
+ self.deferred = None
+ self.interrupted = False
++ self.SecretString = SecretString
+ # a lock to make sure that only one log-handling method runs at a time.
+ # This is really only a problem with old-style steps, which do not
+ # wait for the Deferred from one method before invoking the next.
+@@ -275,6 +276,8 @@ class RemoteCommand(base.RemoteCommandImpl):
+ def cleanup(data):
+ if self.step is None:
+ return data
++ if self.SecretString and isinstance(self.SecretString, list) and len(self.SecretString) == 2:
++ data = data.replace(self.SecretString[0], '<' + self.SecretString[1] + '>')
+ return self.step.build.properties.cleanupTextFromSecrets(data)
+
+ if self.debug:
+@@ -358,7 +361,8 @@ class RemoteShellCommand(RemoteCommand):
+ collectStdout=False, collectStderr=False,
+ interruptSignal=None,
+ initialStdin=None, decodeRC=None,
+- stdioLogName='stdio'):
++ stdioLogName='stdio',
++ SecretString=False):
+ if logfiles is None:
+ logfiles = {}
+ if decodeRC is None:
+@@ -398,7 +402,8 @@ class RemoteShellCommand(RemoteCommand):
+ super().__init__("shell", args, collectStdout=collectStdout,
+ collectStderr=collectStderr,
+ decodeRC=decodeRC,
+- stdioLogName=stdioLogName)
++ stdioLogName=stdioLogName,
++ SecretString=SecretString)
+
+ def _start(self):
+ if self.args['usePTY'] is None:
+diff --git a/master/buildbot/steps/shell.py b/master/buildbot/steps/shell.py
+index d21ca7b9b..cf56a2802 100644
+--- a/master/buildbot/steps/shell.py
++++ b/master/buildbot/steps/shell.py
+@@ -183,6 +183,7 @@ class ShellCommand(buildstep.ShellMixin, buildstep.BuildStep):
+ 'decodeRC',
+ 'stdioLogName',
+ 'workdir',
++ 'SecretString',
+ ] + buildstep.BuildStep.parms
+
+ invalid_args = []
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2022-09-25 11:25 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2022-09-25 11:25 UTC (permalink / raw
To: gentoo-commits
commit: d81ea07c3ae4c6373ec2a798495481995ca15d7d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 25 11:25:32 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Sep 25 11:25:32 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d81ea07c
Update gitlab patch for 3.6.1
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
patches/bb-gitlab.patch | 184 +++++++++++++++---------------------------------
1 file changed, 56 insertions(+), 128 deletions(-)
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
index f57a2ac..f8e559d 100644
--- a/patches/bb-gitlab.patch
+++ b/patches/bb-gitlab.patch
@@ -1,5 +1,49 @@
---- a/buildbot/www/hooks/gitlab.py 2022-03-06 15:10:44.000000000 +0100
-+++ b/buildbot/www/hooks/gitlab.py 2022-08-07 22:22:16.570498909 +0200
+diff --git a/buildbot/reporters/gitlab.py b/buildbot/reporters/gitlab.py
+index 5a403d5d0..a6fc698e2 100644
+--- a/buildbot/reporters/gitlab.py
++++ b/buildbot/reporters/gitlab.py
+@@ -169,8 +169,8 @@ class GitLabStatusPush(ReporterBase):
+ # FIXME: probably only want to report status for the last commit in the changeset
+ for sourcestamp in sourcestamps:
+ sha = sourcestamp['revision']
+- if 'source_project_id' in props:
+- proj_id = props['source_project_id']
++ if 'target_project_id' in props:
++ proj_id = props['target_project_id']
+ else:
+ proj_id = yield self.getProjectId(sourcestamp)
+ if proj_id is None:
+@@ -191,8 +191,11 @@ class GitLabStatusPush(ReporterBase):
+ description=description
+ )
+ if res.code not in (200, 201, 204):
+- message = yield res.json()
+- message = message.get('message', 'unspecified error')
++ if res.code.startswith("{"):
++ message = yield res.json()
++ message = message.get('message', 'unspecified error')
++ else:
++ message = res.code
+ log.msg(
+ f'Could not send status "{state}" for '
+ f'{sourcestamp["repository"]} at {sha}: {message}')
+diff --git a/buildbot/secrets/providers/file.py b/buildbot/secrets/providers/file.py
+index 11daf327a..25e010b38 100644
+--- a/buildbot/secrets/providers/file.py
++++ b/buildbot/secrets/providers/file.py
+@@ -55,7 +55,7 @@ class SecretInAFile(SecretProviderBase):
+ secretvalue = source.read()
+ if suffix:
+ secretfile = secretfile[:-len(suffix)]
+- if strip:
++ if strip and not "OPENSSH PRIVATE KEY" in secretvalue:
+ secretvalue = secretvalue.rstrip("\r\n")
+ secrets[secretfile] = secretvalue
+ return secrets
+diff --git a/master/buildbot/www/hooks/gitlab.py b/master/buildbot/www/hooks/gitlab.py
+index 8f2f80a83..40de0273e 100644
+--- a/buildbot/www/hooks/gitlab.py
++++ b/buildbot/www/hooks/gitlab.py
@@ -19,6 +19,8 @@ import re
from dateutil.parser import parse as dateparse
@@ -9,19 +53,18 @@
from twisted.internet.defer import inlineCallbacks
from twisted.python import log
-@@ -26,6 +28,7 @@ from buildbot.process.properties import
- from buildbot.util import bytes2unicode
- from buildbot.www.hooks.base import BaseHookHandler
+@@ -28,6 +30,7 @@ from buildbot.www.hooks.base import BaseHookHandler
-+_HOSTED_BASE_URL = 'https://gitlab.com'
_HEADER_EVENT = b'X-Gitlab-Event'
_HEADER_GITLAB_TOKEN = b'X-Gitlab-Token'
++_HOSTED_BASE_URL = 'https://gitlab.com'
+
-@@ -94,6 +97,35 @@ class GitLabHandler(BaseHookHandler):
+ class GitLabHandler(BaseHookHandler):
+@@ -94,6 +97,34 @@ class GitLabHandler(BaseHookHandler):
return changes
-+ #@inlineCallbacks
+ def _configGitlabRest(self, token, baseURL=None):
+ if baseURL is None:
+ baseURL = _HOSTED_BASE_URL
@@ -53,82 +96,16 @@
def _process_merge_request_change(self, payload, event, codebase=None):
"""
Consumes the merge_request JSON as a python object and turn it into a buildbot change.
-@@ -126,7 +158,7 @@ class GitLabHandler(BaseHookHandler):
+@@ -126,7 +157,7 @@ class GitLabHandler(BaseHookHandler):
changes = [{
'author': f"{commit['author']['name']} <{commit['author']['email']}>",
- 'files': [], # @todo use rest API
-+ 'files' : self._getFiles(attrs),
++ 'files': [], self._getFiles(attrs)
'comments': f"MR#{attrs['iid']}: {attrs['title']}\n\n{attrs['description']}",
'revision': commit['id'],
'when_timestamp': when_timestamp,
-@@ -151,6 +183,65 @@ class GitLabHandler(BaseHookHandler):
- changes[0]['codebase'] = codebase
- return changes
-
-+ def _process_note_addition_to_merge_request(self, payload, event, codebase=None):
-+ """
-+ Consumes a note event JSON as a python object and turn it into a buildbot change.
-+ :arguments:
-+ payload
-+ Python Object that represents the JSON sent by GitLab Service
-+ Hook.
-+ Comments in merge_requests are send as note events by the API
-+ """
-+ attrs = payload['object_attributes']
-+
-+ # handle only note events coming from merge_requests
-+ # this can be direct comments or comments added to a changeset of the MR
-+ #
-+ # editing a comment does NOT lead to an event at all
-+ if 'merge_request' not in payload:
-+ log.msg(f"Found note event (id {attrs['id']}) without corresponding MR - ignore")
-+ return []
-+
-+ # change handling is very similar to the method above, but
-+ commit = payload['merge_request']['last_commit']
-+ when_timestamp = dateparse(commit['timestamp'])
-+ # @todo provide and document a way to choose between http and ssh url
-+ repo_url = payload['merge_request']['target']['git_http_url']
-+ # project name from http headers is empty for me, so get it from
-+ # object_attributes/target/name
-+ mr = payload['merge_request']
-+ project = mr['target']['name']
-+
-+ log.msg(f"Found notes on MR#{mr['iid']}: {attrs['note']}")
-+ changes = [{
-+ 'author': f"{commit['author']['name']} <{commit['author']['email']}>",
-+ 'files': [], # not provided by rest API
-+ 'comments': f"MR#{mr['iid']}: {mr['title']}\n\n{mr['description']}",
-+ 'revision': commit['id'],
-+ 'when_timestamp': when_timestamp,
-+ 'branch': mr['target_branch'],
-+ 'repository': repo_url,
-+ 'project': project,
-+ 'category': event,
-+ 'revlink': mr['url'],
-+ 'properties': {
-+ 'source_branch': mr['source_branch'],
-+ 'source_project_id': mr['source_project_id'],
-+ 'source_repository': mr['source']['git_http_url'],
-+ 'source_git_ssh_url': mr['source']['git_ssh_url'],
-+ 'target_branch': mr['target_branch'],
-+ 'target_project_id': mr['target_project_id'],
-+ 'target_repository': mr['target']['git_http_url'],
-+ 'target_git_ssh_url': mr['target']['git_ssh_url'],
-+ 'event': event,
-+ 'comments': attrs['note'],
-+ },
-+ }]
-+ if codebase is not None:
-+ changes[0]['codebase'] = codebase
-+ return changes
-+
-+
- @inlineCallbacks
- def getChanges(self, request):
- """
-@@ -160,17 +251,21 @@ class GitLabHandler(BaseHookHandler):
+@@ -220,17 +251,21 @@ class GitLabHandler(BaseHookHandler):
request
the http request object
"""
@@ -153,60 +130,11 @@
try:
content = request.content.read()
payload = json.loads(bytes2unicode(content))
-@@ -189,7 +284,11 @@ class GitLabHandler(BaseHookHandler):
- repo_url = payload['repository']['url']
+@@ -250,6 +285,7 @@ class GitLabHandler(BaseHookHandler):
changes = self._process_change(
payload, user, repo, repo_url, event_type, codebase=codebase)
-+ elif event_type == 'note':
-+ changes = self._process_note_addition_to_merge_request(
-+ payload, event_type, codebase=codebase)
elif event_type == 'merge_request':
+ self.gl = self._configGitlabRest(expected_token_value, baseURL=baseUrl_value)
changes = self._process_merge_request_change(
payload, event_type, codebase=codebase)
- else:
---- a/buildbot/reporters/gitlab.py 2022-03-06 15:10:44.000000000 +0100
-+++ b/buildbot/reporters/gitlab.py 2022-08-23 00:11:55.996195542 +0200
-@@ -95,7 +95,7 @@ class GitLabStatusPush(ReporterBase):
- :param branch: Branch name to create the status for.
- :param sha: Full sha to create the status for.
- :param state: one of the following 'pending', 'success', 'failed'
-- or 'cancelled'.
-+ or 'canceled'.
- :param target_url: Target url to associate with this status.
- :param description: Short description of the status.
- :param context: Context of the result
-@@ -155,10 +155,12 @@ class GitLabStatusPush(ReporterBase):
- SKIPPED: 'success',
- EXCEPTION: 'failed',
- RETRY: 'pending',
-- CANCELLED: 'cancelled'
-+ CANCELLED: 'canceled'
- }.get(build['results'], 'failed')
-- else:
-+ elif build.get('started_at'):
- state = 'running'
-+ else:
-+ state = 'pending'
-
- context = yield props.render(self.context)
-
-@@ -167,8 +169,8 @@ class GitLabStatusPush(ReporterBase):
- # FIXME: probably only want to report status for the last commit in the changeset
- for sourcestamp in sourcestamps:
- sha = sourcestamp['revision']
-- if 'source_project_id' in props:
-- proj_id = props['source_project_id']
-+ if 'target_project_id' in props:
-+ proj_id = props['target_project_id']
- else:
- proj_id = yield self.getProjectId(sourcestamp)
- if proj_id is None:
-@@ -189,6 +191,7 @@ class GitLabStatusPush(ReporterBase):
- description=description
- )
- if res.code not in (200, 201, 204):
-+ log.msg(res.code)
- message = yield res.json()
- message = message.get('message', 'unspecified error')
- log.msg(
+ elif event_type == 'note':
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2022-09-25 12:11 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2022-09-25 12:11 UTC (permalink / raw
To: gentoo-commits
commit: 765bc21ded6f1c3003c877b60dc36235aab48f7f
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 25 12:11:31 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Sep 25 12:11:31 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=765bc21d
Fix a typo in gitlab patch
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
patches/bb-gitlab.patch | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
index f8e559d..8c466f8 100644
--- a/patches/bb-gitlab.patch
+++ b/patches/bb-gitlab.patch
@@ -101,7 +101,7 @@ index 8f2f80a83..40de0273e 100644
changes = [{
'author': f"{commit['author']['name']} <{commit['author']['email']}>",
- 'files': [], # @todo use rest API
-+ 'files': [], self._getFiles(attrs)
++ 'files': self._getFiles(attrs),
'comments': f"MR#{attrs['iid']}: {attrs['title']}\n\n{attrs['description']}",
'revision': commit['id'],
'when_timestamp': when_timestamp,
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2023-05-28 9:51 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2023-05-28 9:51 UTC (permalink / raw
To: gentoo-commits
commit: 79ced78fa0f1b8a74e5582884e650225c658645e
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun May 28 09:51:02 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun May 28 09:51:02 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=79ced78f
Update gitlab patch to bb3.8
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
patches/bb-gitlab.patch | 15 +--------------
1 file changed, 1 insertion(+), 14 deletions(-)
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
index 8c466f8..251991a 100644
--- a/patches/bb-gitlab.patch
+++ b/patches/bb-gitlab.patch
@@ -27,20 +27,7 @@ index 5a403d5d0..a6fc698e2 100644
log.msg(
f'Could not send status "{state}" for '
f'{sourcestamp["repository"]} at {sha}: {message}')
-diff --git a/buildbot/secrets/providers/file.py b/buildbot/secrets/providers/file.py
-index 11daf327a..25e010b38 100644
---- a/buildbot/secrets/providers/file.py
-+++ b/buildbot/secrets/providers/file.py
-@@ -55,7 +55,7 @@ class SecretInAFile(SecretProviderBase):
- secretvalue = source.read()
- if suffix:
- secretfile = secretfile[:-len(suffix)]
-- if strip:
-+ if strip and not "OPENSSH PRIVATE KEY" in secretvalue:
- secretvalue = secretvalue.rstrip("\r\n")
- secrets[secretfile] = secretvalue
- return secrets
-diff --git a/master/buildbot/www/hooks/gitlab.py b/master/buildbot/www/hooks/gitlab.py
+diff --git a/buildbot/www/hooks/gitlab.py b/buildbot/www/hooks/gitlab.py
index 8f2f80a83..40de0273e 100644
--- a/buildbot/www/hooks/gitlab.py
+++ b/buildbot/www/hooks/gitlab.py
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2023-06-26 21:52 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2023-06-26 21:52 UTC (permalink / raw
To: gentoo-commits
commit: aead3c221debd25eba336987e4ad2cda67e870a1
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jun 26 21:44:46 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jun 26 21:44:46 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aead3c22
Add deleteLogChunks patch for bb
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
patches/bb-deleteLogChunks.patch | 83 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 83 insertions(+)
diff --git a/patches/bb-deleteLogChunks.patch b/patches/bb-deleteLogChunks.patch
new file mode 100644
index 0000000..6de332e
--- /dev/null
+++ b/patches/bb-deleteLogChunks.patch
@@ -0,0 +1,83 @@
+--- a/buildbot/db/logs.py 2022-04-02 11:10:34.892310594 +0200
++++ b/buildbot/db/logs.py 2023-06-26 23:06:24.611959431 +0200
+@@ -410,3 +410,80 @@
+ rv = dict(row)
+ rv['complete'] = bool(rv['complete'])
+ return rv
++
++ # returns a Deferred that returns a value
++ def deleteLogChunks(self, buildid):
++ model = self.db.model
++ horizon_per_builder = False
++
++ def countLogchunks(conn):
++ res = conn.execute(sa.select([sa.func.count(model.logchunks.c.logid)]))
++ count = res.fetchone()[0]
++ res.close()
++ return count
++
++ # find the steps.id at the upper bound of steps
++ def getStepidMax(conn, buildid):
++ # N.B.: we utilize the fact that steps.id is auto-increment, thus steps.started_at
++ # times are effectively sorted and we only need to find the steps.id at the upper
++ # bound of steps to update.
++
++ # SELECT steps.id from steps WHERE steps.buildid = buildid ORDER BY
++ # steps.id DESC LIMIT 1;
++ res = conn.execute(
++ sa.select([model.steps.c.id])
++ .where(model.steps.c.buildid == buildid)
++ .order_by(model.steps.c.id.desc())
++ .limit(1)
++ )
++ res_list = res.fetchone()
++ stepid_max = None
++ if res_list:
++ stepid_max = res_list[0]
++ res.close()
++ return stepid_max
++
++ # query all logs with type 'd' and delete their chunks.
++ def deleteLogsWithTypeD(conn):
++ if self.db._engine.dialect.name == 'sqlite':
++ # sqlite does not support delete with a join, so for this case we use a subquery,
++ # which is much slower
++ q = sa.select([model.logs.c.id])
++ q = q.select_from(model.logs)
++ q = q.where(model.logs.c.type == 'd')
++
++ # delete their logchunks
++ q = model.logchunks.delete().where(model.logchunks.c.logid.in_(q))
++ else:
++ q = model.logchunks.delete()
++ q = q.where(model.logs.c.id == model.logchunks.c.logid)
++ q = q.where(model.logs.c.type == 'd')
++
++ res = conn.execute(q)
++ res.close()
++
++ def thddeleteLogs(conn):
++ count_before = countLogchunks(conn)
++
++ # update log types that match buildid
++ # we do it first to avoid having UI discrepancy
++
++ stepid_max = getStepidMax(conn, buildid)
++ if stepid_max:
++ # UPDATE logs SET logs.type = 'd'
++ # WHERE logs.stepid <= stepid_max AND type != 'd';
++ res = conn.execute(
++ model.logs.update()
++ .where(sa.and_(model.logs.c.stepid <= stepid_max,
++ model.logs.c.type != 'd'))
++ .values(type='d')
++ )
++ res.close()
++
++ deleteLogsWithTypeD(conn)
++
++ count_after = countLogchunks(conn)
++ count = count_before - count_after
++
++ return count if count > 0 else 0
++ return self.db.pool.do(thddeleteLogs)
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/
@ 2024-10-30 21:44 Magnus Granberg
0 siblings, 0 replies; 11+ messages in thread
From: Magnus Granberg @ 2024-10-30 21:44 UTC (permalink / raw
To: gentoo-commits
commit: 67dc24f6e2e8ad8b6932c761e6960700a6ada405
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 30 21:40:25 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Oct 30 21:40:25 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=67dc24f6
Update patches to support Buildbot 4.1.0
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
patches/bb-clean-db.patch | 176 ++++++++++++--------------------------
patches/bb-gitlab.patch | 22 ++---
patches/bb-props_master_fix.patch | 2 +-
3 files changed, 67 insertions(+), 133 deletions(-)
diff --git a/patches/bb-clean-db.patch b/patches/bb-clean-db.patch
index 8d370a4..c9eecda 100644
--- a/patches/bb-clean-db.patch
+++ b/patches/bb-clean-db.patch
@@ -1,60 +1,56 @@
--- a/buildbot/db/build_data.py 2023-09-19 23:24:04.714268581 +0200
+++ b/buildbot/db/build_data.py 2023-09-24 22:28:21.778583084 +0200
-@@ -159,6 +159,15 @@ class BuildDataConnectorComponent(base.D
- res = yield self.db.pool.do(thd)
- return res
+@@ -185,6 +185,16 @@
-+ @defer.inlineCallbacks
-+ def pruneBuildData(self, buildid):
-+ def thd(conn):
+ return self.db.pool.do(thd)
+
++ def pruneBuildData(self, buildid: int) -> defer.Deferred[None]:
++ def thd(conn) -> None:
+ tbl = self.db.model.build_data
+ q = tbl.delete()
+ q = q.where(tbl.c.buildid == buildid)
-+ conn.execute(q)
-+ yield self.db.pool.do(thd)
++ res = conn.execute(q)
++ conn.commit()
++ res.close()
++ return self.db.pool.do(thd)
+
- def _row2dict(self, conn, row):
- return BuildDataDict(buildid=row.buildid,
- name=row.name,
+ def _model_from_row(self, row, value: bytes | None):
+ return BuildDataModel(
+ buildid=row.buildid,
--- a/buildbot/db/builds.py 2023-09-19 23:24:04.714268581 +0200
+++ b/buildbot/db/builds.py 2023-09-24 15:31:20.598023751 +0200
-@@ -206,6 +206,15 @@ class BuildsConnectorComponent(base.DBCo
- results=results)
- return self.db.pool.do(thd)
+@@ -335,6 +335,26 @@
-+ @defer.inlineCallbacks
-+ def pruneBuild(self, buildid):
-+ def thd(conn):
+ yield self.db.pool.do_with_transaction(thd)
+
++ def pruneBuild(self, buildid: int) -> defer.Deferred[None]:
++ def thd(conn) -> None:
+ tbl = self.db.model.builds
+ q = tbl.delete()
+ q = q.where(tbl.c.id == buildid)
-+ conn.execute(q)
-+ yield self.db.pool.do(thd)
++ res = conn.execute(q)
++ conn.commit()
++ res.close()
++ return self.db.pool.do(thd)
+
- # returns a Deferred that returns a value
- def getBuildProperties(self, bid, resultSpec=None):
- def thd(conn):
-@@ -251,6 +260,15 @@ class BuildsConnectorComponent(base.DBCo
- {"value": value_js, "source": source})
- yield self.db.pool.do(thd)
-
-+ @defer.inlineCallbacks
-+ def pruneBuildProperties(self, buildid):
-+ def thd(conn):
++ def pruneBuildProperties(self, buildid: int) -> defer.Deferred[None]:
++ def thd(conn) -> None:
+ bp_tbl = self.db.model.build_properties
+ q = bp_tbl.delete()
+ q = q.where(bp_tbl.c.buildid == buildid)
-+ conn.execute(q)
-+ yield self.db.pool.do(thd)
++ res = conn.execute(q)
++ conn.commit()
++ res.close()
++ return self.db.pool.do(thd)
+
- def _builddictFromRow(self, row):
- return {
- "id": row.id,
+ def _model_from_row(self, row):
+ return BuildModel(
+ id=row.id,
--- a/buildbot/db/changes.py 2023-09-19 23:24:04.714268581 +0200
+++ b/buildbot/db/changes.py 2023-09-24 13:53:28.166432187 +0200
-@@ -333,6 +334,40 @@ class ChangesConnectorComponent(base.DBC
- table.delete(table.c.changeid.in_(batch)))
- yield self.db.pool.do(thd)
+@@ -333,6 +334,40 @@ class ChangesConnectorComponent(base.
+
+ yield self.db.pool.do_with_transaction(thd)
+ @defer.inlineCallbacks
+ def pruneChangesId(self, revision):
@@ -87,93 +83,31 @@
+ table = self.db.model.metadata.tables[table_name]
+ conn.execute(
+ table.delete(table.c.changeid.in_(batch)))
-+ yield self.db.pool.do(thd)
++ yield self.db.pool.do_with_transaction(thd)
+
+
- def _chdict_from_change_row_thd(self, conn, ch_row):
- # This method must be run in a db.pool thread, and returns a chdict
- # given a row from the 'changes' table
-
+ def _thd_model_from_row(self, conn, ch_row) -> ChangeModel:
+ # This method must be run in a db.pool thread
+ change_files_tbl = self.db.model.change_files
--- a/buildbot/db/logs.py 2022-04-02 11:10:34.892310594 +0200
+++ b/buildbot/db/logs.py 2023-06-26 23:06:24.611959431 +0200
-@@ -410,3 +410,80 @@
- rv = dict(row)
- rv['complete'] = bool(rv['complete'])
- return rv
-+
-+ # returns a Deferred that returns a value
-+ def deleteLogChunks(self, buildid):
-+ model = self.db.model
-+ horizon_per_builder = False
-+
-+ def countLogchunks(conn):
-+ res = conn.execute(sa.select([sa.func.count(model.logchunks.c.logid)]))
-+ count = res.fetchone()[0]
-+ res.close()
-+ return count
-+
-+ # find the steps.id at the upper bound of steps
-+ def getStepidMax(conn, buildid):
-+ # N.B.: we utilize the fact that steps.id is auto-increment, thus steps.started_at
-+ # times are effectively sorted and we only need to find the steps.id at the upper
-+ # bound of steps to update.
-+
-+ # SELECT steps.id from steps WHERE steps.buildid = buildid ORDER BY
-+ # steps.id DESC LIMIT 1;
-+ res = conn.execute(
-+ sa.select([model.steps.c.id])
-+ .where(model.steps.c.buildid == buildid)
-+ .order_by(model.steps.c.id.desc())
-+ .limit(1)
-+ )
-+ res_list = res.fetchone()
-+ stepid_max = None
-+ if res_list:
-+ stepid_max = res_list[0]
+@@ -502,3 +502,19 @@
+ num_lines=row.num_lines,
+ type=row.type,
+ )
++ def deleteLogChunks(self, logid: int) -> defer.Deferred[None]:
++ def thd(conn) -> None:
++ tbl = self.db.model.logchunks
++ res = conn.execute(tbl.delete().where(tbl.c.logid == logid))
+ res.close()
-+ return stepid_max
-+
-+ # query all logs with type 'd' and delete their chunks.
-+ def deleteLogsWithTypeD(conn):
-+ if self.db._engine.dialect.name == 'sqlite':
-+ # sqlite does not support delete with a join, so for this case we use a subquery,
-+ # which is much slower
-+ q = sa.select([model.logs.c.id])
-+ q = q.select_from(model.logs)
-+ q = q.where(model.logs.c.type == 'd')
-+
-+ # delete their logchunks
-+ q = model.logchunks.delete().where(model.logchunks.c.logid.in_(q))
-+ else:
-+ q = model.logchunks.delete()
-+ q = q.where(model.logs.c.id == model.logchunks.c.logid)
-+ q = q.where(model.logs.c.type == 'd')
-+
-+ res = conn.execute(q)
++ return self.db.pool.do(thd)
++
++ def deleteLog(self, stepid: int) -> defer.Deferred[None]:
++ def thd(conn) -> None:
++ tbl = self.db.model.logs
++ q = tbl.update()
++ q = q.where(tbl.c.id == stepid)
++ q = q.where(tbl.c.type != 'd')
++ res = conn.execute(q.values(type='d'))
+ res.close()
-+
-+ def thddeleteLogs(conn):
-+ count_before = countLogchunks(conn)
-+
-+ # update log types that match buildid
-+ # we do it first to avoid having UI discrepancy
-+
-+ stepid_max = getStepidMax(conn, buildid)
-+ if stepid_max:
-+ # UPDATE logs SET logs.type = 'd'
-+ # WHERE logs.stepid <= stepid_max AND type != 'd';
-+ res = conn.execute(
-+ model.logs.update()
-+ .where(sa.and_(model.logs.c.stepid <= stepid_max,
-+ model.logs.c.type != 'd'))
-+ .values(type='d')
-+ )
-+ res.close()
-+
-+ deleteLogsWithTypeD(conn)
-+
-+ count_after = countLogchunks(conn)
-+ count = count_before - count_after
-+
-+ return count if count > 0 else 0
-+ return self.db.pool.do(thddeleteLogs)
++ return self.db.pool.do(thd)
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
index 251991a..fe185c3 100644
--- a/patches/bb-gitlab.patch
+++ b/patches/bb-gitlab.patch
@@ -37,9 +37,9 @@ index 8f2f80a83..40de0273e 100644
+import gitlab as python_gitlab
+
+ from dateutil.parser import parse as dateparse
from twisted.internet.defer import inlineCallbacks
from twisted.python import log
-
@@ -28,6 +30,7 @@ from buildbot.www.hooks.base import BaseHookHandler
_HEADER_EVENT = b'X-Gitlab-Event'
@@ -84,14 +84,14 @@ index 8f2f80a83..40de0273e 100644
"""
Consumes the merge_request JSON as a python object and turn it into a buildbot change.
@@ -126,7 +157,7 @@ class GitLabHandler(BaseHookHandler):
-
- changes = [{
- 'author': f"{commit['author']['name']} <{commit['author']['email']}>",
-- 'files': [], # @todo use rest API
-+ 'files': self._getFiles(attrs),
- 'comments': f"MR#{attrs['iid']}: {attrs['title']}\n\n{attrs['description']}",
- 'revision': commit['id'],
- 'when_timestamp': when_timestamp,
+ changes = [
+ {
+ 'author': f"{commit['author']['name']} <{commit['author']['email']}>",
+- 'files': [], # @todo use rest API
++ 'files': self._getFiles(attrs),
+ 'comments': f"MR#{attrs['iid']}: {attrs['title']}\n\n{attrs['description']}",
+ 'revision': commit['id'],
+ 'when_timestamp': when_timestamp,
@@ -220,17 +251,21 @@ class GitLabHandler(BaseHookHandler):
request
the http request object
@@ -122,6 +122,6 @@ index 8f2f80a83..40de0273e 100644
payload, user, repo, repo_url, event_type, codebase=codebase)
elif event_type == 'merge_request':
+ self.gl = self._configGitlabRest(expected_token_value, baseURL=baseUrl_value)
- changes = self._process_merge_request_change(
- payload, event_type, codebase=codebase)
+ changes = self._process_merge_request_change(payload, event_type, codebase=codebase)
elif event_type == 'note':
+ changes = self._process_note_addition_to_merge_request(
diff --git a/patches/bb-props_master_fix.patch b/patches/bb-props_master_fix.patch
index f31d4ad..2cf7b64 100644
--- a/patches/bb-props_master_fix.patch
+++ b/patches/bb-props_master_fix.patch
@@ -5,7 +5,7 @@ index df8f9c014..c16e4f64e 100644
@@ -288,6 +288,10 @@ class Builder(util_service.ReconfigurableServiceMixin,
# The instance type of the worker may depend on the properties of
# the build that substantiated it.
- props = setupPropsIfNeeded(props)
+ props = yield self._setup_props_if_needed(props, workerforbuilder, buildrequest)
+ # check if we have props.master when we call renderWorkerProps later
+ # set props.master if needed
+ if props.master is None:
^ permalink raw reply related [flat|nested] 11+ messages in thread
end of thread, other threads:[~2024-10-30 21:44 UTC | newest]
Thread overview: 11+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2023-05-28 9:51 [gentoo-commits] proj/tinderbox-cluster:master commit in: patches/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2024-10-30 21:44 Magnus Granberg
2023-06-26 21:52 Magnus Granberg
2022-09-25 12:11 Magnus Granberg
2022-09-25 11:25 Magnus Granberg
2022-04-03 21:46 Magnus Granberg
2016-01-19 17:39 Magnus Granberg
2015-12-19 11:18 Magnus Granberg
2015-10-04 20:10 Magnus Granberg
2015-10-04 20:06 Magnus Granberg
2015-04-18 20:27 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox