From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 9AFC1158090 for ; Sat, 28 May 2022 14:55:22 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 394D9E084A; Sat, 28 May 2022 14:55:21 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [140.211.166.183]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id A4EECE084A for ; Sat, 28 May 2022 14:55:20 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 2845F341444 for ; Sat, 28 May 2022 14:55:19 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 8821B326 for ; Sat, 28 May 2022 14:55:17 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1653749690.1f57c498694cbdce80ad43b2b5a3c0c44cdee588.zorry@gentoo> Subject: [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/ X-VCS-Repository: proj/tinderbox-cluster X-VCS-Files: buildbot_gentoo_ci/config/workers.py buildbot_gentoo_ci/steps/builders.py X-VCS-Directories: buildbot_gentoo_ci/steps/ buildbot_gentoo_ci/config/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 1f57c498694cbdce80ad43b2b5a3c0c44cdee588 X-VCS-Branch: master Date: Sat, 28 May 2022 14:55:17 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: fbfb8660-7355-4fc2-9dbf-087990635261 X-Archives-Hash: 872652a2ac2492d37d8a7442d8de0f66 commit: 1f57c498694cbdce80ad43b2b5a3c0c44cdee588 Author: Magnus Granberg gentoo org> AuthorDate: Sat May 28 14:54:50 2022 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Sat May 28 14:54:50 2022 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1f57c498 Add support for docker latent workers Signed-off-by: Magnus Granberg gentoo.org> buildbot_gentoo_ci/config/workers.py | 39 ++++++++++++++++++++++++++++++++++-- buildbot_gentoo_ci/steps/builders.py | 15 ++++++++------ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py index fbdc2a9..d4c20f7 100644 --- a/buildbot_gentoo_ci/config/workers.py +++ b/buildbot_gentoo_ci/config/workers.py @@ -1,7 +1,7 @@ -# Copyright 2021 Gentoo Authors +# Copyright 2022 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from buildbot.plugins import worker +from buildbot.plugins import worker, util class gentoo_ci_workers(): def __init__(self, worker_data, **kwargs): @@ -47,17 +47,52 @@ class gentoo_ci_workers(): print(node_worker) return node_worker +@util.renderer +def docker_images(props): + return 'bb-worker-' + props.getProperty('project_uuid') + ':latest' + +@util.renderer +def docker_volumes(props): + volumes_list = [] + #FIXME: set in master.cfg + src_dir = '/srv/gentoo/portage/' + props.getProperty('project_uuid') + dest_dir = '/var/cache/portage' + #add distdir + volumes_list.append(src_dir + '/distfiles' + ':' + dest_dir + '/distfiles') + #add bindir + volumes_list.append(src_dir + '/packages' + ':' + dest_dir + '/packages') + return volumes_list + def gentoo_workers(worker_data): w = [] g_ci_w = gentoo_ci_workers(worker_data) LocalWorkers = g_ci_w.getLocalWorkersUuid() BuildWorkers = g_ci_w.getBuildWorkersAllData() NodeWorkers = g_ci_w.getNodedWorkersAllData() + docker_hostconfig = {} + # For use of sandbox stuff + # FEATURES="ipc-sandbox network-sandbox pid-sandbox" + docker_hostconfig['cap_add'] = ['SYS_ADMIN', 'NET_ADMIN', 'SYS_PTRACE'] + # libseccomp overhead + # https://github.com/seccomp/libseccomp/issues/153 + docker_hostconfig['security_opt'] = ['seccomp=unconfined'] for local_worker in LocalWorkers: w.append(worker.LocalWorker(local_worker)) for build_worker in BuildWorkers: if build_worker['type'] == 'default': w.append(worker.Worker(build_worker['uuid'], build_worker['password'])) + #FIXME: set settings in master.cfg + if build_worker['type'] == 'docker': + w.append(worker.DockerLatentWorker(build_worker['uuid'], + build_worker['password'], + docker_host='tcp://192.168.1.3:2375', + image=docker_images, + volumes=docker_volumes, + hostconfig=docker_hostconfig, + followStartupLogs=True, + masterFQDN='192.168.1.5', + build_wait_timeout=3600 + )) for node_worker in NodeWorkers: if node_worker['type'] == 'node': w.append(worker.Worker(node_worker['uuid'], node_worker['password'])) diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index 8ce6545..1d4f52d 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -236,7 +236,8 @@ class TriggerRunBuildRequest(BuildStep): 'projectrepository_data' : self.projectrepository_data, 'use_data' : self.use_data, 'fullcheck' : self.getProperty("fullcheck"), - 'project_build_data' : project_build_data + 'project_build_data' : project_build_data, + 'project_uuid' : self.project_data['uuid'] } )]) return SUCCESS @@ -384,6 +385,8 @@ class RunEmerge(BuildStep): self.descriptionSuffix = self.step self.name = 'Setup emerge for ' + self.step + ' step' self.build_env = {} + #FIXME: Set build timeout in config + self.build_timeout = 1800 @defer.inlineCallbacks def run(self): @@ -454,7 +457,7 @@ class RunEmerge(BuildStep): strip=True, extract_fn=PersOutputOfEmerge, workdir='/', - timeout=None + timeout=self.build_timeout )) aftersteps_list.append(CheckEmergeLogs('update')) if projects_emerge_options['preserved_libs']: @@ -470,7 +473,7 @@ class RunEmerge(BuildStep): strip=True, extract_fn=PersOutputOfEmerge, workdir='/', - timeout=None + timeout=self.build_timeout )) aftersteps_list.append(CheckEmergeLogs('preserved-libs')) self.setProperty('preserved_libs', False, 'preserved-libs') @@ -532,7 +535,7 @@ class RunEmerge(BuildStep): strip=True, extract_fn=PersOutputOfEmerge, workdir='/', - timeout=None + timeout=self.build_timeout )) aftersteps_list.append(CheckEmergeLogs('match')) @@ -561,7 +564,7 @@ class RunEmerge(BuildStep): strip=True, extract_fn=PersOutputOfEmerge, workdir='/', - timeout=None + timeout=self.build_timeout )) aftersteps_list.append(CheckEmergeLogs('pre-build')) @@ -588,7 +591,7 @@ class RunEmerge(BuildStep): extract_fn=PersOutputOfEmerge, workdir='/', env=self.build_env, - timeout=None + timeout=self.build_timeout )) aftersteps_list.append(CheckEmergeLogs('build')) if projects_emerge_options['preserved_libs']: