From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 2079B138359 for ; Sat, 29 Aug 2020 12:34:41 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 46C3CE0918; Sat, 29 Aug 2020 12:34:40 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 1EB5EE0918 for ; Sat, 29 Aug 2020 12:34:40 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 1BF5E340C50 for ; Sat, 29 Aug 2020 12:34:39 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id C284B2EE for ; Sat, 29 Aug 2020 12:34:37 +0000 (UTC) From: "Andreas K. Hüttel" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Andreas K. Hüttel" Message-ID: <1598704417.e2d27f67f5c832e30b4683917e2bd7e28f22cfb9.dilfridge@gentoo> Subject: [gentoo-commits] proj/releng:master commit in: tools/ X-VCS-Repository: proj/releng X-VCS-Files: tools/catalyst-auto X-VCS-Directories: tools/ X-VCS-Committer: dilfridge X-VCS-Committer-Name: Andreas K. Hüttel X-VCS-Revision: e2d27f67f5c832e30b4683917e2bd7e28f22cfb9 X-VCS-Branch: master Date: Sat, 29 Aug 2020 12:34:37 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 55bae78d-e654-4490-aeaf-719c0e50eced X-Archives-Hash: b1cc8a1abe950ffa0483ae3a74a7b5e0 commit: e2d27f67f5c832e30b4683917e2bd7e28f22cfb9 Author: Andreas K. Huettel gentoo org> AuthorDate: Sat Aug 22 19:11:51 2020 +0000 Commit: Andreas K. Hüttel gentoo org> CommitDate: Sat Aug 29 12:33:37 2020 +0000 URL: https://gitweb.gentoo.org/proj/releng.git/commit/?id=e2d27f67 catalyst-auto: Allow parallel building of spec sets This assumes that the spec sets are logically independent from each other, i.e., building different ABIs. Code shamelessly adapted from locale-gen. Signed-off-by: Andreas K. Huettel gentoo.org> tools/catalyst-auto | 97 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/tools/catalyst-auto b/tools/catalyst-auto index 34238c01..484eaffc 100755 --- a/tools/catalyst-auto +++ b/tools/catalyst-auto @@ -23,6 +23,7 @@ testing=0 preclean=0 lastrun=0 lock_file= +parallel_sets=1 usage() { local msg=$1 @@ -38,6 +39,7 @@ Usage: Options: -c|--config Specifies the config file to use (required) -C|--preclean Clean up loose artifacts from previous runs + -j|--jobs Build spec sets in parallel -v|--verbose Send output of commands to console as well as log -k|--keep-tmpdir Don't remove temp dir when build finishes -t|--test Stop after mangling specs and copying files @@ -133,6 +135,10 @@ parse_args() { config_files+=("$1") shift ;; + -j|--jobs) + parallel_sets="$1" + shift + ;; -v|--verbose) verbose=$(($verbose+1)) ;; @@ -385,47 +391,76 @@ run_catalyst_commands() { timeprefix=() which time >/dev/null && timeprefix=( "time" ) + JOB_PIDS=() + JOB_RETS=() + JOB_IDX_S=0 + JOB_IDX_E=0 + for a in "" ${SETS}; do - if [[ -z ${a} ]]; then - specs_var="SPECS" - optional_specs_var="OPTIONAL_SPECS" - else - specs_var="SET_${a}_SPECS" - optional_specs_var="SET_${a}_OPTIONAL_SPECS" + + if [[ $(( JOB_IDX_E - JOB_IDX_S )) == ${parallel_sets} ]] ; then + wait ${JOB_PIDS[$(( JOB_IDX_S++ ))]} + JOB_RETS+=( $? ) fi - for i in ${!specs_var}; do - LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log" - specpath=$(readlink -f "${i}") - run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}" - if [[ $? != 0 ]]; then - build_failure=1 - send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}" - continue 2 - else - trigger_post_build "${a}" "${i}" - fi - done + ( - for i in ${!optional_specs_var}; do - LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log" - specpath=$(readlink -f "${i}") - run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}" - if [[ $? != 0 ]]; then - build_failure=1 - send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}" - break + if [[ -z ${a} ]]; then + specs_var="SPECS" + optional_specs_var="OPTIONAL_SPECS" else - trigger_post_build "${a}" "${i}" + specs_var="SET_${a}_SPECS" + optional_specs_var="SET_${a}_OPTIONAL_SPECS" fi - done - # Do not purge yet, because there might be interdendency between specs - # in different build sets! + for i in ${!specs_var}; do + LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log" + specpath=$(readlink -f "${i}") + run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}" + if [[ $? != 0 ]]; then + build_failure=1 + send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}" + exit 1 + else + trigger_post_build "${a}" "${i}" + fi + done + + for i in ${!optional_specs_var}; do + LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log" + specpath=$(readlink -f "${i}") + run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}" + if [[ $? != 0 ]]; then + build_failure=1 + send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}" + break + else + trigger_post_build "${a}" "${i}" + fi + done + + # Do not purge yet, because there might be interdendency between specs + # in different build sets! + + update_symlinks + + exit ${build_failure} + + )& + + JOB_PIDS+=( $! ) + : $(( ++JOB_IDX_E )) + - update_symlinks done + for (( i = JOB_IDX_S; i < JOB_IDX_E; ++i )) ; do + wait ${JOB_PIDS[i]} + JOB_RETS+=( $? ) + done + build_failure=$(( 0 ${JOB_RETS[@]/#/+} )) + + # Now do the cleanup for a in "" ${SETS}; do if [[ -z ${a} ]]; then