From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id E25C71382C5 for ; Fri, 15 May 2020 13:07:26 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 87F8EE0B51; Fri, 15 May 2020 13:07:25 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 1D37FE0B47 for ; Fri, 15 May 2020 13:07:25 +0000 (UTC) From: Ben Kohler To: gentoo-releng@lists.gentoo.org Subject: [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred Date: Fri, 15 May 2020 08:07:18 -0500 Message-Id: <20200515130719.3138513-1-bkohler@gentoo.org> X-Mailer: git-send-email 2.26.2 Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-releng@lists.gentoo.org Reply-to: gentoo-releng@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Archives-Salt: e617b5a9-d661-4387-b382-a6568b08fc71 X-Archives-Hash: 141a05f1dd54864d003aa7af5dfc027d Previously when we were copying locally there was no (major) problem with wildcarding a long list of files to be transferred after every single spec finishes. Now that we are transferring to a remote host, this results in a lot of wasted transfer time & bandwidth. I have adjusted the post_build function to only transfer the new files known to be produced by each spec (which has files meant to be published). Signed-off-by: Ben Kohler --- tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++------- tools/catalyst-auto-amd64.conf | 86 +++++++++++++------- 2 files changed, 113 insertions(+), 60 deletions(-) diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf index e7b1ea79..2aa91a03 100644 --- a/tools/catalyst-auto-amd64-experimental.conf +++ b/tools/catalyst-auto-amd64-experimental.conf @@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec" SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec" + SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec" SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec" SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec" @@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]" update_symlinks() { # Symlink the latest stages3 to build from for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do - pushd $d >/dev/null + pushd "${d}" >/dev/null || exit for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do # 20yymmddThhmmssZ # 20yymmddhhmmss @@ -57,11 +58,13 @@ update_symlinks() { <<<"$f") ln -svf "$f" "$of" done - popd >/dev/null + popd >/dev/null || exit done } post_build() { + local set=$1 spec=$2 + cmd=( rsync -e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no' @@ -80,34 +83,58 @@ post_build() { DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened ;; esac - pushd ${BUILD_SRCDIR_BASE}/default >/dev/null - mkdir -p ${TMPDIR}/empty - "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT} - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do - if [ -f $file ]; then - "${cmd[@]}" ${file}* ${DEST_DEFAULT} - fi - done - if [ -f *${DATESTAMP}*.iso ]; then - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT} - elif [ -f *${TIMESTAMP}*.iso ]; then - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT} - fi - popd >/dev/null - - pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null - "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED} - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do - if [ -f $file ]; then - "${cmd[@]}" $file* ${DEST_HARDENED} - fi - done - if [ -f *${DATESTAMP}*.iso ]; then - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED} - elif [ -f *${TIMESTAMP}*.iso ]; then - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED} - fi - popd >/dev/null + + pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit + case ${spec} in + stage3.spec) + "${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + installcd-stage2-minimal.spec) + "${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}" + ;; + stage4-minimal.spec) + "${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + stage3-nomultilib.spec) + "${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + stage4-nomultilib-minimal.spec) + "${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + stage3-x32.spec) + "${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + esac + popd >/dev/null || exit + + pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit + case ${spec} in + hardened/stage3.spec) + "${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/admincd-stage2.spec) + "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}" + ;; + hardened/stage4-minimal.spec) + "${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/stage3-nomultilib.spec) + "${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/stage4-nomultilib-minimal.spec) + "${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/stage3-selinux.spec) + "${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/admincd-stage2-selinux.spec) + "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}" + ;; + hardened/stage3-selinux-nomultilib.spec) + "${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + esac + popd >/dev/null || exit } # vim:ft=sh: diff --git a/tools/catalyst-auto-amd64.conf b/tools/catalyst-auto-amd64.conf index 0b7e9b2f..585c22c6 100644 --- a/tools/catalyst-auto-amd64.conf +++ b/tools/catalyst-auto-amd64.conf @@ -44,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]" update_symlinks() { # Symlink the latest stages3 to build from for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do - pushd $d >/dev/null + pushd "${d}" >/dev/null || exit for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do # 20yymmddThhmmssZ # 20yymmddhhmmss @@ -58,11 +58,13 @@ update_symlinks() { <<<"$f") ln -svf "$f" "$of" done - popd >/dev/null + popd >/dev/null || exit done } post_build() { + local set=$1 spec=$2 + cmd=( rsync -e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no' @@ -81,34 +83,58 @@ post_build() { DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened ;; esac - pushd ${BUILD_SRCDIR_BASE}/default >/dev/null - mkdir -p ${TMPDIR}/empty - "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT} - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do - if [ -f $file ]; then - "${cmd[@]}" ${file}* ${DEST_DEFAULT} - fi - done - if [ -f *${DATESTAMP}*.iso ]; then - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT} - elif [ -f *${TIMESTAMP}*.iso ]; then - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT} - fi - popd >/dev/null - - pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null - "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED} - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do - if [ -f $file ]; then - "${cmd[@]}" $file* ${DEST_HARDENED} - fi - done - if [ -f *${DATESTAMP}*.iso ]; then - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED} - elif [ -f *${TIMESTAMP}*.iso ]; then - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED} - fi - popd >/dev/null + + pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit + case ${spec} in + stage3.spec) + "${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + installcd-stage2-minimal.spec) + "${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}" + ;; + stage4-minimal.spec) + "${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + stage3-nomultilib.spec) + "${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + stage4-nomultilib-minimal.spec) + "${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + stage3-x32.spec) + "${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + esac + popd >/dev/null || exit + + pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit + case ${spec} in + hardened/stage3.spec) + "${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/admincd-stage2.spec) + "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}" + ;; + hardened/stage4-minimal.spec) + "${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/stage3-nomultilib.spec) + "${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/stage4-nomultilib-minimal.spec) + "${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/stage3-selinux.spec) + "${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/admincd-stage2-selinux.spec) + "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}" + ;; + hardened/stage3-selinux-nomultilib.spec) + "${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + esac + popd >/dev/null || exit } # vim:ft=sh: -- 2.26.2