From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 18FF013835A for ; Sat, 16 May 2020 18:26:28 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 3D458E0A52; Sat, 16 May 2020 18:26:26 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 11CEAE0A52 for ; Sat, 16 May 2020 18:26:24 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 2763834F4AD for ; Sat, 16 May 2020 18:26:23 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id C425F1B3 for ; Sat, 16 May 2020 18:26:20 +0000 (UTC) From: "Ben Kohler" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Ben Kohler" Message-ID: <1589547753.151b13ac968bc3eab8226e20489f193b8b3b48d7.bkohler@gentoo> Subject: [gentoo-commits] proj/releng:master commit in: tools/ X-VCS-Repository: proj/releng X-VCS-Files: tools/catalyst-auto-x86-experimental.conf tools/catalyst-auto-x86.conf X-VCS-Directories: tools/ X-VCS-Committer: bkohler X-VCS-Committer-Name: Ben Kohler X-VCS-Revision: 151b13ac968bc3eab8226e20489f193b8b3b48d7 X-VCS-Branch: master Date: Sat, 16 May 2020 18:26:20 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 75f7364b-9677-4bb1-a199-9a006922e09e X-Archives-Hash: 4097ba4cd563f3f43083ca17240d1885 commit: 151b13ac968bc3eab8226e20489f193b8b3b48d7 Author: Ben Kohler gentoo org> AuthorDate: Fri May 15 13:02:33 2020 +0000 Commit: Ben Kohler gentoo org> CommitDate: Fri May 15 13:02:33 2020 +0000 URL: https://gitweb.gentoo.org/proj/releng.git/commit/?id=151b13ac catalyst-auto-x86*.conf: explicitly list files to be transferred Previously when we were copying locally there was no (major) problem with wildcarding a long list of files to be transferred after every single spec finishes. Now that we are transferring to a remote host, this results in a lot of wasted transfer time & bandwidth. I have adjusted the post_build function to only transfer the new files known to be produced by each spec (which has files meant to be published). Signed-off-by: Ben Kohler gentoo.org> tools/catalyst-auto-x86-experimental.conf | 71 +++++++++++++++++-------------- tools/catalyst-auto-x86.conf | 57 ++++++++++++------------- 2 files changed, 67 insertions(+), 61 deletions(-) diff --git a/tools/catalyst-auto-x86-experimental.conf b/tools/catalyst-auto-x86-experimental.conf index 3fd60287..061cab92 100644 --- a/tools/catalyst-auto-x86-experimental.conf +++ b/tools/catalyst-auto-x86-experimental.conf @@ -30,17 +30,27 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]" update_symlinks() { # Symlink the latest stages3 to build from for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do - pushd $d >/dev/null + pushd "${d}" >/dev/null || exit for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do - of=${f/20[0-9][0-9].[0-9]/latest} # for 20XX.Y stuff - of=${of/20[0-9][0-9][0-1][0-9][0-9][0-9]/latest} # for 20YYMMDD stuff - ln -sf $f $of + # 20yymmddThhmmssZ + # 20yymmddhhmmss + # 20yymmdd + # 20yy.n + of=$(perl -p \ + -e 's/20\d{6}T\d{6}Z/latest/g;' \ + -e 's/20\d{6}\d{6}/latest/g;' \ + -e 's/20\d{6}/latest/g;' \ + -e 's/20\d{2}\.\d{2}/latest/g;' \ + <<<"$f") + ln -svf "$f" "$of" done - popd >/dev/null + popd >/dev/null || exit done } post_build() { + local set=$1 spec=$2 + cmd=( rsync -e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no' @@ -59,34 +69,31 @@ post_build() { DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened ;; esac - pushd ${BUILD_SRCDIR_BASE}/default >/dev/null - mkdir -p ${TMPDIR}/empty - "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT} - for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS} ); do - if [ -f $file ]; then - "${cmd[@]}" ${file}* ${DEST_DEFAULT} - fi - done - if [ -f *${DATESTAMP}*.iso ]; then - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT} - elif [ -f *${TIMESTAMP}*.iso ]; then - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT} - fi - popd >/dev/null - pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null - "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED} - for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS} ); do - if [ -f $file ]; then - "${cmd[@]}" $file* ${DEST_HARDENED} - fi - done - if [ -f *${DATESTAMP}*.iso ]; then - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED} - elif [ -f *${TIMESTAMP}*.iso ]; then - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED} - fi - popd >/dev/null + pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit + case ${spec} in + stage3.spec) + "${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + installcd-stage2-minimal.spec) + "${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}" + ;; + i686/stage3.spec) + "${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + esac + popd >/dev/null || exit + + pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit + case ${spec} in + hardened/stage3.spec) + "${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/admincd-stage2.spec) + "${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}" + ;; + esac + popd >/dev/null || exit } # vim:ft=sh: diff --git a/tools/catalyst-auto-x86.conf b/tools/catalyst-auto-x86.conf index 27637e59..e778ce8e 100644 --- a/tools/catalyst-auto-x86.conf +++ b/tools/catalyst-auto-x86.conf @@ -30,7 +30,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]" update_symlinks() { # Symlink the latest stages3 to build from for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do - pushd $d >/dev/null + pushd "${d}" >/dev/null || exit for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do # 20yymmddThhmmssZ # 20yymmddhhmmss @@ -44,11 +44,13 @@ update_symlinks() { <<<"$f") ln -svf "$f" "$of" done - popd >/dev/null + popd >/dev/null || exit done } post_build() { + local set=$1 spec=$2 + cmd=( rsync -e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no' @@ -67,34 +69,31 @@ post_build() { DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened ;; esac - pushd ${BUILD_SRCDIR_BASE}/default >/dev/null - mkdir -p ${TMPDIR}/empty - "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT} - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do - if [ -f $file ]; then - "${cmd[@]}" ${file}* ${DEST_DEFAULT} - fi - done - if [ -f *${DATESTAMP}*.iso ]; then - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT} - elif [ -f *${TIMESTAMP}*.iso ]; then - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT} - fi - popd >/dev/null - pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null - "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED} - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do - if [ -f $file ]; then - "${cmd[@]}" $file* ${DEST_HARDENED} - fi - done - if [ -f *${DATESTAMP}*.iso ]; then - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED} - elif [ -f *${TIMESTAMP}*.iso ]; then - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED} - fi - popd >/dev/null + pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit + case ${spec} in + stage3.spec) + "${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + installcd-stage2-minimal.spec) + "${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}" + ;; + i686/stage3.spec) + "${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" + ;; + esac + popd >/dev/null || exit + + pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit + case ${spec} in + hardened/stage3.spec) + "${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" + ;; + hardened/admincd-stage2.spec) + "${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}" + ;; + esac + popd >/dev/null || exit } # vim:ft=sh: