From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 6A9F5158086 for ; Fri, 12 Nov 2021 18:00:52 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id B038B2BC008; Fri, 12 Nov 2021 18:00:51 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [140.211.166.183]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 927272BC008 for ; Fri, 12 Nov 2021 18:00:51 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 7A10A335D91 for ; Fri, 12 Nov 2021 18:00:50 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 9BEF7163 for ; Fri, 12 Nov 2021 18:00:48 +0000 (UTC) From: "Robin H. Johnson" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Robin H. Johnson" Message-ID: <1636740031.8623ef538cf32aaf4d46982c5e22569ae72ed8b0.robbat2@gentoo> Subject: [gentoo-commits] proj/releng:master commit in: scripts/ X-VCS-Repository: proj/releng X-VCS-Files: scripts/copy_buildsync.sh X-VCS-Directories: scripts/ X-VCS-Committer: robbat2 X-VCS-Committer-Name: Robin H. Johnson X-VCS-Revision: 8623ef538cf32aaf4d46982c5e22569ae72ed8b0 X-VCS-Branch: master Date: Fri, 12 Nov 2021 18:00:48 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 861c7c41-17e3-4d92-bc82-4265aaf37757 X-Archives-Hash: a43a5fc9cffef4b84517d01ba1e717f1 commit: 8623ef538cf32aaf4d46982c5e22569ae72ed8b0 Author: Robin H. Johnson gentoo org> AuthorDate: Fri Nov 12 17:59:08 2021 +0000 Commit: Robin H. Johnson gentoo org> CommitDate: Fri Nov 12 18:00:31 2021 +0000 URL: https://gitweb.gentoo.org/proj/releng.git/commit/?id=8623ef53 scripts/copy_buildsync: dead detection works too well, send to files instead Signed-off-by: Robin H. Johnson gentoo.org> scripts/copy_buildsync.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh index f7f9a3df..0308da39 100755 --- a/scripts/copy_buildsync.sh +++ b/scripts/copy_buildsync.sh @@ -6,6 +6,8 @@ INCOMING_BASE="/release/weekly/builds" OUTGOING_BASE="/release/distfiles/weekly" # Scratch space used when moving files from incoming to outgoing. TMPDIR_BASE="/release/distfiles/tmp/buildsync/partial" +# Keep some records +LOGDIR_BASE="/release/distfiles/tmp/buildsync/logs" ARCHES=( alpha @@ -64,7 +66,7 @@ EOF # Copy artifacts for an arch to the outgoing directory. copy_arch_to_outgoing() { - local ARCH=$1 indir=$2 outdir=$3 tmpdir=$4 + local ARCH=$1 indir=$2 outdir=$3 tmpdir=$4 logdir=$5 local i t rc timestamps if [[ ! -d ${indir} ]]; then @@ -91,6 +93,7 @@ copy_arch_to_outgoing() { "${RSYNC_OPTS[@]}" \ --temp-dir="${tmpdir}" \ --partial-dir="${tmpdir}" \ + --log-file="${logdir}/rsync.log" \ --filter '- **/.*' \ --filter "S *${i}*" \ --filter 'S **/' \ @@ -126,11 +129,12 @@ process_arch() { indir="${INCOMING_BASE}/${ARCH}" outdir="${OUTGOING_BASE}/${ARCH}" tmpdir="${TMPDIR_BASE}/${ARCH}" + logdir="${LOGDIR_BASE}/${ARCH}" - mkdir -p "${tmpdir}" 2>/dev/null + mkdir -p "${tmpdir}" "${logdir}" 2>/dev/null # Sync incoming->outgoing first. - copy_arch_to_outgoing "${ARCH}" "${indir}" "${outdir}" "${tmpdir}" + copy_arch_to_outgoing "${ARCH}" "${indir}" "${outdir}" "${tmpdir}" "${logdir}" # ================================================================ # Build data for revealing latest: @@ -216,10 +220,7 @@ process_arch() { # Find the dead links for cleanup _dead="${tmpdir}"/dead-link find -L $(pwd) -type l >"${_dead}" - if test -s "${_dead}"; then - echo "copy_buildsync: dead links to verify:" 1>&2 - cat "${_dead}" 1>&2 - fi + mv -f "${_dead}" "${logdir}/dead-links.txt" # Find the dead latest txt files _dead="${tmpdir}"/dead-latest @@ -235,6 +236,7 @@ process_arch() { | fgrep -l -f - $f \ | xargs -n1 --no-run-if-empty readlink -f done >"${_dead}" + if test -s "${_dead}"; then echo "copy_buildsync: removing dead latest*txt files:" 1>&2 foreach txtfil in $(cat "${_dead}") ; do @@ -242,11 +244,14 @@ process_arch() { done fi + mv -f "${_dead}" "${logdir}/dead-latest-txt.txt" + # Cleanup tmpdir $DEBUGP rm $VERBOSEP -rf "${tmpdir}" else echo "There was some failure for $ARCH during the weekly sync. Not doing cleanup for fear of dataloss." 1>&2 + echo "See logs in $logdir" 1>&2 fi }