[gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred

[gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred

[Ben Kohler]

Previously when we were copying locally there was no (major) problem
with wildcarding a long list of files to be transferred after every
single spec finishes.  Now that we are transferring to a remote host,
this results in a lot of wasted transfer time & bandwidth.

I have adjusted the post_build function to only transfer the new files
known to be produced by each spec (which has files meant to be
published).

Signed-off-by: Ben Kohler <bkohler@gentoo.org>
---
 tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++-------
 tools/catalyst-auto-amd64.conf              | 86 +++++++++++++-------
 2 files changed, 113 insertions(+), 60 deletions(-)

diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf
index e7b1ea79..2aa91a03 100644
--- a/tools/catalyst-auto-amd64-experimental.conf
+++ b/tools/catalyst-auto-amd64-experimental.conf
@@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec"
 
 SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec"
 
+
 SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec"
 SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec"
 SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec"
@@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
 update_symlinks() {
 	# Symlink the latest stages3 to build from
 	for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
-		pushd $d >/dev/null
+		pushd "${d}" >/dev/null || exit
 		for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
 			# 20yymmddThhmmssZ
 			# 20yymmddhhmmss
@@ -57,11 +58,13 @@ update_symlinks() {
 				<<<"$f")
 			ln -svf "$f" "$of"
 		done
-		popd >/dev/null
+		popd >/dev/null || exit
 	done
 }
 
 post_build() {
+	local set=$1 spec=$2
+
 	cmd=(
 		rsync
 		-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -80,34 +83,58 @@ post_build() {
 			DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
 			;;
 	esac
-	pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
-	mkdir -p ${TMPDIR}/empty
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
-	for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" ${file}* ${DEST_DEFAULT}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
-	fi
-	popd >/dev/null
-
-	pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
-	for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" $file* ${DEST_HARDENED}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
-	fi
-	popd >/dev/null
+
+	pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+	case ${spec} in
+	stage3.spec)
+		"${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	installcd-stage2-minimal.spec)
+		"${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+		;;
+	stage4-minimal.spec)
+		"${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	stage3-nomultilib.spec)
+		"${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	stage4-nomultilib-minimal.spec)
+		"${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	stage3-x32.spec)
+		"${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	esac
+	popd >/dev/null || exit
+
+	pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+	case ${spec} in
+	hardened/stage3.spec)
+		"${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/admincd-stage2.spec)
+		"${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+		;;
+	hardened/stage4-minimal.spec)
+		"${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/stage3-nomultilib.spec)
+		"${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/stage4-nomultilib-minimal.spec)
+		"${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/stage3-selinux.spec)
+		"${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/admincd-stage2-selinux.spec)
+		"${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+		;;
+	hardened/stage3-selinux-nomultilib.spec)
+		"${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	esac
+	popd >/dev/null || exit
 }
 
 # vim:ft=sh:
diff --git a/tools/catalyst-auto-amd64.conf b/tools/catalyst-auto-amd64.conf
index 0b7e9b2f..585c22c6 100644
--- a/tools/catalyst-auto-amd64.conf
+++ b/tools/catalyst-auto-amd64.conf
@@ -44,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
 update_symlinks() {
 	# Symlink the latest stages3 to build from
 	for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
-		pushd $d >/dev/null
+		pushd "${d}" >/dev/null || exit
 		for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
 			# 20yymmddThhmmssZ
 			# 20yymmddhhmmss
@@ -58,11 +58,13 @@ update_symlinks() {
 				<<<"$f")
 			ln -svf "$f" "$of"
 		done
-		popd >/dev/null
+		popd >/dev/null || exit
 	done
 }
 
 post_build() {
+	local set=$1 spec=$2
+
 	cmd=(
 		rsync
 		-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -81,34 +83,58 @@ post_build() {
 			DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
 			;;
 	esac
-	pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
-	mkdir -p ${TMPDIR}/empty
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
-	for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" ${file}* ${DEST_DEFAULT}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
-	fi
-	popd >/dev/null
-
-	pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
-	for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" $file* ${DEST_HARDENED}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
-	fi
-	popd >/dev/null
+
+	pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+	case ${spec} in
+	stage3.spec)
+		"${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	installcd-stage2-minimal.spec)
+		"${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+		;;
+	stage4-minimal.spec)
+		"${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	stage3-nomultilib.spec)
+		"${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	stage4-nomultilib-minimal.spec)
+		"${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	stage3-x32.spec)
+		"${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	esac
+	popd >/dev/null || exit
+
+	pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+	case ${spec} in
+	hardened/stage3.spec)
+		"${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/admincd-stage2.spec)
+		"${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+		;;
+	hardened/stage4-minimal.spec)
+		"${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/stage3-nomultilib.spec)
+		"${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/stage4-nomultilib-minimal.spec)
+		"${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/stage3-selinux.spec)
+		"${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/admincd-stage2-selinux.spec)
+		"${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+		;;
+	hardened/stage3-selinux-nomultilib.spec)
+		"${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	esac
+	popd >/dev/null || exit
 }
 
 # vim:ft=sh:
-- 
2.26.2

[gentoo-releng] [PATCH 2/2] catalyst-auto-x86*.conf: explicitly list files to be transferred

[Ben Kohler]

Previously when we were copying locally there was no (major) problem
with wildcarding a long list of files to be transferred after every
single spec finishes.  Now that we are transferring to a remote host,
this results in a lot of wasted transfer time & bandwidth.

I have adjusted the post_build function to only transfer the new files
known to be produced by each spec (which has files meant to be
published).

Signed-off-by: Ben Kohler <bkohler@gentoo.org>
---
 tools/catalyst-auto-x86-experimental.conf | 71 +++++++++++++----------
 tools/catalyst-auto-x86.conf              | 57 +++++++++---------
 2 files changed, 67 insertions(+), 61 deletions(-)

diff --git a/tools/catalyst-auto-x86-experimental.conf b/tools/catalyst-auto-x86-experimental.conf
index 3fd60287..061cab92 100644
--- a/tools/catalyst-auto-x86-experimental.conf
+++ b/tools/catalyst-auto-x86-experimental.conf
@@ -30,17 +30,27 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
 update_symlinks() {
 	# Symlink the latest stages3 to build from
 	for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
-		pushd $d >/dev/null
+		pushd "${d}" >/dev/null || exit
 		for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
-			of=${f/20[0-9][0-9].[0-9]/latest} # for 20XX.Y stuff
-			of=${of/20[0-9][0-9][0-1][0-9][0-9][0-9]/latest} # for 20YYMMDD stuff
-			ln -sf $f $of
+			# 20yymmddThhmmssZ
+			# 20yymmddhhmmss
+			# 20yymmdd
+			# 20yy.n
+			of=$(perl -p \
+				-e 's/20\d{6}T\d{6}Z/latest/g;' \
+				-e 's/20\d{6}\d{6}/latest/g;' \
+				-e 's/20\d{6}/latest/g;' \
+				-e 's/20\d{2}\.\d{2}/latest/g;' \
+				<<<"$f")
+			ln -svf "$f" "$of"
 		done
-		popd >/dev/null
+		popd >/dev/null || exit
 	done
 }
 
 post_build() {
+	local set=$1 spec=$2
+
 	cmd=(
 		rsync
 		-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -59,34 +69,31 @@ post_build() {
 			DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
 			;;
 	esac
-	pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
-	mkdir -p ${TMPDIR}/empty
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
-	for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" ${file}* ${DEST_DEFAULT}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
-	fi
-	popd >/dev/null
 
-	pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
-	for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" $file* ${DEST_HARDENED}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
-	fi
-	popd >/dev/null
+	pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+	case ${spec} in
+	stage3.spec)
+		"${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	installcd-stage2-minimal.spec)
+		"${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+		;;
+	i686/stage3.spec)
+		"${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	esac
+	popd >/dev/null || exit
+
+	pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+	case ${spec} in
+	hardened/stage3.spec)
+		"${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/admincd-stage2.spec)
+		"${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+		;;
+	esac
+	popd >/dev/null || exit
 }
 
 # vim:ft=sh:
diff --git a/tools/catalyst-auto-x86.conf b/tools/catalyst-auto-x86.conf
index 27637e59..e778ce8e 100644
--- a/tools/catalyst-auto-x86.conf
+++ b/tools/catalyst-auto-x86.conf
@@ -30,7 +30,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
 update_symlinks() {
 	# Symlink the latest stages3 to build from
 	for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
-		pushd $d >/dev/null
+		pushd "${d}" >/dev/null || exit
 		for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
 			# 20yymmddThhmmssZ
 			# 20yymmddhhmmss
@@ -44,11 +44,13 @@ update_symlinks() {
 				<<<"$f")
 			ln -svf "$f" "$of"
 		done
-		popd >/dev/null
+		popd >/dev/null || exit
 	done
 }
 
 post_build() {
+	local set=$1 spec=$2
+
 	cmd=(
 		rsync
 		-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -67,34 +69,31 @@ post_build() {
 			DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
 			;;
 	esac
-	pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
-	mkdir -p ${TMPDIR}/empty
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
-	for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" ${file}* ${DEST_DEFAULT}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
-	fi
-	popd >/dev/null
 
-	pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
-	for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" $file* ${DEST_HARDENED}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
-	fi
-	popd >/dev/null
+	pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+	case ${spec} in
+	stage3.spec)
+		"${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	installcd-stage2-minimal.spec)
+		"${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+		;;
+	i686/stage3.spec)
+		"${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	esac
+	popd >/dev/null || exit
+
+	pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+	case ${spec} in
+	hardened/stage3.spec)
+		"${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/admincd-stage2.spec)
+		"${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+		;;
+	esac
+	popd >/dev/null || exit
 }
 
 # vim:ft=sh:
-- 
2.26.2

Re: [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred

[Matt Turner]

On Fri, May 15, 2020 at 6:07 AM Ben Kohler <bkohler@gentoo.org> wrote:
>
> Previously when we were copying locally there was no (major) problem
> with wildcarding a long list of files to be transferred after every
> single spec finishes.  Now that we are transferring to a remote host,
> this results in a lot of wasted transfer time & bandwidth.
>
> I have adjusted the post_build function to only transfer the new files
> known to be produced by each spec (which has files meant to be
> published).

Thanks Ben. Both patches look good to me. Thanks for working on this.

Just a couple small comments inline.

> Signed-off-by: Ben Kohler <bkohler@gentoo.org>
> ---
>  tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++-------

Do we actually use this file in production? It looks like the main
difference is TYPE="experimental" vs TYPE="auto" and I don't see TYPE
used anywhere. Can we just remove this file?

>  tools/catalyst-auto-amd64.conf              | 86 +++++++++++++-------
>  2 files changed, 113 insertions(+), 60 deletions(-)
>
> diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf
> index e7b1ea79..2aa91a03 100644
> --- a/tools/catalyst-auto-amd64-experimental.conf
> +++ b/tools/catalyst-auto-amd64-experimental.conf
> @@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec"
>
>  SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec"
>
> +

Stray newline, but I see that this is to reduce the differences
between the experimental and non-experimental versions of the file. I
might remove the newline from the other instead, but I don't feel
strongly.

>  SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec"
>  SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec"
>  SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec"
> @@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
>  update_symlinks() {
>         # Symlink the latest stages3 to build from
>         for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
> -               pushd $d >/dev/null
> +               pushd "${d}" >/dev/null || exit
>                 for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
>                         # 20yymmddThhmmssZ
>                         # 20yymmddhhmmss
> @@ -57,11 +58,13 @@ update_symlinks() {
>                                 <<<"$f")
>                         ln -svf "$f" "$of"
>                 done
> -               popd >/dev/null
> +               popd >/dev/null || exit
>         done
>  }
>
>  post_build() {
> +       local set=$1 spec=$2
> +
>         cmd=(
>                 rsync

This would be a good opportunity to create and use an upload()
function like the other conf files have.

Re: [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred

[Kohler, Benjamin]

On 5/15/2020 11:32 AM, Matt Turner wrote:
>> Signed-off-by: Ben Kohler <bkohler@gentoo.org>
>> ---
>>   tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++-------
> Do we actually use this file in production? It looks like the main
> difference is TYPE="experimental" vs TYPE="auto" and I don't see TYPE
> used anywhere. Can we just remove this file?
I think it can probably be removed, but I will wait a bit for comments 
and look into removing it later (same for x86).
>>   tools/catalyst-auto-amd64.conf              | 86 +++++++++++++-------
>>   2 files changed, 113 insertions(+), 60 deletions(-)
>>
>> diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf
>> index e7b1ea79..2aa91a03 100644
>> --- a/tools/catalyst-auto-amd64-experimental.conf
>> +++ b/tools/catalyst-auto-amd64-experimental.conf
>> @@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec"
>>
>>   SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec"
>>
>> +
> Stray newline, but I see that this is to reduce the differences
> between the experimental and non-experimental versions of the file. I
> might remove the newline from the other instead, but I don't feel
> strongly.
Yeah this was just an artifact of trying to unify the 2 files, I will 
remove it along with the next other fixes.
>>   SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec"
>>   SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec"
>>   SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec"
>> @@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
>>   update_symlinks() {
>>          # Symlink the latest stages3 to build from
>>          for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
>> -               pushd $d >/dev/null
>> +               pushd "${d}" >/dev/null || exit
>>                  for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
>>                          # 20yymmddThhmmssZ
>>                          # 20yymmddhhmmss
>> @@ -57,11 +58,13 @@ update_symlinks() {
>>                                  <<<"$f")
>>                          ln -svf "$f" "$of"
>>                  done
>> -               popd >/dev/null
>> +               popd >/dev/null || exit
>>          done
>>   }
>>
>>   post_build() {
>> +       local set=$1 spec=$2
>> +
>>          cmd=(
>>                  rsync
> This would be a good opportunity to create and use an upload()
> function like the other conf files have.

Yes I would like to do that too, I will follow up with another commit 
doing that.  I didn't want to cram too many changes into this first patch.

Thanks for the review.

-Ben