* [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred
@ 2020-05-15 13:07 Ben Kohler
2020-05-15 13:07 ` [gentoo-releng] [PATCH 2/2] catalyst-auto-x86*.conf: " Ben Kohler
2020-05-15 16:32 ` [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: " Matt Turner
0 siblings, 2 replies; 4+ messages in thread
From: Ben Kohler @ 2020-05-15 13:07 UTC (permalink / raw
To: gentoo-releng
Previously when we were copying locally there was no (major) problem
with wildcarding a long list of files to be transferred after every
single spec finishes. Now that we are transferring to a remote host,
this results in a lot of wasted transfer time & bandwidth.
I have adjusted the post_build function to only transfer the new files
known to be produced by each spec (which has files meant to be
published).
Signed-off-by: Ben Kohler <bkohler@gentoo.org>
---
tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++-------
tools/catalyst-auto-amd64.conf | 86 +++++++++++++-------
2 files changed, 113 insertions(+), 60 deletions(-)
diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf
index e7b1ea79..2aa91a03 100644
--- a/tools/catalyst-auto-amd64-experimental.conf
+++ b/tools/catalyst-auto-amd64-experimental.conf
@@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec"
SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec"
+
SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec"
SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec"
SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec"
@@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
update_symlinks() {
# Symlink the latest stages3 to build from
for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
- pushd $d >/dev/null
+ pushd "${d}" >/dev/null || exit
for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
# 20yymmddThhmmssZ
# 20yymmddhhmmss
@@ -57,11 +58,13 @@ update_symlinks() {
<<<"$f")
ln -svf "$f" "$of"
done
- popd >/dev/null
+ popd >/dev/null || exit
done
}
post_build() {
+ local set=$1 spec=$2
+
cmd=(
rsync
-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -80,34 +83,58 @@ post_build() {
DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
;;
esac
- pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
- mkdir -p ${TMPDIR}/empty
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" ${file}* ${DEST_DEFAULT}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
- fi
- popd >/dev/null
-
- pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" $file* ${DEST_HARDENED}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
- fi
- popd >/dev/null
+
+ pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+ case ${spec} in
+ stage3.spec)
+ "${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ installcd-stage2-minimal.spec)
+ "${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+ ;;
+ stage4-minimal.spec)
+ "${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ stage3-nomultilib.spec)
+ "${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ stage4-nomultilib-minimal.spec)
+ "${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ stage3-x32.spec)
+ "${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ esac
+ popd >/dev/null || exit
+
+ pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+ case ${spec} in
+ hardened/stage3.spec)
+ "${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/admincd-stage2.spec)
+ "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+ ;;
+ hardened/stage4-minimal.spec)
+ "${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/stage3-nomultilib.spec)
+ "${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/stage4-nomultilib-minimal.spec)
+ "${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/stage3-selinux.spec)
+ "${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/admincd-stage2-selinux.spec)
+ "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+ ;;
+ hardened/stage3-selinux-nomultilib.spec)
+ "${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ esac
+ popd >/dev/null || exit
}
# vim:ft=sh:
diff --git a/tools/catalyst-auto-amd64.conf b/tools/catalyst-auto-amd64.conf
index 0b7e9b2f..585c22c6 100644
--- a/tools/catalyst-auto-amd64.conf
+++ b/tools/catalyst-auto-amd64.conf
@@ -44,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
update_symlinks() {
# Symlink the latest stages3 to build from
for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
- pushd $d >/dev/null
+ pushd "${d}" >/dev/null || exit
for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
# 20yymmddThhmmssZ
# 20yymmddhhmmss
@@ -58,11 +58,13 @@ update_symlinks() {
<<<"$f")
ln -svf "$f" "$of"
done
- popd >/dev/null
+ popd >/dev/null || exit
done
}
post_build() {
+ local set=$1 spec=$2
+
cmd=(
rsync
-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -81,34 +83,58 @@ post_build() {
DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
;;
esac
- pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
- mkdir -p ${TMPDIR}/empty
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" ${file}* ${DEST_DEFAULT}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
- fi
- popd >/dev/null
-
- pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" $file* ${DEST_HARDENED}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
- fi
- popd >/dev/null
+
+ pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+ case ${spec} in
+ stage3.spec)
+ "${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ installcd-stage2-minimal.spec)
+ "${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+ ;;
+ stage4-minimal.spec)
+ "${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ stage3-nomultilib.spec)
+ "${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ stage4-nomultilib-minimal.spec)
+ "${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ stage3-x32.spec)
+ "${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ esac
+ popd >/dev/null || exit
+
+ pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+ case ${spec} in
+ hardened/stage3.spec)
+ "${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/admincd-stage2.spec)
+ "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+ ;;
+ hardened/stage4-minimal.spec)
+ "${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/stage3-nomultilib.spec)
+ "${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/stage4-nomultilib-minimal.spec)
+ "${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/stage3-selinux.spec)
+ "${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/admincd-stage2-selinux.spec)
+ "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+ ;;
+ hardened/stage3-selinux-nomultilib.spec)
+ "${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ esac
+ popd >/dev/null || exit
}
# vim:ft=sh:
--
2.26.2
^ permalink raw reply related [flat|nested] 4+ messages in thread* [gentoo-releng] [PATCH 2/2] catalyst-auto-x86*.conf: explicitly list files to be transferred
2020-05-15 13:07 [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred Ben Kohler
@ 2020-05-15 13:07 ` Ben Kohler
2020-05-15 16:32 ` [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: " Matt Turner
1 sibling, 0 replies; 4+ messages in thread
From: Ben Kohler @ 2020-05-15 13:07 UTC (permalink / raw
To: gentoo-releng
Previously when we were copying locally there was no (major) problem
with wildcarding a long list of files to be transferred after every
single spec finishes. Now that we are transferring to a remote host,
this results in a lot of wasted transfer time & bandwidth.
I have adjusted the post_build function to only transfer the new files
known to be produced by each spec (which has files meant to be
published).
Signed-off-by: Ben Kohler <bkohler@gentoo.org>
---
tools/catalyst-auto-x86-experimental.conf | 71 +++++++++++++----------
tools/catalyst-auto-x86.conf | 57 +++++++++---------
2 files changed, 67 insertions(+), 61 deletions(-)
diff --git a/tools/catalyst-auto-x86-experimental.conf b/tools/catalyst-auto-x86-experimental.conf
index 3fd60287..061cab92 100644
--- a/tools/catalyst-auto-x86-experimental.conf
+++ b/tools/catalyst-auto-x86-experimental.conf
@@ -30,17 +30,27 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
update_symlinks() {
# Symlink the latest stages3 to build from
for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
- pushd $d >/dev/null
+ pushd "${d}" >/dev/null || exit
for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
- of=${f/20[0-9][0-9].[0-9]/latest} # for 20XX.Y stuff
- of=${of/20[0-9][0-9][0-1][0-9][0-9][0-9]/latest} # for 20YYMMDD stuff
- ln -sf $f $of
+ # 20yymmddThhmmssZ
+ # 20yymmddhhmmss
+ # 20yymmdd
+ # 20yy.n
+ of=$(perl -p \
+ -e 's/20\d{6}T\d{6}Z/latest/g;' \
+ -e 's/20\d{6}\d{6}/latest/g;' \
+ -e 's/20\d{6}/latest/g;' \
+ -e 's/20\d{2}\.\d{2}/latest/g;' \
+ <<<"$f")
+ ln -svf "$f" "$of"
done
- popd >/dev/null
+ popd >/dev/null || exit
done
}
post_build() {
+ local set=$1 spec=$2
+
cmd=(
rsync
-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -59,34 +69,31 @@ post_build() {
DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
;;
esac
- pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
- mkdir -p ${TMPDIR}/empty
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
- for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" ${file}* ${DEST_DEFAULT}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
- fi
- popd >/dev/null
- pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
- for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" $file* ${DEST_HARDENED}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
- fi
- popd >/dev/null
+ pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+ case ${spec} in
+ stage3.spec)
+ "${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ installcd-stage2-minimal.spec)
+ "${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+ ;;
+ i686/stage3.spec)
+ "${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ esac
+ popd >/dev/null || exit
+
+ pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+ case ${spec} in
+ hardened/stage3.spec)
+ "${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/admincd-stage2.spec)
+ "${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+ ;;
+ esac
+ popd >/dev/null || exit
}
# vim:ft=sh:
diff --git a/tools/catalyst-auto-x86.conf b/tools/catalyst-auto-x86.conf
index 27637e59..e778ce8e 100644
--- a/tools/catalyst-auto-x86.conf
+++ b/tools/catalyst-auto-x86.conf
@@ -30,7 +30,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
update_symlinks() {
# Symlink the latest stages3 to build from
for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
- pushd $d >/dev/null
+ pushd "${d}" >/dev/null || exit
for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
# 20yymmddThhmmssZ
# 20yymmddhhmmss
@@ -44,11 +44,13 @@ update_symlinks() {
<<<"$f")
ln -svf "$f" "$of"
done
- popd >/dev/null
+ popd >/dev/null || exit
done
}
post_build() {
+ local set=$1 spec=$2
+
cmd=(
rsync
-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -67,34 +69,31 @@ post_build() {
DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
;;
esac
- pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
- mkdir -p ${TMPDIR}/empty
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" ${file}* ${DEST_DEFAULT}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
- fi
- popd >/dev/null
- pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" $file* ${DEST_HARDENED}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
- fi
- popd >/dev/null
+ pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+ case ${spec} in
+ stage3.spec)
+ "${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ installcd-stage2-minimal.spec)
+ "${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+ ;;
+ i686/stage3.spec)
+ "${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ esac
+ popd >/dev/null || exit
+
+ pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+ case ${spec} in
+ hardened/stage3.spec)
+ "${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+ ;;
+ hardened/admincd-stage2.spec)
+ "${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+ ;;
+ esac
+ popd >/dev/null || exit
}
# vim:ft=sh:
--
2.26.2
^ permalink raw reply related [flat|nested] 4+ messages in thread* Re: [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred
2020-05-15 13:07 [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred Ben Kohler
2020-05-15 13:07 ` [gentoo-releng] [PATCH 2/2] catalyst-auto-x86*.conf: " Ben Kohler
@ 2020-05-15 16:32 ` Matt Turner
2020-05-15 16:45 ` Kohler, Benjamin
1 sibling, 1 reply; 4+ messages in thread
From: Matt Turner @ 2020-05-15 16:32 UTC (permalink / raw
To: gentoo-releng
On Fri, May 15, 2020 at 6:07 AM Ben Kohler <bkohler@gentoo.org> wrote:
>
> Previously when we were copying locally there was no (major) problem
> with wildcarding a long list of files to be transferred after every
> single spec finishes. Now that we are transferring to a remote host,
> this results in a lot of wasted transfer time & bandwidth.
>
> I have adjusted the post_build function to only transfer the new files
> known to be produced by each spec (which has files meant to be
> published).
Thanks Ben. Both patches look good to me. Thanks for working on this.
Just a couple small comments inline.
> Signed-off-by: Ben Kohler <bkohler@gentoo.org>
> ---
> tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++-------
Do we actually use this file in production? It looks like the main
difference is TYPE="experimental" vs TYPE="auto" and I don't see TYPE
used anywhere. Can we just remove this file?
> tools/catalyst-auto-amd64.conf | 86 +++++++++++++-------
> 2 files changed, 113 insertions(+), 60 deletions(-)
>
> diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf
> index e7b1ea79..2aa91a03 100644
> --- a/tools/catalyst-auto-amd64-experimental.conf
> +++ b/tools/catalyst-auto-amd64-experimental.conf
> @@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec"
>
> SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec"
>
> +
Stray newline, but I see that this is to reduce the differences
between the experimental and non-experimental versions of the file. I
might remove the newline from the other instead, but I don't feel
strongly.
> SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec"
> SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec"
> SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec"
> @@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
> update_symlinks() {
> # Symlink the latest stages3 to build from
> for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
> - pushd $d >/dev/null
> + pushd "${d}" >/dev/null || exit
> for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
> # 20yymmddThhmmssZ
> # 20yymmddhhmmss
> @@ -57,11 +58,13 @@ update_symlinks() {
> <<<"$f")
> ln -svf "$f" "$of"
> done
> - popd >/dev/null
> + popd >/dev/null || exit
> done
> }
>
> post_build() {
> + local set=$1 spec=$2
> +
> cmd=(
> rsync
This would be a good opportunity to create and use an upload()
function like the other conf files have.
^ permalink raw reply [flat|nested] 4+ messages in thread* Re: [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred
2020-05-15 16:32 ` [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: " Matt Turner
@ 2020-05-15 16:45 ` Kohler, Benjamin
0 siblings, 0 replies; 4+ messages in thread
From: Kohler, Benjamin @ 2020-05-15 16:45 UTC (permalink / raw
To: gentoo-releng, mattst88
On 5/15/2020 11:32 AM, Matt Turner wrote:
>> Signed-off-by: Ben Kohler <bkohler@gentoo.org>
>> ---
>> tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++-------
> Do we actually use this file in production? It looks like the main
> difference is TYPE="experimental" vs TYPE="auto" and I don't see TYPE
> used anywhere. Can we just remove this file?
I think it can probably be removed, but I will wait a bit for comments
and look into removing it later (same for x86).
>> tools/catalyst-auto-amd64.conf | 86 +++++++++++++-------
>> 2 files changed, 113 insertions(+), 60 deletions(-)
>>
>> diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf
>> index e7b1ea79..2aa91a03 100644
>> --- a/tools/catalyst-auto-amd64-experimental.conf
>> +++ b/tools/catalyst-auto-amd64-experimental.conf
>> @@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec"
>>
>> SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec"
>>
>> +
> Stray newline, but I see that this is to reduce the differences
> between the experimental and non-experimental versions of the file. I
> might remove the newline from the other instead, but I don't feel
> strongly.
Yeah this was just an artifact of trying to unify the 2 files, I will
remove it along with the next other fixes.
>> SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec"
>> SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec"
>> SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec"
>> @@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
>> update_symlinks() {
>> # Symlink the latest stages3 to build from
>> for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
>> - pushd $d >/dev/null
>> + pushd "${d}" >/dev/null || exit
>> for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
>> # 20yymmddThhmmssZ
>> # 20yymmddhhmmss
>> @@ -57,11 +58,13 @@ update_symlinks() {
>> <<<"$f")
>> ln -svf "$f" "$of"
>> done
>> - popd >/dev/null
>> + popd >/dev/null || exit
>> done
>> }
>>
>> post_build() {
>> + local set=$1 spec=$2
>> +
>> cmd=(
>> rsync
> This would be a good opportunity to create and use an upload()
> function like the other conf files have.
Yes I would like to do that too, I will follow up with another commit
doing that. I didn't want to cram too many changes into this first patch.
Thanks for the review.
-Ben
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2020-05-15 16:45 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-05-15 13:07 [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred Ben Kohler
2020-05-15 13:07 ` [gentoo-releng] [PATCH 2/2] catalyst-auto-x86*.conf: " Ben Kohler
2020-05-15 16:32 ` [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: " Matt Turner
2020-05-15 16:45 ` Kohler, Benjamin
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox