* [gentoo-commits] repo/proj/prefix:master commit in: scripts/, scripts/rsync-generation/
@ 2023-02-24 18:52 Sam James
0 siblings, 0 replies; 3+ messages in thread
From: Sam James @ 2023-02-24 18:52 UTC (permalink / raw
To: gentoo-commits
commit: 82a3b8ba04a52268330028ea239a91bbdeb55890
Author: Peter Foley <pefoley2 <AT> pefoley <DOT> com>
AuthorDate: Fri Feb 24 03:55:49 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Feb 24 18:51:38 2023 +0000
URL: https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=82a3b8ba
scripts: Resolve shellcheck errors
The action is failing at head, which makes it less than useful for
evaluating new changes.
e.g.
Error: ./scripts/rsync-generation/refresh-mirror.sh:44:2: error: Shells disambiguate (( differently or not at all. For subshell, add spaces around ( . For ((, fix parsing errors. [SC1105]
Error: ./scripts/rsync-generation/refresh-mirror.sh:44:3: error: Shells disambiguate (( differently or not at all. For subshell, add spaces around ( . For ((, fix parsing errors. [SC1105]
Error: ./scripts/bootstrap-prefix.sh:5:20: error: SIGKILL/SIGSTOP can not be trapped. [SC2173]
Error: ./scripts/bootstrap-prefix.sh:1669:19: error: Double quote array expansions to avoid re-splitting elements. [SC2068]
Error: ./scripts/bootstrap-prefix.sh:1964:19: error: Double quote array expansions to avoid re-splitting elements. [SC2068]
Error: ./scripts/bootstrap-prefix.sh:2286:19: error: Double quote array expansions to avoid re-splitting elements. [SC2068]
Error: ./scripts/bootstrap-prefix.sh:3194:11: error: -e doesn't work with globs. Use a for loop. [SC2144]
Signed-off-by: Peter Foley <pefoley2 <AT> pefoley.com>
Closes: https://github.com/gentoo/prefix/pull/19
Signed-off-by: Sam James <sam <AT> gentoo.org>
scripts/bootstrap-prefix.sh | 12 +++++++-----
scripts/rsync-generation/refresh-mirror.sh | 2 +-
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/scripts/bootstrap-prefix.sh b/scripts/bootstrap-prefix.sh
index 1ba5108ce1..ce939bfaf5 100755
--- a/scripts/bootstrap-prefix.sh
+++ b/scripts/bootstrap-prefix.sh
@@ -2,7 +2,7 @@
# Copyright 2006-2022 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-trap 'exit 1' TERM KILL INT QUIT ABRT
+trap 'exit 1' TERM INT QUIT ABRT
# RAP (libc) mode is triggered on Linux kernel and glibc.
is-rap() { [[ ${PREFIX_DISABLE_RAP} != "yes" && ${CHOST} = *linux-gnu* ]]; }
@@ -1666,7 +1666,7 @@ bootstrap_stage1() {
}
bootstrap_stage1_log() {
- bootstrap_stage1 ${@} 2>&1 | tee -a ${ROOT}/stage1.log
+ bootstrap_stage1 "${@}" 2>&1 | tee -a ${ROOT}/stage1.log
local ret=${PIPESTATUS[0]}
[[ ${ret} == 0 ]] && touch ${ROOT}/.stage1-finished
return ${ret}
@@ -1961,7 +1961,7 @@ bootstrap_stage2() {
}
bootstrap_stage2_log() {
- bootstrap_stage2 ${@} 2>&1 | tee -a ${ROOT}/stage2.log
+ bootstrap_stage2 "${@}" 2>&1 | tee -a ${ROOT}/stage2.log
local ret=${PIPESTATUS[0]}
[[ ${ret} == 0 ]] && touch "${ROOT}/.stage2-finished"
return ${ret}
@@ -2283,7 +2283,7 @@ bootstrap_stage3() {
}
bootstrap_stage3_log() {
- bootstrap_stage3 ${@} 2>&1 | tee -a ${ROOT}/stage3.log
+ bootstrap_stage3 "${@}" 2>&1 | tee -a ${ROOT}/stage3.log
local ret=${PIPESTATUS[0]}
[[ ${ret} == 0 ]] && touch "${ROOT}/.stage3-finished"
return ${ret}
@@ -3191,7 +3191,9 @@ if [[ -z ${CHOST} ]]; then
case `uname -s` in
Linux)
plt="gnu"
- [[ -e /lib/ld-musl-*.so.1 ]] && plt="musl"
+ for f in /lib/ld-musl-*.so.1; do
+ [[ -e "$f" ]] && plt="musl"
+ done
sfx="unknown-linux-${plt}"
case `uname -m` in
ppc*)
diff --git a/scripts/rsync-generation/refresh-mirror.sh b/scripts/rsync-generation/refresh-mirror.sh
index 072c82e171..6e2baffaa4 100755
--- a/scripts/rsync-generation/refresh-mirror.sh
+++ b/scripts/rsync-generation/refresh-mirror.sh
@@ -41,7 +41,7 @@ else
}
# get a free filedescriptor in FD
exec {FD}>/tmp/rsync-master-busy
- (((genandpush | tee -a "${LOGFILE}") {FD}>&1 1>&2 2>&${FD} \
+ ( ( (genandpush | tee -a "${LOGFILE}") {FD}>&1 1>&2 2>&${FD} \
| tee -a "${LOGFILE}") 2> /dev/null)
echo "generation done $(date)" >> ${LOGFILE}
exec {FD}>&-
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] repo/proj/prefix:master commit in: scripts/, scripts/rsync-generation/
@ 2023-09-03 7:36 Fabian Groffen
0 siblings, 0 replies; 3+ messages in thread
From: Fabian Groffen @ 2023-09-03 7:36 UTC (permalink / raw
To: gentoo-commits
commit: 69b7e2d62be6532978740e3e18b72b8fa8d0788d
Author: Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 3 07:31:57 2023 +0000
Commit: Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Sep 3 07:31:57 2023 +0000
URL: https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=69b7e2d6
scripts: make bootstrap snapshot available on rsync slaves
In bug #913415 a bunch of confusing and wrong things came out, and in an
attempt to fix this, the historical "prefix-overlay" fetched from dev
was replaced with ordinary "portage" from rsync snapshots dir. However,
this relies on the prefix mirror to provide the file, which can easily
be overridden by the user.
So retrieve the snapshot in use and retain it on the rsync slaves to
make this in the very least less confusing for those following the
locations, and ensure it works for people with custom mirror setups.
Bug: https://bugs.gentoo.org/913415
Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>
scripts/bootstrap-prefix.sh | 3 +++
scripts/rsync-generation/mksnapshot.sh | 36 ++++++++++++++++++++++------------
2 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/scripts/bootstrap-prefix.sh b/scripts/bootstrap-prefix.sh
index 142ef8213e..735e896dd0 100755
--- a/scripts/bootstrap-prefix.sh
+++ b/scripts/bootstrap-prefix.sh
@@ -562,6 +562,9 @@ do_tree() {
}
bootstrap_tree() {
+ # retain this comment and the line below to
+ # keep this snapshot around in the snapshots
+ # MKSNAPSHOT-ANCHOR -- directory of rsync slaves
local PV="20230709"
# RAP uses the latest gentoo main repo snapshot to bootstrap.
diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 5c13154cf2..d53d135693 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -4,39 +4,49 @@ SCRIPTLOC="$(readlink -f "${BASH_SOURCE[0]%/*}")"
# for gpg's keys
export HOME="${SCRIPTLOC}/misc"
-cd /export/scratch/home/gentoo/snapshots
+cd /export/gentoo-rsync/rsync0-prefix-tree/snapshots || exit 1
TODAY=$(date +%s)
YESTERDAY=$((TODAY - 86400))
-RSYNCTREE=${PWD%/*}/prefix-rsync-tree
+RSYNCTREE=${PWD%/*}
SNAME=${PWD}/portage-$(date +%Y%m%d -d @${YESTERDAY}).tar
-TMPDIR=${PWD%/*}/tmp-prefix-snapshot
+TMPDIR=${PWD}/tmp-prefix-snapshot
# clean up
find . -maxdepth 2 -daystart -ctime +4 -type f | xargs --no-run-if-empty rm
+# pull in active snapshot
+BOOTSTRAP_SNAPSHOT=$( \
+ grep -A1 MKSNAPSHOT-ANCHOR "${RSYNCTREE}"/scripts/bootstrap-prefix.sh | \
+ sed -n 's/^.*PV="\([0-9]\+\)"\s*$/portage-\1.tar.bz2/p' \
+)
+curl -s -L "https://distfiles.prefix.bitzolder.nl/prefix/distfiles/${BOOTSTRAP_SNAPSHOT}" > "${BOOTSTRAP_SNAPSHOT}"
+
rm -Rf ${TMPDIR}
mkdir -p ${TMPDIR}
# quickly take a snapshot, such that we get a consistent image
pushd ${RSYNCTREE} > /dev/null
-tar -cf ${SNAME} *
+tar -cf ${SNAME} --exclude=snapshots * || exit 1
popd > /dev/null
# now revamp it such that it's in a directory "portage"
+rm -Rf ${TMPDIR}
+mkdir -p ${TMPDIR}
pushd ${TMPDIR} > /dev/null
mkdir portage
tar -xf ${SNAME} -C portage/
-tar -cf ${SNAME} portage/
+tar --numeric-owner --format=posix --hard-dereference -cf ${SNAME} portage/
popd > /dev/null
rm -Rf ${TMPDIR}
-# be nice, and this machine only has one CPU (so can't parallelise)
-nice -n19 bzip2 -c -9 ${SNAME} > ${SNAME}.bz2
-nice -n19 xz -c -9 ${SNAME} > ${SNAME}.xz
-nice -n19 gzip -c -9 ${SNAME} > ${SNAME}.gz
+# be nice
+nice -n19 bzip2 -c -9 ${SNAME} > ${SNAME}.bz2 &
+nice -n19 xz -c -9 ${SNAME} > ${SNAME}.xz &
+nice -n19 gzip -c -9 ${SNAME} > ${SNAME}.gz &
+wait
# generate accompanying meta files
md5sum ${SNAME##*/} > ${SNAME}.xz.umd5sum
@@ -46,9 +56,11 @@ md5sum ${SNAME##*/}.bz2 > ${SNAME}.bz2.md5sum
md5sum ${SNAME##*/} > ${SNAME}.gz.umd5sum
md5sum ${SNAME##*/}.bz2 > ${SNAME}.gz.md5sum
# gpg is really stupid, or I am too stupid to find the right option
-gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C --detach-sign --armor -o ${SNAME}.xz.gpgsig ${SNAME}.xz < ${SCRIPTLOC}/autosigner.pwd
-gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C --detach-sign --armor -o ${SNAME}.bz2.gpgsig ${SNAME}.bz2 < ${SCRIPTLOC}/autosigner.pwd
-gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C --detach-sign --armor -o ${SNAME}.gz.gpgsig ${SNAME}.gz < ${SCRIPTLOC}/autosigner.pwd
+gpgopts="--quiet --batch --no-tty --passphrase-fd 0 --pinentry-mode loopback"
+gpgopts+=" --default-key C6317B3C --detach-sign --armor"
+gpg ${gpgopts} -o ${SNAME}.xz.gpgsig ${SNAME}.xz < ${SCRIPTLOC}/autosigner.pwd
+gpg ${gpgopts} -o ${SNAME}.bz2.gpgsig ${SNAME}.bz2 < ${SCRIPTLOC}/autosigner.pwd
+gpg ${gpgopts} -o ${SNAME}.gz.gpgsig ${SNAME}.gz < ${SCRIPTLOC}/autosigner.pwd
# we no longer need the tar
rm ${SNAME}
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] repo/proj/prefix:master commit in: scripts/, scripts/rsync-generation/
@ 2024-03-30 12:26 Fabian Groffen
0 siblings, 0 replies; 3+ messages in thread
From: Fabian Groffen @ 2024-03-30 12:26 UTC (permalink / raw
To: gentoo-commits
commit: 00075c640010369a4cb9c5d6ae9dba37c7cc47fb
Author: Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 30 12:26:13 2024 +0000
Commit: Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sat Mar 30 12:26:28 2024 +0000
URL: https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=00075c64
scripts/bootstrap-prefix: shellcheck
Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>
scripts/bootstrap-prefix.sh | 59 +++++++++++++++++++---------------
scripts/rsync-generation/mksnapshot.sh | 2 +-
2 files changed, 34 insertions(+), 27 deletions(-)
diff --git a/scripts/bootstrap-prefix.sh b/scripts/bootstrap-prefix.sh
index 2e4f542440..15e2c1098e 100755
--- a/scripts/bootstrap-prefix.sh
+++ b/scripts/bootstrap-prefix.sh
@@ -43,7 +43,7 @@ emake() {
else
estatus "stage1: building ${PWD##*/}"
fi
- read -a makeopts <<< "${MAKEOPTS}"
+ read -r -a makeopts <<< "${MAKEOPTS}"
v "${MAKE}" "${makeopts[@]}" "$@" || return 1
}
@@ -101,7 +101,7 @@ efetch() {
done
locs=( "${locs[@]}" "$1" )
- read -a fetchcmd <<< "${FETCH_COMMAND}"
+ read -r -a fetchcmd <<< "${FETCH_COMMAND}"
for loc in "${locs[@]}" ; do
v "${fetchcmd[@]}" "${loc}" < /dev/null
[[ -f ${1##*/} ]] && break
@@ -811,7 +811,7 @@ bootstrap_gnu() {
fi
URL=${GNU_URL}/${PN}/${A}
- efetch ${URL} || continue
+ efetch "${URL}" || continue
einfo "Unpacking ${A%.tar.*}"
S="${PORTAGE_TMPDIR}/${PN}-${PV}"
@@ -850,12 +850,12 @@ bootstrap_gnu() {
# have a suffix. Remove suffix by copy, not move, to not
# trigger refetch on repeated invocations of this script.
if [[ -f "${DISTDIR}/${tar_patch_file}?${tar_patch_id}" ]]; then
- cp ${DISTDIR}/${tar_patch_file}{?${tar_patch_id},} || return 1
+ cp "${DISTDIR}/${tar_patch_file}"{"?${tar_patch_id}",} || return 1
fi
- patch -p1 < ${DISTDIR}/${tar_patch_file} || return 1
+ patch -p1 < "${DISTDIR}/${tar_patch_file}" || return 1
fi
- local myconf=""
+ local -a myconf
if [[ ${PN} == "make" && ${PV} == "4.2.1" ]] ; then
if [[ ${CHOST} == *-linux-gnu* ]] ; then
# force this, macros aren't set correctly with newer glibc
@@ -887,25 +887,27 @@ bootstrap_gnu() {
# so just don't set it at all.
# Solaris 11 has a messed up prce installation. We don't need
# it anyway, so just disable it
- myconf="${myconf} --disable-perl-regexp"
+ myconf+=( "--disable-perl-regexp" )
fi
if [[ ${PN} == "mpfr" || ${PN} == "mpc" || ${PN} == "gcc" ]] ; then
[[ -e "${ROOT}"/tmp/usr/include/gmp.h ]] \
- && myconf="${myconf} --with-gmp=${ROOT}/tmp/usr"
+ && myconf+=( "--with-gmp=${ROOT}/tmp/usr" )
fi
if [[ ${PN} == "mpc" || ${PN} == "gcc" ]] ; then
[[ -e "${ROOT}"/tmp/usr/include/mpfr.h ]] \
- && myconf="${myconf} --with-mpfr=${ROOT}/tmp/usr"
+ && myconf+=( "--with-mpfr=${ROOT}/tmp/usr" )
fi
if [[ ${PN} == "gcc" ]] ; then
[[ -e "${ROOT}"/tmp/usr/include/mpc.h ]] \
- && myconf="${myconf} --with-mpc=${ROOT}/tmp/usr"
+ && myconf+=( "--with-mpc=${ROOT}/tmp/usr" )
- myconf="${myconf} --enable-languages=c,c++"
- myconf="${myconf} --disable-bootstrap"
- myconf="${myconf} --disable-multilib"
- myconf="${myconf} --disable-nls"
+ myconf+=(
+ "--enable-languages=c,c++"
+ "--disable-bootstrap"
+ "--disable-multilib"
+ "--disable-nls"
+ )
export CFLAGS="-O1 -pipe"
export CXXFLAGS="-O1 -pipe"
@@ -917,7 +919,7 @@ bootstrap_gnu() {
# On e.g. musl systems bash will crash with a malloc error if we use
# bash' internal malloc, so disable it during it this stage
- [[ ${PN} == "bash" ]] && myconf="${myconf} --without-bash-malloc"
+ [[ ${PN} == "bash" ]] && myconf+=( "--without-bash-malloc" )
# Ensure we don't read system-wide shell initialisation, it may
# contain cruft, bug #650284
@@ -934,30 +936,35 @@ bootstrap_gnu() {
# stdbuf is giving many problems, and we don't really care about it
# at this level, so disable it too
if [[ ${PN} == "coreutils" ]] ; then
- myconf="${myconf} --disable-acl --without-gmp"
- myconf="${myconf} --enable-no-install-program=stdbuf"
+ myconf+=(
+ "--disable-acl"
+ "--without-gmp"
+ "--enable-no-install-program=stdbuf"
+ )
fi
# Gentoo Bug 400831, fails on Ubuntu with libssl-dev installed
if [[ ${PN} == "wget" ]] ; then
if [[ -x ${ROOT}/tmp/usr/bin/openssl ]] ; then
- myconf="${myconf} --with-ssl=openssl"
- myconf="${myconf} --with-libssl-prefix=${ROOT}/tmp/usr"
+ myconf+=(
+ "-with-ssl=openssl"
+ "--with-libssl-prefix=${ROOT}/tmp/usr"
+ )
export CPPFLAGS="${CPPFLAGS} -I${ROOT}/tmp/usr/include"
export LDFLAGS="${LDFLAGS} -L${ROOT}/tmp/usr/lib"
else
- myconf="${myconf} --without-ssl"
+ myconf+=( "--without-ssl" )
fi
fi
# SuSE 11.1 has GNU binutils-2.20, choking on crc32_x86
- [[ ${PN} == "xz" ]] && myconf="${myconf} --disable-assembler"
+ [[ ${PN} == "xz" ]] && myconf+=( "--disable-assembler" )
if [[ ${PN} == "libffi" ]] ; then
# we do not have pkg-config to find lib/libffi-*/include/ffi.h
sed -i -e '/includesdir =/s/=.*/= $(includedir)/' include/Makefile.in
# force install into libdir
- myconf="${myconf} --libdir=${ROOT}/tmp/usr/lib"
+ myconf+=( "--libdir=${ROOT}/tmp/usr/lib" )
sed -i -e '/toolexeclibdir =/s/=.*/= $(libdir)/' Makefile.in
# we have to build the libraries for correct bitwidth
case $CHOST in
@@ -975,7 +982,7 @@ bootstrap_gnu() {
fi
einfo "Compiling ${A%.tar.*}"
- econf ${myconf} || return 1
+ econf "${myconf[@]}" || return 1
if [[ ${PN} == "make" && $(type -t $MAKE) != "file" ]]; then
estatus "stage1: building ${A%.tar.*}"
v ./build.sh || return 1
@@ -1255,7 +1262,7 @@ bootstrap_zlib_core() {
fi
local makeopts=()
# 1.2.5 suffers from a concurrency problem
- [[ ${PV} == 1.2.5 ]] || read -a makeopts <<< "${MAKEOPTS}"
+ [[ ${PV} == 1.2.5 ]] || read -r -a makeopts <<< "${MAKEOPTS}"
einfo "Compiling ${A%.tar.*}"
CHOST='' ${CONFIG_SHELL} ./configure --prefix="${ROOT}"/tmp/usr || return 1
@@ -1752,7 +1759,7 @@ do_emerge_pkgs() {
smyuse=" ${smyuse/ ${use#-} /} "
smyuse=" ${smyuse} ${use} "
done
- read -a myuse <<< "${smyuse}"
+ read -r -a myuse <<< "${smyuse}"
# Disable the STALE warning because the snapshot frequently gets stale.
#
@@ -2100,7 +2107,7 @@ bootstrap_stage3() {
cp -a "${ROOT}"{/tmp,}/usr/share/portage
fi
- read -a linker_pkgs <<< "${linker}"
+ read -r -a linker_pkgs <<< "${linker}"
if is-rap ; then
# We need ${ROOT}/usr/bin/perl to merge glibc.
diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh
index 1feece5577..f932102daa 100755
--- a/scripts/rsync-generation/mksnapshot.sh
+++ b/scripts/rsync-generation/mksnapshot.sh
@@ -54,7 +54,7 @@ COMPRS=(
# produce compressed variants, use as much cpu as left on the system, do
# all in parallel
for compr in "${COMPRS[@]}" ; do
- read -a args <<< "${compr#*:}"
+ read -r -a args <<< "${compr#*:}"
nice -n19 "${args[@]}" "${SNAME}" > "${SNAME}.${compr%%:*}" &
done
wait
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2024-03-30 12:26 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-03-30 12:26 [gentoo-commits] repo/proj/prefix:master commit in: scripts/, scripts/rsync-generation/ Fabian Groffen
-- strict thread matches above, loose matches on Subject: below --
2023-09-03 7:36 Fabian Groffen
2023-02-24 18:52 Sam James
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox