[gentoo-commits] proj/releng:master commit in: scripts/

[gentoo-commits] proj/releng:master commit in: scripts/

[Jorge Manuel B. S. Vicetto]

commit:     498f935c119b4b8093b2abeb1f29916aab67232f
Author:     Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Mon Jul  9 17:47:42 2012 +0000
Commit:     Jorge Manuel B. S. Vicetto <jmbsvicetto <AT> gentoo <DOT> org>
CommitDate: Mon Jul  9 17:48:44 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=498f935c

Add releng scripts to the repository.

---
 scripts/backup_snapshot_repo |   10 +
 scripts/cache-tools.py       |  700 ++++++++++++++++++++++++++++++++++++++++++
 scripts/copy_buildsync.sh    |  127 ++++++++
 scripts/run_catalyst         |    2 +
 scripts/run_official         |   39 +++
 scripts/run_snapshot         |    2 +
 scripts/stage_build.sh       |  162 ++++++++++
 scripts/sudo_catalyst        |   28 ++
 scripts/sudo_official        |   46 +++
 scripts/sudo_snapshot        |   20 ++
 scripts/update_auto_tree     |    2 +
 scripts/update_official_tree |    2 +
 scripts/update_snapshot_tree |    2 +
 13 files changed, 1142 insertions(+), 0 deletions(-)

diff --git a/scripts/backup_snapshot_repo b/scripts/backup_snapshot_repo
new file mode 100755
index 0000000..94b2828
--- /dev/null
+++ b/scripts/backup_snapshot_repo
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Start our rsyncs
+RSYNC_OPTS="--archive --delete --sparse --whole-file"
+
+if [ -e /release/repos/snapshot-tree/hooks/post-commit ]
+then
+	echo "$(date): Starting rsync of trees from tmpfs to disk..." >> /var/log/snapshot-tree-backup.log
+	rsync ${RSYNC_OPTS} /release/repos/snapshot-tree/* /release/repos/snapshot-tree-disk 2>&1 >> /var/log/snapshot-tree-backup.log || echo "$(date): rsync failed!" >> /var/log/snapshot-tree-backup.log
+fi

diff --git a/scripts/cache-tools.py b/scripts/cache-tools.py
new file mode 100755
index 0000000..0364450
--- /dev/null
+++ b/scripts/cache-tools.py
@@ -0,0 +1,700 @@
+#!/usr/bin/env python
+# Copyright 1999-2006 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: $
+#
+# Zac Medico <zmedico@gentoo.org>
+#
+
+import errno, fpformat, os, sys, time
+
+if not hasattr(__builtins__, "set"):
+	from sets import Set as set
+from itertools import chain
+
+def create_syncronized_func(myfunc, mylock):
+	def newfunc(*pargs, **kwargs):
+		mylock.acquire()
+		try:
+			myfunc(*pargs, **kwargs)
+		finally:
+			mylock.release()
+	return myfunc
+
+class ConsoleUpdate(object):
+
+	_synchronized_methods = ["append", "carriageReturn",
+		"newLine", "reset", "update"]
+
+	def __init__(self):
+		self.offset = 0
+		import sys
+		self.stream = sys.stdout
+		self.quiet = False
+		import threading
+		self._lock = threading.RLock()
+		for method_name in self._synchronized_methods:
+			myfunc = create_syncronized_func(
+				getattr(self, method_name), self._lock)
+			setattr(self, method_name, myfunc)
+		# ANSI code that clears from the cursor to the end of the line
+		self._CLEAR_EOL = None
+		try:
+			import curses
+			try:
+				curses.setupterm()
+				self._CLEAR_EOL = curses.tigetstr('el')
+			except curses.error:
+				pass
+		except ImportError:
+			pass
+		if not self._CLEAR_EOL:
+			self._CLEAR_EOL = '\x1b[K'
+
+	def acquire(self, **kwargs):
+		return self._lock.acquire(**kwargs)
+
+	def release(self):
+		self._lock.release()
+
+	def reset(self):
+		self.offset = 0
+
+	def carriageReturn(self):
+		if not self.quiet:
+			self.stream.write("\r")
+			self.stream.write(self._CLEAR_EOL)
+			self.offset = 0
+
+	def newLine(self):
+		if not self.quiet:
+			self.stream.write("\n")
+			self.stream.flush()
+			self.reset()
+
+	def update(self, msg):
+		if not self.quiet:
+			self.carriageReturn()
+			self.append(msg)
+
+	def append(self, msg):
+		if not self.quiet:
+			self.offset += len(msg)
+			self.stream.write(msg)
+			self.stream.flush()
+
+class ProgressCounter(object):
+	def __init__(self):
+		self.total = 0
+		self.current = 0
+
+class ProgressAnalyzer(ProgressCounter):
+	def __init__(self):
+		self.start_time = time.time()
+		self.currentTime = self.start_time
+		self._samples = []
+		self.sampleCount = 20
+	def percentage(self, digs=0):
+		if self.total > 0:
+			float_percent = 100 * float(self.current) / float(self.total)
+		else:
+			float_percent = 0.0
+		return fpformat.fix(float_percent,digs)
+	def totalTime(self):
+		self._samples.append((self.currentTime, self.current))
+		while len(self._samples) > self.sampleCount:
+			self._samples.pop(0)
+		prev_time, prev_count = self._samples[0]
+		time_delta = self.currentTime - prev_time
+		if time_delta > 0:
+			rate = (self.current - prev_count) / time_delta
+			if rate > 0:
+				return self.total / rate
+		return 0
+	def remaining_time(self):
+		return self.totalTime() - self.elapsed_time()
+	def elapsed_time(self):
+		return self.currentTime - self.start_time
+
+class ConsoleProgress(object):
+	def __init__(self, name="Progress", console=None):
+		self.name = name
+		self.analyzer = ProgressAnalyzer()
+		if console is None:
+			self.console = ConsoleUpdate()
+		else:
+			self.console = console
+		self.time_format="%H:%M:%S"
+		self.quiet = False
+		self.lastUpdate = 0
+		self.latency = 0.5
+
+	def formatTime(self, t):
+		return time.strftime(self.time_format, time.gmtime(t))
+
+	def displayProgress(self, current, total):
+		if self.quiet:
+			return
+
+		self.analyzer.currentTime = time.time()
+		if self.analyzer.currentTime - self.lastUpdate < self.latency:
+			return
+		self.lastUpdate = self.analyzer.currentTime
+		self.analyzer.current = current
+		self.analyzer.total = total
+
+		output = ((self.name, self.analyzer.percentage(1).rjust(4) + "%"),
+		("Elapsed", self.formatTime(self.analyzer.elapsed_time())),
+		("Remaining", self.formatTime(self.analyzer.remaining_time())),
+		("Total", self.formatTime(self.analyzer.totalTime())))
+		self.console.update(" ".join([ x[0] + ": " + x[1] for x in output ]))
+
+class ProgressHandler(object):
+	def __init__(self):
+		self.curval = 0
+		self.maxval = 0
+		self.last_update = 0
+		self.min_display_latency = 0.2
+
+	def onProgress(self, maxval, curval):
+		self.maxval = maxval
+		self.curval = curval
+		cur_time = time.time()
+		if cur_time - self.last_update >= self.min_display_latency:
+			self.last_update = cur_time
+			self.display()
+
+	def display(self):
+		raise NotImplementedError(self)
+
+def open_file(filename=None):
+	if filename is None:
+		f = sys.stderr
+	elif filename == "-":
+		f = sys.stdout
+	else:
+		try:
+			filename = os.path.expanduser(filename)
+			f = open(filename, "a")
+		except (IOError, OSError), e:
+			sys.stderr.write("%s\n" % e)
+			sys.exit(e.errno)
+	return f
+
+def create_log(name="", logfile=None, loglevel=0):
+	import logging
+	log = logging.getLogger(name)
+	log.setLevel(loglevel)
+	handler = logging.StreamHandler(open_file(logfile))
+	handler.setFormatter(logging.Formatter("%(levelname)s %(message)s"))
+	log.addHandler(handler)
+	return log
+
+def is_interrupt(e):
+	if isinstance(e, (SystemExit, KeyboardInterrupt)):
+		return True
+	return hasattr(e, "errno") and e.errno == errno.EINTR
+
+def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, log,
+	eclass_cache, cleanse_on_transfer_failure):
+
+	cleanse_candidates = set(trg_cache.iterkeys())
+	update_count = 0
+
+	# Since the loop below is mission critical, we continue after *any*
+	# exception that is not an interrupt.
+
+	for x in valid_nodes_iterable:
+		log.debug("%s mirroring" % x)
+		if not cleanse_on_transfer_failure:
+			cleanse_candidates.discard(x)
+
+		try:
+			entry = copy_dict(src_cache[x])
+		except KeyError, e:
+			log.error("%s missing source: %s" % (x, str(e)))
+			del e
+			continue
+		except Exception, e:
+			if is_interrupt(e):
+				raise
+			log.error("%s reading source: %s" % (x, str(e)))
+			del e
+			continue
+
+		write_it = True
+		trg = None
+
+		try:
+			trg = copy_dict(trg_cache[x])
+			if long(trg["_mtime_"]) == long(entry["_mtime_"]) and \
+				eclass_cache.is_eclass_data_valid(trg["_eclasses_"]) and \
+				set(trg["_eclasses_"]) == set(entry["_eclasses_"]):
+				write_it = False
+		except KeyError:
+			pass
+		except Exception, e:
+			if is_interrupt(e):
+				raise
+			log.error("%s reading target: %s" % (x, str(e)))
+			del e
+
+		if trg and not write_it:
+			""" We don't want to skip the write unless we're really sure that
+			the existing cache is identical, so don't trust _mtime_ and
+			_eclasses_ alone."""
+			for d in (entry, trg):
+				if "EAPI" in d and d["EAPI"] in ("", "0"):
+					del d["EAPI"]
+			for k in set(chain(entry, trg)).difference(
+				("_mtime_", "_eclasses_")):
+				if trg.get(k, "") != entry.get(k, ""):
+					write_it = True
+					break
+
+		if write_it:
+			update_count += 1
+			log.info("%s transferring" % x)
+			inherited = entry.get("INHERITED", None)
+			if inherited:
+				if src_cache.complete_eclass_entries:
+					if not "_eclasses_" in entry:
+						log.error("%s missing _eclasses_" % x)
+						continue
+					if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]):
+						log.error("%s stale _eclasses_" % x)
+						continue
+				else:
+					entry["_eclasses_"] = eclass_cache.get_eclass_data(entry["INHERITED"].split(), \
+						from_master_only=True)
+					if not entry["_eclasses_"]:
+						log.error("%s stale _eclasses_" % x)
+						continue
+			try:
+				trg_cache[x] = entry
+				cleanse_candidates.discard(x)
+			except Exception, e:
+				if is_interrupt(e):
+					raise
+				log.error("%s writing target: %s" % (x, str(e)))
+				del e
+		else:
+			cleanse_candidates.discard(x)
+
+	if not trg_cache.autocommits:
+		try:
+			trg_cache.commit()
+		except Exception, e:
+			if is_interrupt(e):
+				raise
+			log.error("committing target: %s" % str(e))
+			del e
+
+	return update_count, cleanse_candidates
+
+def copy_dict(src, dest=None):
+	"""Some cache implementations throw cache errors when accessing the values.
+	We grab all the values at once here so that we don't have to be concerned
+	about exceptions later."""
+	if dest is None:
+		dest = {}
+	for k, v in src.iteritems():
+		dest[k] = v
+	return dest
+
+class ListPackages(object):
+	def __init__(self, portdb, log, shuffle=False):
+		self._portdb = portdb
+		self._log = log
+		self._shuffle = shuffle
+
+	def run(self):
+		log = self._log
+		cp_list = self._portdb.cp_list
+		cp_all = self._portdb.cp_all()
+		if self._shuffle:
+			from random import shuffle
+			shuffle(cp_all)
+		else:
+			cp_all.sort()
+		cpv_all = []
+		# Since the loop below is mission critical, we continue after *any*
+		# exception that is not an interrupt.
+		for cp in cp_all:
+			log.debug("%s cp_list" % cp)
+			try:
+				cpv_all.extend(cp_list(cp))
+			except Exception, e:
+				if is_interrupt(e):
+					raise
+				self._log.error("%s cp_list: %s" % (cp, str(e)))
+
+		self.cpv_all = cpv_all
+
+class MetadataGenerate(object):
+	"""When cache generation fails for some reason, cleanse the stale cache
+	entry if it exists.  This prevents the master mirror from distributing
+	stale cache, and will allow clients to safely assume that all cache is
+	valid.  The mtime requirement is especially annoying due to bug #139134
+	(timestamps of cache entries don't change when an eclass changes) and the
+	interaction of timestamps with rsync."""
+	def __init__(self, portdb, cpv_all, log):
+		self._portdb = portdb
+		self._cpv_all = cpv_all
+		self._log = log
+
+	def run(self, onProgress=None):
+		log = self._log
+		portdb = self._portdb
+		cpv_all = self._cpv_all
+		auxdb = portdb.auxdb[portdb.porttree_root]
+		cleanse_candidates = set(auxdb.iterkeys())
+
+		# Since the loop below is mission critical, we continue after *any*
+		# exception that is not an interrupt.
+		maxval = len(cpv_all)
+		curval = 0
+		if onProgress:
+			onProgress(maxval, curval)
+		while cpv_all:
+			cpv = cpv_all.pop(0)
+			log.debug("%s generating" % cpv)
+			try:
+				portdb.aux_get(cpv, ["EAPI"])
+				# Cleanse if the above doesn't succeed (prevent clients from
+				# receiving stale cache, and let them assume it is valid).
+				cleanse_candidates.discard(cpv)
+			except Exception, e:
+				if is_interrupt(e):
+					raise
+				log.error("%s generating: %s" % (cpv, str(e)))
+				del e
+			curval += 1
+			if onProgress:
+				onProgress(maxval, curval)
+
+		self.target_cache = auxdb
+		self.dead_nodes = cleanse_candidates
+
+class MetadataTransfer(object):
+	def __init__(self, portdb, cpv_all, forward, cleanse_on_transfer_failure,
+		log):
+		self._portdb = portdb
+		self._cpv_all = cpv_all
+		self._log = log
+		self._forward = forward
+		self._cleanse_on_transfer_failure = cleanse_on_transfer_failure
+
+	def run(self, onProgress=None):
+		log = self._log
+		portdb = self._portdb
+		cpv_all = self._cpv_all
+		aux_cache = portdb.auxdb[portdb.porttree_root]
+		import portage
+		auxdbkeys = portage.auxdbkeys[:]
+		metadbmodule = portdb.mysettings.load_best_module("portdbapi.metadbmodule")
+		portdir_cache = metadbmodule(portdb.porttree_root, "metadata/cache",
+			auxdbkeys)
+
+		maxval = len(cpv_all)
+		curval = 0
+		if onProgress:
+			onProgress(maxval, curval)
+		class pkg_iter(object):
+			def __init__(self, pkg_list, onProgress=None):
+				self.pkg_list = pkg_list
+				self.maxval = len(pkg_list)
+				self.curval = 0
+				self.onProgress = onProgress
+			def __iter__(self):
+				while self.pkg_list:
+					yield self.pkg_list.pop()
+					self.curval += 1
+					if self.onProgress:
+						self.onProgress(self.maxval, self.curval)
+
+		if self._forward:
+			src_cache = portdir_cache
+			trg_cache = aux_cache
+		else:
+			src_cache = aux_cache
+			trg_cache = portdir_cache
+
+		""" This encapsulates validation of eclass timestamps and also fills in
+		missing data (mtimes and/or paths) as necessary for the given cache
+		format."""
+		eclass_cache = portage.eclass_cache.cache(portdb.porttree_root)
+
+		if not trg_cache.autocommits:
+			trg_cache.sync(100)
+
+		self.target_cache = trg_cache
+		self.update_count, self.dead_nodes = mirror_cache(
+			pkg_iter(cpv_all, onProgress=onProgress),
+			src_cache, trg_cache, log, eclass_cache,
+			self._cleanse_on_transfer_failure)
+
+class CacheCleanse(object):
+	def __init__(self, auxdb, dead_nodes, log):
+		self._auxdb = auxdb
+		self._dead_nodes = dead_nodes
+		self._log = log
+	def run(self):
+		auxdb = self._auxdb
+		log = self._log
+		for cpv in self._dead_nodes:
+			try:
+				log.info("%s cleansing" % cpv)
+				del auxdb[cpv]
+			except Exception, e:
+				if is_interrupt(e):
+					raise
+				log.error("%s cleansing: %s" % (cpv, str(e)))
+				del e
+
+def import_portage():
+	try:
+		from portage import data as portage_data
+	except ImportError:
+		import portage_data
+	# If we're not already root or in the portage group, we make the gid of the
+	# current process become portage_gid.
+	if os.getgid() != 0 and portage_data.portage_gid not in os.getgroups():
+		portage_data.portage_gid = os.getgid()
+		portage_data.secpass = 1
+
+	os.environ["PORTAGE_LEGACY_GLOBALS"] = "false"
+	import portage
+	del os.environ["PORTAGE_LEGACY_GLOBALS"]
+	return portage
+
+def create_portdb(portdir=None, cachedir=None, config_root=None,
+	target_root=None, profile=None, **kwargs):
+
+	if cachedir is not None:
+		os.environ["PORTAGE_DEPCACHEDIR"] = cachedir
+	if config_root is None:
+		config_root = os.environ.get("PORTAGE_CONFIGROOT", "/")
+	if target_root is None:
+		target_root = os.environ.get("ROOT", "/")
+	if profile is None:
+		profile = ""
+
+	portage = import_portage()
+	try:
+		from portage import const as portage_const
+	except ImportError:
+		import portage_const
+
+	# Disable overlays because we only generate metadata for the main repo.
+	os.environ["PORTDIR_OVERLAY"] = ""
+	conf = portage.config(config_profile_path=profile,
+		config_incrementals=portage_const.INCREMENTALS,
+		target_root=target_root,
+		config_root=config_root)
+
+	if portdir is None:
+		portdir = conf["PORTDIR"]
+
+	# The cannonical path is the key for portdb.auxdb.
+	portdir = os.path.realpath(portdir)
+	conf["PORTDIR"] = portdir
+	conf.backup_changes("PORTDIR")
+
+	portdb = portage.portdbapi(portdir,
+		mysettings=conf)
+
+	return portdb
+
+def parse_args(myargv):
+	description = "This program will ensure that the metadata cache is up to date for entire portage tree."
+	usage = "usage: cache-tools [options] --generate || --transfer"
+	from optparse import OptionParser
+	parser = OptionParser(description=description, usage=usage)
+	parser.add_option("--portdir",
+		help="location of the portage tree",
+		dest="portdir")
+	parser.add_option("--cachedir",
+		help="location of the metadata cache",
+		dest="cachedir")
+	parser.add_option("--profile",
+		help="location of the profile",
+		dest="profile")
+	parser.add_option("--generate",
+		help="generate metadata as necessary to ensure that the cache is fully populated",
+		action="store_true", dest="generate", default=False)
+	parser.add_option("--shuffle",
+		help="generate cache in random rather than sorted order (useful to prevent two separate instances from competing to generate metadata for the same packages simultaneously)",
+		action="store_true", dest="shuffle", default=False)
+	parser.add_option("--transfer",
+		help="transfer metadata from portdir to cachedir or vice versa",
+		action="store_true", dest="transfer", default=False)
+	parser.add_option("--cleanse-on-transfer-failure",
+		help="cleanse target cache when transfer fails for any reason (such as the source being unavailable)",
+		action="store_true", dest="cleanse_on_transfer_failure", default=False)
+	parser.add_option("--forward",
+		help="forward metadata transfer flows from portdir to cachedir (default)",
+		action="store_true", dest="forward", default=True)
+	parser.add_option("--reverse",
+		help="reverse metadata transfer flows from cachedir to portdir",
+		action="store_false", dest="forward", default=True)
+	parser.add_option("--logfile",
+		help="send status messages to a file (default is stderr)",
+		dest="logfile", default=None)
+	parser.add_option("--loglevel",
+		help="numeric log level (defauls to 0 and may range from 0 to 50 corresponding to the default levels of the python logging module)",
+		dest="loglevel", default="0")
+	parser.add_option("--reportfile",
+		help="send a report to a file",
+		dest="reportfile", default=None)
+	parser.add_option("--spawn-outfile",
+		help="redirect ouput of spawned processes to a file instead of stdout/stderr",
+		dest="spawn_outfile", default=None)
+	parser.add_option("--no-progress",
+		action="store_false", dest="progress", default=True,
+		help="disable progress output to tty")
+	options, args = parser.parse_args(args=myargv)
+
+	# Conversion to dict allows us to use **opts as function args later on.
+	opts = {}
+	all_options = ("portdir", "cachedir", "profile", "progress", "logfile",
+		"loglevel", "generate", "transfer", "forward", "shuffle",
+		"spawn_outfile", "reportfile", "cleanse_on_transfer_failure")
+	for opt_name in all_options:
+		v = getattr(options, opt_name)
+		opts[opt_name] = v
+	return opts
+
+def run_command(args):
+	opts = parse_args(sys.argv[1:])
+
+	if opts["spawn_outfile"]:
+		fd = os.dup(1)
+		sys.stdout = os.fdopen(fd, 'w')
+		fd = os.dup(2)
+		sys.stderr = os.fdopen(fd, 'w')
+		f = open_file(opts["spawn_outfile"])
+		os.dup2(f.fileno(), 1)
+		os.dup2(f.fileno(), 2)
+		del fd, f
+
+	console = ConsoleUpdate()
+	if not opts["progress"] or not sys.stdout.isatty():
+		console.quiet = True
+	job = None
+	import signal, thread, threading
+	shutdown_initiated = threading.Event()
+	shutdown_complete = threading.Event()
+	def shutdown_console():
+		console.acquire()
+		try:
+			console.update("Interrupted.")
+			console.newLine()
+			console.quiet = True
+			shutdown_complete.set()
+			# Kill the main thread if necessary.
+			# This causes the SIGINT signal handler to be invoked in the
+			# main thread.  The signal handler needs to be an actual
+			# callable object (rather than something like signal.SIG_DFL)
+			# in order to avoid TypeError: 'int' object is not callable.
+			thread.interrupt_main()
+			thread.exit()
+		finally:
+			console.release()
+
+	def handle_interrupt(*args):
+		if shutdown_complete.isSet():
+			sys.exit(1)
+		# Lock the console from a new thread so that the main thread is allowed
+		# to cleanly complete any console interaction that may have been in
+		# progress when this interrupt arrived.
+		if not shutdown_initiated.isSet():
+			thread.start_new_thread(shutdown_console, ())
+			shutdown_initiated.set()
+
+	signal.signal(signal.SIGINT, handle_interrupt)
+	signal.signal(signal.SIGTERM, handle_interrupt)
+
+	try:
+		import datetime
+		datestamp = str(datetime.datetime.now())
+		time_begin = time.time()
+		log = create_log(name="MetadataGenerate",
+			logfile=opts["logfile"], loglevel=int(opts["loglevel"]))
+		if opts["reportfile"]:
+			reportfile = open_file(opts["reportfile"])
+		portdb = create_portdb(**opts)
+		try:
+			os.nice(int(portdb.mysettings.get("PORTAGE_NICENESS", "0")))
+		except (OSError, ValueError), e:
+			log.error("PORTAGE_NICENESS failed: '%s'" % str(e))
+			del e
+
+		job = ListPackages(portdb, log, shuffle=opts["shuffle"])
+		console.update("Listing packages in repository...")
+		job.run()
+		cpv_all = job.cpv_all
+		total_count = len(cpv_all)
+		if opts["generate"]:
+			job = MetadataGenerate(portdb, cpv_all, log)
+			name = "Cache generation"
+			complete_msg = "Metadata generation is complete."
+		elif opts["transfer"]:
+			job = MetadataTransfer(portdb, cpv_all, opts["forward"],
+			opts["cleanse_on_transfer_failure"], log)
+			if opts["forward"]:
+				name = "Forward transfer"
+				complete_msg = "Forward metadata transfer is complete."
+			else:
+				name = "Reverse transfer"
+				complete_msg = "Reverse metadata transfer is complete."
+		else:
+			sys.stderr.write("required options: --generate || --transfer\n")
+			sys.exit(os.EX_USAGE)
+		job.opts = opts
+
+		onProgress = None
+		if not console.quiet:
+			ui = ConsoleProgress(name=name, console=console)
+			progressHandler = ProgressHandler()
+			onProgress = progressHandler.onProgress
+			def display():
+				ui.displayProgress(progressHandler.curval, progressHandler.maxval)
+			progressHandler.display = display
+
+		job.run(onProgress=onProgress)
+
+		if not console.quiet:
+			# make sure the final progress is displayed
+			progressHandler.display()
+
+		update_count = None
+		if opts["transfer"]:
+			update_count = job.update_count
+		target_cache = job.target_cache
+		dead_nodes = job.dead_nodes
+		cleanse_count = len(dead_nodes)
+		console.update("Cleansing cache...")
+		job = CacheCleanse(target_cache, dead_nodes, log)
+		job.run()
+		console.update(complete_msg)
+		console.newLine()
+		time_end = time.time()
+		if opts["reportfile"]:
+			width = 20
+			reportfile.write(name.ljust(width) + "%s\n" % datestamp)
+			reportfile.write("Elapsed seconds".ljust(width) + "%f\n" % (time_end - time_begin))
+			reportfile.write("Total packages".ljust(width) + "%i\n" % total_count)
+			if update_count is not None:
+				reportfile.write("Updated packages".ljust(width) + "%i\n" % update_count)
+			reportfile.write("Cleansed packages".ljust(width) + "%i\n" % cleanse_count)
+			reportfile.write(("-"*50)+"\n")
+	except Exception, e:
+		if not is_interrupt(e):
+			raise
+		del e
+		handle_interrupt()
+	sys.exit(0)
+
+if __name__ == "__main__":
+	run_command(sys.argv[1:])

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
new file mode 100755
index 0000000..729ff38
--- /dev/null
+++ b/scripts/copy_buildsync.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+
+ARCHES="alpha amd64 arm hppa ia64      ppc sparc x86 sh s390"
+       #alpha amd64 arm hppa ia64 mips ppc s390 sh sparc x86
+#ARCHES="s390"
+RSYNC_OPTS="-aO --delay-updates"
+DEBUG=
+VERBOSE=
+
+OUT_STAGE3="latest-stage3.txt"
+OUT_ISO="latest-iso.txt"
+
+# Nothing to edit beyond this point
+
+DEBUGP=
+VERBOSEP=
+[ -n "$DEBUG" ] && DEBUGP=echo
+[ -n "$DEBUG" ] && RSYNC_OPTS="${RSYNC_OPTS} -n"
+[ -n "$VERBOSE" ] && RSYNC_OPTS="${RSYNC_OPTS} -v"
+[ -n "$VERBOSEP" ] && VERBOSEP="-v"
+
+for ARCH in $ARCHES; do
+	rc=0
+	fail=0
+
+	indir=/home/buildsync/builds/${ARCH}
+	outdir=/release/weekly/${ARCH}
+	tmpdir=/release/tmp/buildsync/partial/${ARCH}
+
+	mkdir -p ${tmpdir} 2>/dev/null
+	# Copying
+	if [ -d "${indir}" ]; then
+		for i in $(find ${indir} -type f | grep -- '-20[0123][0-9]\{5\}' | sed -e 's:^.*-\(20[^.]\+\).*$:\1:' | sort -ur); do
+			#echo "Doing $i"
+			t="${outdir}/${i}"
+			mkdir -p ${t} 2>/dev/null
+			rsync ${RSYNC_OPTS} --temp-dir=${tmpdir} --partial-dir=${tmpdir} ${indir}/ --filter "S *${i}*" --filter 'S **/' --filter 'H *' ${t}
+			rc=$?
+			if [ $rc -eq 0 ]; then
+				find ${indir} -type f -name "*${i}*" -print0 | xargs -0 --no-run-if-empty $DEBUGP rm $VERBOSEP -f
+			else
+				echo "Not deleting ${indir}/*${i}*, rsync failed!" 1>&2
+				fail=1
+			fi
+		done
+		find ${outdir} -mindepth 1 -type d \
+			| egrep -v current \
+			| sort -r \
+			| tr '\n' '\0' \
+			|xargs -0 --no-run-if-empty rmdir --ignore-fail-on-non-empty
+	fi
+
+	# ================================================================
+	# Build data for revealing latest:
+	# *.iso
+	# stage3*bz2
+	cd "${outdir}"
+	# %T@
+
+	iso_list="$(find 20* -name '*.iso' -printf '%h %f %h/%f\n' |grep -v hardened | sort -n)"
+	stage3_list="$(find 20* -name 'stage3*bz2' -printf '%h %f %h/%f\n' |grep -v hardened | sort -n)"
+	latest_iso_date="$(echo -e "${iso_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
+	latest_stage3_date="$(echo -e "${stage3_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
+	header="$(echo -e "# Latest as of $(date -uR)\n# ts=$(date -u +%s)")"
+	
+	# Do not remove this
+	[ -z "${latest_iso_date}" ] && latest_iso_date="NONE-FOUND"
+	[ -z "${latest_stage3_date}" ] && latest_stage3_date="NONE-FOUND"
+	
+	if [ -n "${iso_list}" ]; then
+		echo -e "${header}" >"${OUT_ISO}"
+		echo -e "${iso_list}" |awk '{print $3}' | grep "$latest_iso_date" >>${OUT_ISO}
+		rm -f current-iso
+		ln -sf "$latest_iso_date" current-iso
+	fi
+	if [ -n "${stage3_list}" ]; then
+		echo -e "${header}" >"${OUT_STAGE3}"
+		echo -e "${stage3_list}" |awk '{print $3}' |grep "$latest_stage3_date" >>${OUT_STAGE3}
+		rm -f current-stage3
+		# The "latest stage3" concept doesn't apply to the arm variants
+		# that are pushed on different days of the week.
+		if [[ ! $(echo ${outdir} | grep arm) ]]; then
+			ln -sf "$latest_stage3_date" current-stage3
+		fi
+	fi
+	
+	# new variant preserve code
+	variants="$(find 20* \( -iname '*.iso' -o -iname '*.tar.bz2' \) -printf '%f\n' |sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)"
+	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
+	for v in $variants ; do 
+		#date_variant=$(find 20* -iname "${v}*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -printf '%h\n' | sed -e "s,.*/$a/autobuilds/,,g" -e 's,/.*,,g' |sort -n | tail -n1 )
+		variant_path=$(find 20* -iname "${v}*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		f="latest-${v}.txt"
+		echo -e "${header}" >"${f}"
+		echo -e "${variant_path}" >>${f}
+		rm -f "current-$v"
+		ln -sf "${variant_path%/*}" "current-$v"
+		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${tmpdir}"/.keep.${ARCH}.txt
+	done 
+	#echo "$date_variant" \
+	#| sort | uniq | sed -e 's,^,/,g' -e 's,$,$,g' >"${tmpdir}"/.keep.${ARCH}.txt
+
+	# ================================================================
+	# Cleanup
+	if [ $fail -eq 0 ]; then
+		# Clean up all but latest 4 from mirror dir
+		cd "${outdir}"
+		#echo regex "/${latest_iso_date}\$|/${latest_stage3_date}\$"
+		for i in $(find -regextype posix-basic -mindepth 1 -maxdepth 1 -type d -regex '.*20[012][0-9]\{5\}.*' \
+				| sed -e 's:^.*-\(20[^.]\+\).*$:\1:' \
+				| sort -ur \
+				| egrep -v "/${latest_iso_date}\$|/${latest_stage3_date}\$" \
+				| egrep -v -f "${tmpdir}"/.keep.${ARCH}.txt \
+				| tail -n +5); do
+
+			$DEBUGP rm $VERBOSEP -rf $(pwd)/${i}
+		done
+		
+		$DEBUGP rm $VERBOSEP -rf ${tmpdir}
+
+	else
+		echo "There was some failure for $ARCH during the weekly sync. Not doing cleanup for fear of dataloss." 1>&2
+	fi
+
+done
+
+# vim:ts=2 sw=2 noet ft=sh:

diff --git a/scripts/run_catalyst b/scripts/run_catalyst
new file mode 100755
index 0000000..997f652
--- /dev/null
+++ b/scripts/run_catalyst
@@ -0,0 +1,2 @@
+#!/bin/bash
+sudo /release/bin/sudo_catalyst "$@"

diff --git a/scripts/run_official b/scripts/run_official
new file mode 100755
index 0000000..dfb29f2
--- /dev/null
+++ b/scripts/run_official
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+email_from="auto"
+email_to="releng@gentoo.org"
+url="https://poseidon.amd64.dev.gentoo.org/snapshots"
+snapshot_uri="/release/webroot/snapshots"
+svn_repo="/release/repos/snapshot-tree"
+
+send_email() {
+  subject="[Snapshot] ${1}"
+
+  echo -e "From: ${email_from}\r\nTo: ${email_to}\r\nSubject: ${subject}\r\n\r\nA new snapshot has been built from revision `svnlook history ${svn_repo} | head -n 3 | tail -n 1 | sed -e 's:^ *::' -e 's: .*$::'` of ${svn_repo}.  You can find it at ${url}.\r\n\r\n$(cat /release/snapshots/portage-${1}.tar.bz2.DIGESTS)\r\n" | /usr/sbin/sendmail -f ${email_from} ${email_to}
+}
+
+if [ "${email_from}" == "auto" ]
+then
+	username="$(whoami)"
+	if [ "${username}" == "root" ]
+	then
+		email_from="catalyst@poseidon.amd64.dev.gentoo.org"
+	else
+		email_from="${username}@gentoo.org"
+	fi
+fi
+
+sudo /release/bin/sudo_official "$@" && \
+echo "Starting rsync from /release/snapshots/portage-${1}.tar.bz2* to ${snapshot_uri}" && \
+rsync --archive --stats --progress /release/snapshots/portage-${1}.tar.bz2* \
+	${snapshot_uri}
+ret=$?
+
+if [ "${email_from}" == "none" ]
+then
+	echo "Skipping email step as configured..."
+else
+	[ $ret -eq 0 ] && send_email ${1}
+fi
+
+exit $ret

diff --git a/scripts/run_snapshot b/scripts/run_snapshot
new file mode 100755
index 0000000..20cc460
--- /dev/null
+++ b/scripts/run_snapshot
@@ -0,0 +1,2 @@
+#!/bin/bash
+sudo /release/bin/sudo_snapshot "$@"

diff --git a/scripts/stage_build.sh b/scripts/stage_build.sh
new file mode 100755
index 0000000..0dd89a9
--- /dev/null
+++ b/scripts/stage_build.sh
@@ -0,0 +1,162 @@
+#!/bin/bash
+
+PID=$$
+
+profile=
+version_stamp=
+subarch=
+stage1_seed=
+snapshot=
+config=/etc/catalyst/catalyst.conf
+email_from="catalyst@localhost"
+email_to="root@localhost"
+verbose=0
+
+usage() {
+  msg=$1
+
+  if [ -n "${msg}" ]; then
+    echo -e "${msg}\n";
+  fi
+
+  cat <<EOH
+Usage:
+  stage_build [-p|--profile <profile>] [-v|--version-stamp <stamp>]
+              [-a|--arch <arch>] [-s|--stage1-seed <seed>] [--verbose]
+              [-f|--email-from <from>] [-t|--email-to <to>] [-h|--help]
+
+Options:
+  -p|--profile        Sets the portage profile (required)
+  -v|--version-stamp  Sets the version stamp (required)
+  -a|--arch           Sets the 'subarch' in the spec (required)
+  -s|--stage1-seed    Sets the seed for the stage1 (required)
+  -S|--snapshot       Sets the snapshot name (if not given defaults to today's
+                      date)
+  -c|--config         catalyst config to use, defaults to catalyst default
+  --verbose           Send output of commands to console as well as log
+  -f|--email-from     Sets the 'From' on emails sent from this script (defaults
+                      to catalyst@localhost)
+  -t|--email-to       Sets the 'To' on emails sent from this script (defaults
+                      to root@localhost)
+  -h|--help           Show this message and quit
+
+Example:
+  stage_build -p default-linux/x86/2006.1 -v 2007.0_pre -a i686 -s default/stage3-i686-2006.1
+EOH
+}
+
+send_email() {
+  subject="[${subarch}] $1"
+  body=$2
+
+  echo -e "From: ${email_from}\r\nTo: ${email_to}\r\nSubject: ${subject}\r\n\r\nArch: ${subarch}\r\nProfile: ${profile}\r\nVersion stamp: ${version_stamp}\r\nStage1 seed: ${stage1_seed}\r\nSnapshot: ${snapshot}\r\n\r\n${body}\r\n" | /usr/sbin/sendmail -f ${email_from} ${email_to}
+}
+
+run_cmd() {
+  cmd=$1
+  logfile=$2
+
+  if [ $verbose = 1 ]; then
+    ${cmd} 2>&1 | tee ${logfile}
+  else
+    ${cmd} &> ${logfile}
+  fi
+}
+
+# Parse args
+params=${#}
+while [ ${#} -gt 0 ]
+do
+  a=${1}
+  shift
+  case "${a}" in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    -p|--profile)
+      profile=$1
+      shift
+      ;;
+    -v|--version-stamp)
+      version_stamp=$1
+      shift
+      ;;
+    -a|--arch)
+      subarch=$1
+      shift
+      ;;
+    -f|--email-from)
+      email_from=$1
+      shift
+      ;;
+    -t|--email-to)
+      email_to=$1
+      shift
+      ;;
+    -s|--stage1-seed)
+      stage1_seed=$1
+      shift
+      ;;
+    -S|--snapshot)
+      snapshot=$1
+      shift
+      ;;
+    -c|--config)
+      config=$1
+      shift
+      ;;
+    --verbose)
+      verbose=1
+      ;;
+    -*)
+      echo "You have specified an invalid option: ${a}"
+      usage
+      exit 1
+      ;;
+    esac
+done
+
+# Make sure all required values were specified
+if [ -z "${profile}" ]; then
+  usage "You must specify a profile."
+  exit 1
+fi
+if [ -z "${version_stamp}" ]; then
+  usage "You must specify a version stamp."
+  exit 1
+fi
+if [ -z "${subarch}" ]; then
+  usage "You must specify an arch."
+  exit 1
+fi
+if [ -z "${stage1_seed}" ]; then
+  usage "You must specify a stage1 seed."
+  exit 1
+fi
+cd /tmp
+
+if [ -z "${snapshot}" ]; then
+  snapshot=`date +%Y%m%d`
+  run_cmd "catalyst -c ${config} -s '${snapshot}'" "/tmp/catalyst_build_snapshot.${PID}.log"
+  if [ $? != 0 ]; then
+    send_email "Catalyst build error - snapshot" "$(</tmp/catalyst_build_snapshot.${PID}.log)"
+    exit 1
+  fi
+fi
+
+for i in 1 2 3; do
+  echo -e "subarch: ${subarch}\ntarget: stage${i}\nversion_stamp: ${version_stamp}\nrel_type: default\nprofile: ${profile}\nsnapshot: ${snapshot}" > stage${i}.spec
+  if [ ${i} = 1 ]; then
+    echo "source_subpath: ${stage1_seed}" >> stage${i}.spec
+  else 
+    echo "source_subpath: default/stage$(expr ${i} - 1)-${subarch}-${version_stamp}" >> stage${i}.spec
+  fi
+  run_cmd "catalyst -a -p -c ${config} -f stage${i}.spec" "/tmp/catalyst_build_stage${i}.${PID}.log"
+  if [ $? != 0 ]; then
+    send_email "Catalyst build error - stage${i}" "$(tail -n 200 /tmp/catalyst_build_stage${i}.${PID}.log)\r\n\r\nFull build log at /tmp/catalyst_build_stage${i}.${PID}.log"
+    exit 1
+  fi
+done
+
+send_email "Catalyst build success" "Everything finished successfully."

diff --git a/scripts/sudo_catalyst b/scripts/sudo_catalyst
new file mode 100755
index 0000000..19ecc90
--- /dev/null
+++ b/scripts/sudo_catalyst
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+usage() {
+	echo "Usage: $(basename ${0}) <arch> <target> <spec>"
+	echo "Where arch is either amd64 or x86, target is default, dev, hardened,"
+	echo "or uclibc, and spec is your spec file."
+	echo
+}
+
+if [ -z "$1" -o -z "$2" -o -z "$3" ]
+then
+	usage
+else
+	target="$(grep target ${3} | cut -d' ' -f2)"
+	/usr/bin/catalyst -c /etc/catalyst/${1}-${2}.conf -f ${3} ${4} ${5}
+#	&& \
+#	case ${target} in
+#		stage*|grp*|livecd-stage2)
+#			echo "Cleaning out ${target} temp files"
+#			rel_type="$(grep rel_type ${3} | cut -d' ' -f2)"
+#			subarch="$(grep subarch ${3} | cut -d' ' -f2)"
+#			version="$(grep version ${3} | cut -d' ' -f2)"
+#			storedir="$(grep storedir /etc/catalyst/${1}-${2}.conf | cut -d\" -f2)"
+#			echo "Removing ${storedir}/tmp/${rel_type}/${target}-${subarch}-${version}"
+#			rm -rf ${storedir}/tmp/${rel_type}/${target}-${subarch}-${version}
+#		;;
+#	esac
+fi

diff --git a/scripts/sudo_official b/scripts/sudo_official
new file mode 100755
index 0000000..80e7ca0
--- /dev/null
+++ b/scripts/sudo_official
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+tree="/release/trees/snapshot-tree"
+portdir="${tree}/${1/_beta2/}/portage"
+cache_args="--portdir=${portdir} --cachedir=/release/tmp/depcache"
+
+usage() {
+	echo "Usage: $(basename ${0}) <version>"
+}
+
+if [ -z "${1}" ]
+then
+	usage
+else
+	cd ${tree}
+	echo "Clearing out old metadata cache"
+	rm -rf ${portdir}/metadata/cache
+	echo "Performing a svn up on ${tree}"
+	svn up || exit 1
+	mkdir -p ${portdir}/metadata/cache
+	echo "Recreating portage metadata cache"
+	cache-tools.py ${cache_args} --generate || exit 1
+	cache-tools.py ${cache_args} --transfer --reverse \
+		--cleanse-on-transfer-failure || exit 1
+	if [ ! -d ${portdir}/metadata/cache/sys-kernel ]
+	then
+		echo "Metadata update failed! Bailing out!"
+		exit 1
+	fi
+	catalyst -c /etc/catalyst/snapshot-official.conf -s ${1} \
+		-C portdir="${portdir}" || exit 1
+	for i in amd64 x86
+	do
+		for j in default dev hardened uclibc
+		do
+			cd /release/buildroot/${i}-${j}/snapshots
+			rm -f portage-official.tar.bz2 portage-${1}.tar.bz2*
+			ln -sf /release/snapshots/portage-${1}.tar.bz2 \
+				portage-official.tar.bz2
+			ln -sf /release/snapshots/portage-${1}.tar.bz2 \
+				portage-${1}.tar.bz2
+			ln -sf /release/snapshots/portage-${1}.tar.bz2.DIGESTS \
+				portage-${1}.tar.bz2.DIGESTS
+		done
+	done
+fi

diff --git a/scripts/sudo_snapshot b/scripts/sudo_snapshot
new file mode 100755
index 0000000..1ba6485
--- /dev/null
+++ b/scripts/sudo_snapshot
@@ -0,0 +1,20 @@
+#!/bin/bash
+usage() {
+	echo "Usage: $(basename ${0}) <version>"
+}
+if [ -z "${1}" ]
+then
+	usage
+else
+	catalyst -c /etc/catalyst/snapshot.conf -s ${1}
+	for i in amd64 x86
+	do
+		for j in default dev hardened uclibc
+		do
+			cd /release/buildroot/${i}-${j}/snapshots
+			rm -f portage-${1}.tar.bz2
+			ln -sf /release/snapshots/portage-${1}.tar.bz2 \
+				portage-${1}.tar.bz2
+		done
+	done
+fi

diff --git a/scripts/update_auto_tree b/scripts/update_auto_tree
new file mode 100755
index 0000000..08909e7
--- /dev/null
+++ b/scripts/update_auto_tree
@@ -0,0 +1,2 @@
+#!/bin/bash
+PORTDIR="/release/trees/portage-auto/" FEATURES="$FEATURES -news" emerge --sync -q

diff --git a/scripts/update_official_tree b/scripts/update_official_tree
new file mode 100755
index 0000000..250e905
--- /dev/null
+++ b/scripts/update_official_tree
@@ -0,0 +1,2 @@
+#!/bin/bash
+PORTDIR="/release/trees/portage-official/" emerge --sync

diff --git a/scripts/update_snapshot_tree b/scripts/update_snapshot_tree
new file mode 100755
index 0000000..f64742d
--- /dev/null
+++ b/scripts/update_snapshot_tree
@@ -0,0 +1,2 @@
+#!/bin/bash
+PORTDIR="/release/trees/portage-snapshot/" emerge --sync

[gentoo-commits] proj/releng:master commit in: scripts/

[Raúl Porcel]

commit:     62d8bf9930952885cca9df1b9c6b316c8192b35d
Author:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 22 17:35:50 2012 +0000
Commit:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
CommitDate: Sun Jul 22 17:35:50 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=62d8bf99

Add a - to fix arm* variants symlink, bug #366123

---
 scripts/copy_buildsync.sh |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 729ff38..c328ccc 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -89,7 +89,7 @@ for ARCH in $ARCHES; do
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do 
 		#date_variant=$(find 20* -iname "${v}*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -printf '%h\n' | sed -e "s,.*/$a/autobuilds/,,g" -e 's,/.*,,g' |sort -n | tail -n1 )
-		variant_path=$(find 20* -iname "${v}*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		variant_path=$(find 20* -iname "${v}-*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
 		f="latest-${v}.txt"
 		echo -e "${header}" >"${f}"
 		echo -e "${variant_path}" >>${f}

[gentoo-commits] proj/releng:master commit in: scripts/

[Jorge Manuel B. S. Vicetto]

commit:     675bfe9bb2b6d3f7f83e57ca00639534b120d2c7
Author:     Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Sun Nov  4 13:22:22 2012 +0000
Commit:     Jorge Manuel B. S. Vicetto <jmbsvicetto <AT> gentoo <DOT> org>
CommitDate: Sun Nov  4 13:22:22 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=675bfe9b

Update locations of source builds and destination dirs for distfile mirroring.

---
 scripts/copy_buildsync.sh |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index c328ccc..4fff928 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -23,8 +23,8 @@ for ARCH in $ARCHES; do
 	rc=0
 	fail=0
 
-	indir=/home/buildsync/builds/${ARCH}
-	outdir=/release/weekly/${ARCH}
+	indir=/release/weekly/builds/${ARCH}
+	outdir=/release/distfiles/weekly/${ARCH}
 	tmpdir=/release/tmp/buildsync/partial/${ARCH}
 
 	mkdir -p ${tmpdir} 2>/dev/null

[gentoo-commits] proj/releng:master commit in: scripts/

[Raúl Porcel]

commit:     25a9f336227c8d6807100c658bd68d6371262024
Author:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 20 07:44:12 2013 +0000
Commit:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
CommitDate: Sun Jan 20 07:44:12 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=25a9f336

Adjust to skimmer's layout, as /release/distfiles is another filesystem and therefore rsync complains when doing the tmp and partial stuff

---
 scripts/copy_buildsync.sh |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 4fff928..3d998e6 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -25,7 +25,7 @@ for ARCH in $ARCHES; do
 
 	indir=/release/weekly/builds/${ARCH}
 	outdir=/release/distfiles/weekly/${ARCH}
-	tmpdir=/release/tmp/buildsync/partial/${ARCH}
+	tmpdir=/release/distfiles/tmp/buildsync/partial/${ARCH}
 
 	mkdir -p ${tmpdir} 2>/dev/null
 	# Copying

[gentoo-commits] proj/releng:master commit in: scripts/

[Jorge Manuel B. S. Vicetto]

commit:     08c9ca74d61cba28bec59cedcf84a19fe1899d82
Author:     Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 18 08:04:50 2013 +0000
Commit:     Jorge Manuel B. S. Vicetto <jmbsvicetto <AT> gentoo <DOT> org>
CommitDate: Thu Apr 18 08:04:50 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=08c9ca74

Whitespace.

---
 scripts/copy_buildsync.sh |   12 ++++++------
 1 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 3d998e6..4375a3d 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -62,11 +62,11 @@ for ARCH in $ARCHES; do
 	latest_iso_date="$(echo -e "${iso_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
 	latest_stage3_date="$(echo -e "${stage3_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
 	header="$(echo -e "# Latest as of $(date -uR)\n# ts=$(date -u +%s)")"
-	
+
 	# Do not remove this
 	[ -z "${latest_iso_date}" ] && latest_iso_date="NONE-FOUND"
 	[ -z "${latest_stage3_date}" ] && latest_stage3_date="NONE-FOUND"
-	
+
 	if [ -n "${iso_list}" ]; then
 		echo -e "${header}" >"${OUT_ISO}"
 		echo -e "${iso_list}" |awk '{print $3}' | grep "$latest_iso_date" >>${OUT_ISO}
@@ -83,11 +83,11 @@ for ARCH in $ARCHES; do
 			ln -sf "$latest_stage3_date" current-stage3
 		fi
 	fi
-	
+
 	# new variant preserve code
 	variants="$(find 20* \( -iname '*.iso' -o -iname '*.tar.bz2' \) -printf '%f\n' |sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)"
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
-	for v in $variants ; do 
+	for v in $variants ; do
 		#date_variant=$(find 20* -iname "${v}*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -printf '%h\n' | sed -e "s,.*/$a/autobuilds/,,g" -e 's,/.*,,g' |sort -n | tail -n1 )
 		variant_path=$(find 20* -iname "${v}-*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
 		f="latest-${v}.txt"
@@ -96,7 +96,7 @@ for ARCH in $ARCHES; do
 		rm -f "current-$v"
 		ln -sf "${variant_path%/*}" "current-$v"
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${tmpdir}"/.keep.${ARCH}.txt
-	done 
+	done
 	#echo "$date_variant" \
 	#| sort | uniq | sed -e 's,^,/,g' -e 's,$,$,g' >"${tmpdir}"/.keep.${ARCH}.txt
 
@@ -115,7 +115,7 @@ for ARCH in $ARCHES; do
 
 			$DEBUGP rm $VERBOSEP -rf $(pwd)/${i}
 		done
-		
+
 		$DEBUGP rm $VERBOSEP -rf ${tmpdir}
 
 	else

[gentoo-commits] proj/releng:master commit in: scripts/

[Raúl Porcel]

commit:     609e096a7d18f6434fc32ca5ba12da92e96c9e22
Author:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
AuthorDate: Sun Aug  4 18:01:17 2013 +0000
Commit:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
CommitDate: Sun Aug  4 18:01:17 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=609e096a

Add m68k

---
 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 4375a3d..4b48e0f 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-ARCHES="alpha amd64 arm hppa ia64      ppc sparc x86 sh s390"
+ARCHES="alpha amd64 arm hppa ia64      ppc sparc x86 sh s390 m68k"
        #alpha amd64 arm hppa ia64 mips ppc s390 sh sparc x86
 #ARCHES="s390"
 RSYNC_OPTS="-aO --delay-updates"

[gentoo-commits] proj/releng:master commit in: scripts/

[Raúl Porcel]

commit:     0e32c9e4356bec2dbc83104304e9b4d40eef3628
Author:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  5 12:02:44 2013 +0000
Commit:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
CommitDate: Mon Aug  5 12:02:44 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=0e32c9e4

To pull a new arch something must be done from infra side, and since m68k isn't automated and its not in a good state, I'll push them to experimental instead

---
 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 4b48e0f..4375a3d 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-ARCHES="alpha amd64 arm hppa ia64      ppc sparc x86 sh s390 m68k"
+ARCHES="alpha amd64 arm hppa ia64      ppc sparc x86 sh s390"
        #alpha amd64 arm hppa ia64 mips ppc s390 sh sparc x86
 #ARCHES="s390"
 RSYNC_OPTS="-aO --delay-updates"

[gentoo-commits] proj/releng:master commit in: scripts/

[Raúl Porcel]

commit:     6cfdd7f82b3519b47026f532fb9bf94321d2d2f8
Author:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 17 16:51:54 2013 +0000
Commit:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
CommitDate: Sat Aug 17 16:51:54 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=6cfdd7f8

Narrow the regex a bit so f.ex stage3-amd64-nomultilib doesn't show up in latest-stage3-amd4.txt

---
 scripts/copy_buildsync.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 4375a3d..6b3f83f 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -88,8 +88,7 @@ for ARCH in $ARCHES; do
 	variants="$(find 20* \( -iname '*.iso' -o -iname '*.tar.bz2' \) -printf '%f\n' |sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)"
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do
-		#date_variant=$(find 20* -iname "${v}*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -printf '%h\n' | sed -e "s,.*/$a/autobuilds/,,g" -e 's,/.*,,g' |sort -n | tail -n1 )
-		variant_path=$(find 20* -iname "${v}-*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		variant_path=$(find 20* -iname "${v}-20*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
 		f="latest-${v}.txt"
 		echo -e "${header}" >"${f}"
 		echo -e "${variant_path}" >>${f}

[gentoo-commits] proj/releng:master commit in: scripts/

[Raúl Porcel]

commit:     35ff9cd6161b0121e2ced6baa5553d68c56093cd
Author:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 17 18:58:44 2013 +0000
Commit:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
CommitDate: Sat Aug 17 18:58:44 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=35ff9cd6

Use the variant function for the generic latest-stage3.txt

---
 scripts/copy_buildsync.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 6b3f83f..5c4ac37 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -75,7 +75,8 @@ for ARCH in $ARCHES; do
 	fi
 	if [ -n "${stage3_list}" ]; then
 		echo -e "${header}" >"${OUT_STAGE3}"
-		echo -e "${stage3_list}" |awk '{print $3}' |grep "$latest_stage3_date" >>${OUT_STAGE3}
+		# In the new variant preserve code there is a better way to do this
+		#echo -e "${stage3_list}" |awk '{print $3}' |grep "$latest_stage3_date" >>${OUT_STAGE3}
 		rm -f current-stage3
 		# The "latest stage3" concept doesn't apply to the arm variants
 		# that are pushed on different days of the week.
@@ -92,6 +93,7 @@ for ARCH in $ARCHES; do
 		f="latest-${v}.txt"
 		echo -e "${header}" >"${f}"
 		echo -e "${variant_path}" >>${f}
+		echo -e "${variant_path}" >>${OUT_STAGE3}
 		rm -f "current-$v"
 		ln -sf "${variant_path%/*}" "current-$v"
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${tmpdir}"/.keep.${ARCH}.txt

[gentoo-commits] proj/releng:master commit in: scripts/

[Raúl Porcel]

commit:     b7a016e9ca3f91441f76774a4ecebfb346f954b1
Author:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 17 19:13:22 2013 +0000
Commit:     Raúl Porcel <armin76 <AT> gentoo <DOT> org>
CommitDate: Sat Aug 17 19:13:22 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=b7a016e9

Do not use current-stage3 symlink on arm/hppa/s390/sh since releases are pushed with different datestamp and therefore current-stage3 only links to the latest that was pushed

---
 scripts/copy_buildsync.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 5c4ac37..39f7e19 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -78,9 +78,9 @@ for ARCH in $ARCHES; do
 		# In the new variant preserve code there is a better way to do this
 		#echo -e "${stage3_list}" |awk '{print $3}' |grep "$latest_stage3_date" >>${OUT_STAGE3}
 		rm -f current-stage3
-		# The "latest stage3" concept doesn't apply to the arm variants
+		# The "latest stage3" concept doesn't apply to the arm/hppa/s390/sh variants
 		# that are pushed on different days of the week.
-		if [[ ! $(echo ${outdir} | grep arm) ]]; then
+		if [[ ! $(echo ${outdir} | egrep 'arm|hppa|ppc|s390|sh') ]]; then
 			ln -sf "$latest_stage3_date" current-stage3
 		fi
 	fi

[gentoo-commits] proj/releng:master commit in: scripts/

[Jorge Manuel B. S. Vicetto]

commit:     b7c90993c194e2c458e5eaabd2a7dd24f8fb0966
Author:     Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 27 03:07:49 2014 +0000
Commit:     Jorge Manuel B. S. Vicetto <jmbsvicetto <AT> gentoo <DOT> org>
CommitDate: Thu Mar 27 03:07:49 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=b7c90993

Disable latest-stage3 for amd64 and x86 to avoid confusion for users when some sets fail to build.

Signed-off-by: Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo.org>

---
 scripts/copy_buildsync.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 39f7e19..45eddb2 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -80,7 +80,8 @@ for ARCH in $ARCHES; do
 		rm -f current-stage3
 		# The "latest stage3" concept doesn't apply to the arm/hppa/s390/sh variants
 		# that are pushed on different days of the week.
-		if [[ ! $(echo ${outdir} | egrep 'arm|hppa|ppc|s390|sh') ]]; then
+		# Disable it for amd64/x86 as well as any failures cause confusion to users
+		if [[ ! $(echo ${outdir} | egrep 'amd64|arm|hppa|ppc|s390|sh|x86') ]]; then
 			ln -sf "$latest_stage3_date" current-stage3
 		fi
 	fi

[gentoo-commits] proj/releng:master commit in: scripts/

[Jorge Manuel B. S. Vicetto]

commit:     9b17aafc0a23515b0c6e243128d996371d626aa5
Author:     Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Fri Feb  6 17:33:26 2015 +0000
Commit:     Jorge Manuel B. S. Vicetto <jmbsvicetto <AT> gentoo <DOT> org>
CommitDate: Fri Feb  6 17:33:26 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=9b17aafc

Sort arches.

Signed-off-by: Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo.org>

---
 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 45eddb2..b0b2b61 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-ARCHES="alpha amd64 arm hppa ia64      ppc sparc x86 sh s390"
+ARCHES="alpha amd64 arm hppa ia64      ppc s390 sh sparc x86"
        #alpha amd64 arm hppa ia64 mips ppc s390 sh sparc x86
 #ARCHES="s390"
 RSYNC_OPTS="-aO --delay-updates"

[gentoo-commits] proj/releng:master commit in: scripts/

[Jorge Manuel B. S. Vicetto]

commit:     e0189f9b76508e61844cffa11a2ac89adfae28e5
Author:     Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Fri Feb  6 20:11:54 2015 +0000
Commit:     Jorge Manuel B. S. Vicetto <jmbsvicetto <AT> gentoo <DOT> org>
CommitDate: Fri Feb  6 20:11:54 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/releng.git;a=commit;h=e0189f9b

Update the script to improve the readbility of the latest-{iso,stage3} for amd64/x86.

Signed-off-by: Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo.org>

---
 scripts/copy_buildsync.sh | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index b0b2b61..30b25a6 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -69,9 +69,11 @@ for ARCH in $ARCHES; do
 
 	if [ -n "${iso_list}" ]; then
 		echo -e "${header}" >"${OUT_ISO}"
-		echo -e "${iso_list}" |awk '{print $3}' | grep "$latest_iso_date" >>${OUT_ISO}
-		rm -f current-iso
-		ln -sf "$latest_iso_date" current-iso
+		if [[ ! $(echo ${iso_list} | egrep "amd64|x86") ]]; then
+			echo -e "${iso_list}" |awk '{print $3}' | grep "$latest_iso_date" >>${OUT_ISO}
+			rm -f current-iso
+			ln -sf "$latest_iso_date" current-iso
+		fi
 	fi
 	if [ -n "${stage3_list}" ]; then
 		echo -e "${header}" >"${OUT_STAGE3}"
@@ -91,10 +93,12 @@ for ARCH in $ARCHES; do
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do
 		variant_path=$(find 20* -iname "${v}-20*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		size=$(stat --format=%s ${variant_path})
 		f="latest-${v}.txt"
 		echo -e "${header}" >"${f}"
-		echo -e "${variant_path}" >>${f}
-		echo -e "${variant_path}" >>${OUT_STAGE3}
+		echo -e "${variant_path} ${size}" >>${f}
+		[[ ${variant_path} =~ tar.*$ ]] && echo -e "${variant_path} ${size}" >>${OUT_STAGE3}
+		[[ ${variant_path} =~ iso$ ]] && echo -e "${variant_path} ${size}" >>${OUT_ISO}
 		rm -f "current-$v"
 		ln -sf "${variant_path%/*}" "current-$v"
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${tmpdir}"/.keep.${ARCH}.txt

[gentoo-commits] proj/releng:master commit in: scripts/

[Jorge Manuel B. S. Vicetto]

commit:     260f36c938374d4169a50ce21d48953daf8ed5cc
Author:     Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Fri Oct  2 22:43:36 2015 +0000
Commit:     Jorge Manuel B. S. Vicetto <jmbsvicetto <AT> gentoo <DOT> org>
CommitDate: Fri Oct  2 22:43:36 2015 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=260f36c9

Update copy_buildsync.sh script to support stages with other suffixes.

Signed-off-by: Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo.org>

 scripts/copy_buildsync.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 30b25a6..30ae721 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -6,6 +6,7 @@ ARCHES="alpha amd64 arm hppa ia64      ppc s390 sh sparc x86"
 RSYNC_OPTS="-aO --delay-updates"
 DEBUG=
 VERBOSE=
+EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
 
 OUT_STAGE3="latest-stage3.txt"
 OUT_ISO="latest-iso.txt"
@@ -58,7 +59,7 @@ for ARCH in $ARCHES; do
 	# %T@
 
 	iso_list="$(find 20* -name '*.iso' -printf '%h %f %h/%f\n' |grep -v hardened | sort -n)"
-	stage3_list="$(find 20* -name 'stage3*bz2' -printf '%h %f %h/%f\n' |grep -v hardened | sort -n)"
+	stage3_list=$(find 20* -name "stage3*${EXTENSIONS}" -printf '%h %f %h/%f\n' | grep -v hardened | sort -n)
 	latest_iso_date="$(echo -e "${iso_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
 	latest_stage3_date="$(echo -e "${stage3_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
 	header="$(echo -e "# Latest as of $(date -uR)\n# ts=$(date -u +%s)")"
@@ -89,10 +90,10 @@ for ARCH in $ARCHES; do
 	fi
 
 	# new variant preserve code
-	variants="$(find 20* \( -iname '*.iso' -o -iname '*.tar.bz2' \) -printf '%f\n' |sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)"
+	variants=$(find 20* \( -iname '*.iso' -o -iname "*${EXTENSIONS}" \) -printf '%f\n' | sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do
-		variant_path=$(find 20* -iname "${v}-20*" \( -name '*.tar.bz2' -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		variant_path=$(find 20* -iname "${v}-20*" \( -name "*${EXTENSIONS}" -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
 		size=$(stat --format=%s ${variant_path})
 		f="latest-${v}.txt"
 		echo -e "${header}" >"${f}"

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     3b8ed31f0863a19d1da80f9b86ea7983d9f65369
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 20 17:30:13 2015 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Tue Oct 20 17:30:13 2015 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=3b8ed31f

Bail in the variant loop if the variant_path comes out empty, as seen in cron mail from the script.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 30ae721..3ef2e50 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -94,6 +94,10 @@ for ARCH in $ARCHES; do
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do
 		variant_path=$(find 20* -iname "${v}-20*" \( -name "*${EXTENSIONS}" -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		if [ -z "${variant_path}" -o ! -e "${variant_path}" ]; then
+			echo "$ARCH: Variant ${v} is missing" 1>&2
+			continue
+		fi
 		size=$(stat --format=%s ${variant_path})
 		f="latest-${v}.txt"
 		echo -e "${header}" >"${f}"

[gentoo-commits] proj/releng:master commit in: scripts/

[Jorge Manuel B. S. Vicetto]

commit:     1a3bf718107273f946c0f49f84951d940e91d41c
Author:     Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 22 09:59:09 2015 +0000
Commit:     Jorge Manuel B. S. Vicetto <jmbsvicetto <AT> gentoo <DOT> org>
CommitDate: Thu Oct 22 09:59:09 2015 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=1a3bf718

Remove old code, clean-up and cosmetic changes.

Signed-off-by: Jorge Manuel B. S. Vicetto (jmbsvicetto) <jmbsvicetto <AT> gentoo.org>

 scripts/copy_buildsync.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 3ef2e50..69b08c9 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -2,7 +2,6 @@
 
 ARCHES="alpha amd64 arm hppa ia64      ppc s390 sh sparc x86"
        #alpha amd64 arm hppa ia64 mips ppc s390 sh sparc x86
-#ARCHES="s390"
 RSYNC_OPTS="-aO --delay-updates"
 DEBUG=
 VERBOSE=
@@ -15,6 +14,7 @@ OUT_ISO="latest-iso.txt"
 
 DEBUGP=
 VERBOSEP=
+
 [ -n "$DEBUG" ] && DEBUGP=echo
 [ -n "$DEBUG" ] && RSYNC_OPTS="${RSYNC_OPTS} -n"
 [ -n "$VERBOSE" ] && RSYNC_OPTS="${RSYNC_OPTS} -v"
@@ -29,6 +29,7 @@ for ARCH in $ARCHES; do
 	tmpdir=/release/distfiles/tmp/buildsync/partial/${ARCH}
 
 	mkdir -p ${tmpdir} 2>/dev/null
+
 	# Copying
 	if [ -d "${indir}" ]; then
 		for i in $(find ${indir} -type f | grep -- '-20[0123][0-9]\{5\}' | sed -e 's:^.*-\(20[^.]\+\).*$:\1:' | sort -ur); do
@@ -78,9 +79,11 @@ for ARCH in $ARCHES; do
 	fi
 	if [ -n "${stage3_list}" ]; then
 		echo -e "${header}" >"${OUT_STAGE3}"
+
 		# In the new variant preserve code there is a better way to do this
 		#echo -e "${stage3_list}" |awk '{print $3}' |grep "$latest_stage3_date" >>${OUT_STAGE3}
 		rm -f current-stage3
+
 		# The "latest stage3" concept doesn't apply to the arm/hppa/s390/sh variants
 		# that are pushed on different days of the week.
 		# Disable it for amd64/x86 as well as any failures cause confusion to users
@@ -89,7 +92,7 @@ for ARCH in $ARCHES; do
 		fi
 	fi
 
-	# new variant preserve code
+	# New variant preserve code
 	variants=$(find 20* \( -iname '*.iso' -o -iname "*${EXTENSIONS}" \) -printf '%f\n' | sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do
@@ -108,15 +111,13 @@ for ARCH in $ARCHES; do
 		ln -sf "${variant_path%/*}" "current-$v"
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${tmpdir}"/.keep.${ARCH}.txt
 	done
-	#echo "$date_variant" \
-	#| sort | uniq | sed -e 's,^,/,g' -e 's,$,$,g' >"${tmpdir}"/.keep.${ARCH}.txt
 
 	# ================================================================
 	# Cleanup
 	if [ $fail -eq 0 ]; then
+
 		# Clean up all but latest 4 from mirror dir
 		cd "${outdir}"
-		#echo regex "/${latest_iso_date}\$|/${latest_stage3_date}\$"
 		for i in $(find -regextype posix-basic -mindepth 1 -maxdepth 1 -type d -regex '.*20[012][0-9]\{5\}.*' \
 				| sed -e 's:^.*-\(20[^.]\+\).*$:\1:' \
 				| sort -ur \

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     d14007c774e6d255e8a3fdc5890b06b1629f8348
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 20:46:11 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 20:46:11 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=d14007c7

copy_buildsync.sh: convert code to use arrays

This makes it a bit cleaner to work with arguments.

 scripts/copy_buildsync.sh | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 69b08c9..6105197 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -1,8 +1,22 @@
 #!/bin/bash
 
-ARCHES="alpha amd64 arm hppa ia64      ppc s390 sh sparc x86"
-       #alpha amd64 arm hppa ia64 mips ppc s390 sh sparc x86
-RSYNC_OPTS="-aO --delay-updates"
+ARCHES=(
+	alpha
+	amd64
+	arm
+	hppa
+	ia64
+	#mips
+	ppc
+	s390
+	sh
+	sparc
+	x86
+)
+RSYNC_OPTS=(
+	-aO
+	--delay-updates
+)
 DEBUG=
 VERBOSE=
 EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
@@ -16,11 +30,11 @@ DEBUGP=
 VERBOSEP=
 
 [ -n "$DEBUG" ] && DEBUGP=echo
-[ -n "$DEBUG" ] && RSYNC_OPTS="${RSYNC_OPTS} -n"
-[ -n "$VERBOSE" ] && RSYNC_OPTS="${RSYNC_OPTS} -v"
+[ -n "$DEBUG" ] && RSYNC_OPTS+=( -n )
+[ -n "$VERBOSE" ] && RSYNC_OPTS+=( -v )
 [ -n "$VERBOSEP" ] && VERBOSEP="-v"
 
-for ARCH in $ARCHES; do
+for ARCH in "${ARCHES[@]}"; do
 	rc=0
 	fail=0
 
@@ -36,7 +50,7 @@ for ARCH in $ARCHES; do
 			#echo "Doing $i"
 			t="${outdir}/${i}"
 			mkdir -p ${t} 2>/dev/null
-			rsync ${RSYNC_OPTS} --temp-dir=${tmpdir} --partial-dir=${tmpdir} ${indir}/ --filter "S *${i}*" --filter 'S **/' --filter 'H *' ${t}
+			rsync "${RSYNC_OPTS[@]}" --temp-dir=${tmpdir} --partial-dir=${tmpdir} ${indir}/ --filter "S *${i}*" --filter 'S **/' --filter 'H *' ${t}
 			rc=$?
 			if [ $rc -eq 0 ]; then
 				find ${indir} -type f -name "*${i}*" -print0 | xargs -0 --no-run-if-empty $DEBUGP rm $VERBOSEP -f

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     a1b63fb90eddf827e465205f1f853236032ba2e3
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 21:10:47 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 21:10:47 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=a1b63fb9

copy_buildsync.sh: convert code to use funcs

This makes the code a bit easier to follow and break up.

 scripts/copy_buildsync.sh | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 6105197..a9c18a0 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -34,7 +34,21 @@ VERBOSEP=
 [ -n "$VERBOSE" ] && RSYNC_OPTS+=( -v )
 [ -n "$VERBOSEP" ] && VERBOSEP="-v"
 
-for ARCH in "${ARCHES[@]}"; do
+usage() {
+	cat <<EOF
+Usage: $0
+
+Move releases from the incoming upload directory to the outgoing release
+directory so they can be pushed out to mirrors.
+Also update the "latest" links/files so people can easily find the current
+version for any particular arch/release.
+EOF
+	exit 1
+}
+
+process_arch() {
+	local ARCH=$1
+
 	rc=0
 	fail=0
 
@@ -147,7 +161,18 @@ for ARCH in "${ARCHES[@]}"; do
 	else
 		echo "There was some failure for $ARCH during the weekly sync. Not doing cleanup for fear of dataloss." 1>&2
 	fi
+}
 
-done
+main() {
+	if [[ $# -ne 0 ]]; then
+		usage
+	fi
+
+	local arch
+	for arch in "${ARCHES[@]}"; do
+		process_arch "${arch}"
+	done
+}
+main "$@"
 
 # vim:ts=2 sw=2 noet ft=sh:

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     2c172ab2489162b77d7e2f60642ae8b42db3e761
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 21:18:25 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 21:18:25 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=2c172ab2

copy_buildsync.sh: add command line options for verbose/debug mode

This makes it easier to do some debugging on the fly w/out having to
edit the script directly.

 scripts/copy_buildsync.sh | 39 +++++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index a9c18a0..b2e5f8a 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -17,8 +17,6 @@ RSYNC_OPTS=(
 	-aO
 	--delay-updates
 )
-DEBUG=
-VERBOSE=
 EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
 
 OUT_STAGE3="latest-stage3.txt"
@@ -29,21 +27,20 @@ OUT_ISO="latest-iso.txt"
 DEBUGP=
 VERBOSEP=
 
-[ -n "$DEBUG" ] && DEBUGP=echo
-[ -n "$DEBUG" ] && RSYNC_OPTS+=( -n )
-[ -n "$VERBOSE" ] && RSYNC_OPTS+=( -v )
-[ -n "$VERBOSEP" ] && VERBOSEP="-v"
-
 usage() {
 	cat <<EOF
-Usage: $0
+Usage: $0 [options]
 
 Move releases from the incoming upload directory to the outgoing release
 directory so they can be pushed out to mirrors.
 Also update the "latest" links/files so people can easily find the current
 version for any particular arch/release.
+
+Options:
+  -v, --verbose    Run in verbose mode
+  -d, --debug      Run in debug mode
 EOF
-	exit 1
+	exit ${1:-1}
 }
 
 process_arch() {
@@ -164,10 +161,28 @@ process_arch() {
 }
 
 main() {
-	if [[ $# -ne 0 ]]; then
-		usage
-	fi
+	# Process all the command line options first.
+	while [[ $# -ne 0 ]]; do
+		case $1 in
+		-d|--debug)
+			DEBUGP="echo"
+			RSYNC_OPTS+=( -n )
+			;;
+		-v|--verbose)
+			VERBOSEP="-v"
+			RSYNC_OPTS+=( -v )
+			;;
+		-h|--help)
+			usage 0
+			;;
+		*)
+			usage 1
+			;;
+		esac
+		shift
+	done
 
+	# Process all the architectures.
 	local arch
 	for arch in "${ARCHES[@]}"; do
 		process_arch "${arch}"

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     6325704c0812f0752b94e4e23e807860bae6f184
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 21:44:59 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 21:44:59 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=6325704c

copy_buildsync.sh: convert input copying logic to a func

Minor clean up ... should not be any real functional changes.

 scripts/copy_buildsync.sh | 68 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 25 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index b2e5f8a..927a1fd 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -1,5 +1,12 @@
 #!/bin/bash
 
+# Where artifacts are uploaded by builders.
+INCOMING_BASE="/release/weekly/builds"
+# Where artifacts are moved to so they can be uploaded to mirrors.
+OUTGOING_BASE="/release/distfiles/weekly"
+# Scratch space used when moving files from incoming to outgoing.
+TMPDIR_BASE="/release/distfiles/tmp/buildsync/partial"
+
 ARCHES=(
 	alpha
 	amd64
@@ -43,39 +50,50 @@ EOF
 	exit ${1:-1}
 }
 
+# Copy artifacts for an arch to the outgoing directory.
+copy_arch_to_outgoing() {
+	local ARCH=$1 indir=$2 outdir=$3 tmpdir=$4
+	local i t rc
+
+	if [[ ! -d ${indir} ]]; then
+		# Nothing to do for this arch.
+		return
+	fi
+
+	# Copying
+	for i in $(find ${indir} -type f | grep -- '-20[0123][0-9]\{5\}' | sed -e 's:^.*-\(20[^.]\+\).*$:\1:' | sort -ur); do
+		#echo "Doing $i"
+		t="${outdir}/${i}"
+		mkdir -p ${t} 2>/dev/null
+		rsync "${RSYNC_OPTS[@]}" --temp-dir=${tmpdir} --partial-dir=${tmpdir} ${indir}/ --filter "S *${i}*" --filter 'S **/' --filter 'H *' ${t}
+		rc=$?
+		if [ $rc -eq 0 ]; then
+			find ${indir} -type f -name "*${i}*" -print0 | xargs -0 --no-run-if-empty $DEBUGP rm $VERBOSEP -f
+		else
+			echo "Not deleting ${indir}/*${i}*, rsync failed!" 1>&2
+			fail=1
+		fi
+	done
+	find ${outdir} -mindepth 1 -type d \
+		| egrep -v current \
+		| sort -r \
+		| tr '\n' '\0' \
+		| xargs -0 --no-run-if-empty rmdir --ignore-fail-on-non-empty
+}
+
 process_arch() {
 	local ARCH=$1
 
-	rc=0
 	fail=0
 
-	indir=/release/weekly/builds/${ARCH}
-	outdir=/release/distfiles/weekly/${ARCH}
-	tmpdir=/release/distfiles/tmp/buildsync/partial/${ARCH}
+	indir="${INCOMING_BASE}/${ARCH}"
+	outdir="${OUTGOING_BASE}/${ARCH}"
+	tmpdir="${TMPDIR_BASE}/${ARCH}"
 
 	mkdir -p ${tmpdir} 2>/dev/null
 
-	# Copying
-	if [ -d "${indir}" ]; then
-		for i in $(find ${indir} -type f | grep -- '-20[0123][0-9]\{5\}' | sed -e 's:^.*-\(20[^.]\+\).*$:\1:' | sort -ur); do
-			#echo "Doing $i"
-			t="${outdir}/${i}"
-			mkdir -p ${t} 2>/dev/null
-			rsync "${RSYNC_OPTS[@]}" --temp-dir=${tmpdir} --partial-dir=${tmpdir} ${indir}/ --filter "S *${i}*" --filter 'S **/' --filter 'H *' ${t}
-			rc=$?
-			if [ $rc -eq 0 ]; then
-				find ${indir} -type f -name "*${i}*" -print0 | xargs -0 --no-run-if-empty $DEBUGP rm $VERBOSEP -f
-			else
-				echo "Not deleting ${indir}/*${i}*, rsync failed!" 1>&2
-				fail=1
-			fi
-		done
-		find ${outdir} -mindepth 1 -type d \
-			| egrep -v current \
-			| sort -r \
-			| tr '\n' '\0' \
-			|xargs -0 --no-run-if-empty rmdir --ignore-fail-on-non-empty
-	fi
+	# Sync incoming->outgoing first.
+	copy_arch_to_outgoing "${ARCH}" "${indir}" "${outdir}" "${tmpdir}"
 
 	# ================================================================
 	# Build data for revealing latest:

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     ffa42e7a104454b10466badcbc0539fcfdfc27b4
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 22:29:45 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 22:29:45 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=ffa42e7a

copy_buildsync.sh: simplify the latest-iso symlink checks

 scripts/copy_buildsync.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 4d9f0b1..1d45921 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -112,11 +112,18 @@ process_arch() {
 
 	if [ -n "${iso_list}" ]; then
 		echo -e "${header}" >"${OUT_ISO}"
-		if [[ ! $(echo ${iso_list} | egrep "amd64|x86") ]]; then
-			echo -e "${iso_list}" |awk '{print $3}' | grep "$latest_iso_date" >>${OUT_ISO}
+		# Some arches produce more than one type of iso.
+		# Only apply the current-iso link logic to them.
+		# TODO: Should make this dynamic based on the iso list.
+		case ${ARCH} in
+		amd64|x86)
 			rm -f current-iso
-			ln -sf "$latest_iso_date" current-iso
-		fi
+			;;
+		*)
+			echo -e "${iso_list}" |awk '{print $3}' | grep "$latest_iso_date" >>${OUT_ISO}
+			ln -sfT "$latest_iso_date" current-iso
+			;;
+		esac
 	fi
 	if [ -n "${stage3_list}" ]; then
 		echo -e "${header}" >"${OUT_STAGE3}"

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     867545ebd8fb51535dd2a715aac033830d29f112
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 21:55:08 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 21:55:08 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=867545eb

copy_buildsync.sh: simplify find clean up

Use find options to process everything rather than running through
grep/sort/tr/xargs.

 scripts/copy_buildsync.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 927a1fd..b4351cb 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -74,11 +74,9 @@ copy_arch_to_outgoing() {
 			fail=1
 		fi
 	done
-	find ${outdir} -mindepth 1 -type d \
-		| egrep -v current \
-		| sort -r \
-		| tr '\n' '\0' \
-		| xargs -0 --no-run-if-empty rmdir --ignore-fail-on-non-empty
+	find "${outdir}" \
+		-depth -mindepth 1 -type d \
+		-exec rmdir --ignore-fail-on-non-empty {} +
 }
 
 process_arch() {

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     7c16ed27d6de4c92dd6c172e00997f5e0a17ccb7
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 22:12:57 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 22:12:57 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=7c16ed27

copy_buildsync.sh: simplify the latest-stage3 symlink checks

The current code is a bit overwrought when it's a simple string compare.

 scripts/copy_buildsync.sh | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index b4351cb..4d9f0b1 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -123,14 +123,17 @@ process_arch() {
 
 		# In the new variant preserve code there is a better way to do this
 		#echo -e "${stage3_list}" |awk '{print $3}' |grep "$latest_stage3_date" >>${OUT_STAGE3}
-		rm -f current-stage3
 
-		# The "latest stage3" concept doesn't apply to the arm/hppa/s390/sh variants
-		# that are pushed on different days of the week.
-		# Disable it for amd64/x86 as well as any failures cause confusion to users
-		if [[ ! $(echo ${outdir} | egrep 'amd64|arm|hppa|ppc|s390|sh|x86') ]]; then
-			ln -sf "$latest_stage3_date" current-stage3
-		fi
+		# The "latest stage3" concept works for only a few arches -- ones that
+		# do not have more than one stage3 target per arch (i.e. multilib, etc...).
+		case ${ARCH} in
+		amd64|arm|hppa|ppc|s390|sh|x86)
+			rm -f current-stage3
+			;;
+		*)
+			ln -sfT "$latest_stage3_date" current-stage3
+			;;
+		esac
 	fi
 
 	# New variant preserve code

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     62f25f4995cb2a7b37a4bce84bb781a9451d8e70
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 23:37:44 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 23:37:44 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=62f25f49

copy_buildsync.sh: fix extension handling

The change to use a list of file extensions w/find doesn't actually work:
the [] operators sets up a group of chars to match, and it isn't split by
the comma char.  It's equiv to doing "*[,.2abfgrstxz]".

 scripts/copy_buildsync.sh | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 1d45921..ac823fe 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -24,7 +24,16 @@ RSYNC_OPTS=(
 	-aO
 	--delay-updates
 )
-EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
+# Command line for `find` to figure out what files are release artifacts.
+EXTENSIONS=(
+	'('
+	-name '*.tar.xz' -o
+	-name '*.tar.bz2' -o
+	-name '*.tar.gz' -o
+	-name '*.tar' -o
+	-name '*.sfs'
+	')'
+)
 
 OUT_STAGE3="latest-stage3.txt"
 OUT_ISO="latest-iso.txt"
@@ -101,7 +110,7 @@ process_arch() {
 	# %T@
 
 	iso_list="$(find 20* -name '*.iso' -printf '%h %f %h/%f\n' |grep -v hardened | sort -n)"
-	stage3_list=$(find 20* -name "stage3*${EXTENSIONS}" -printf '%h %f %h/%f\n' | grep -v hardened | sort -n)
+	stage3_list=$(find 20* -name "stage3*" -a "${EXTENSIONS[@]}" -printf '%h %f %h/%f\n' | grep -v hardened | sort -n)
 	latest_iso_date="$(echo -e "${iso_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
 	latest_stage3_date="$(echo -e "${stage3_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
 	header="$(echo -e "# Latest as of $(date -uR)\n# ts=$(date -u +%s)")"
@@ -144,10 +153,10 @@ process_arch() {
 	fi
 
 	# New variant preserve code
-	variants=$(find 20* \( -iname '*.iso' -o -iname "*${EXTENSIONS}" \) -printf '%f\n' | sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)
+	variants=$(find 20* \( -iname '*.iso' -o "${EXTENSIONS[@]}" \) -printf '%f\n' | sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do
-		variant_path=$(find 20* -iname "${v}-20*" \( -name "*${EXTENSIONS}" -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		variant_path=$(find 20* -iname "${v}-20*" \( "${EXTENSIONS[@]}" -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
 		if [ -z "${variant_path}" -o ! -e "${variant_path}" ]; then
 			echo "$ARCH: Variant ${v} is missing" 1>&2
 			continue

[gentoo-commits] proj/releng:master commit in: scripts/

[Mike Frysinger]

commit:     2736e468eb55e5e46fd448bb60c922d43ef12ac0
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 20 23:51:56 2016 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Sun Mar 20 23:51:56 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=2736e468

copy_buildsync.sh: handle netboot artifacts

Make sure we process netboot kernels as releases too.

 scripts/copy_buildsync.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index ac823fe..a917b9d 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -153,10 +153,11 @@ process_arch() {
 	fi
 
 	# New variant preserve code
-	variants=$(find 20* \( -iname '*.iso' -o "${EXTENSIONS[@]}" \) -printf '%f\n' | sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort | uniq)
+	find_variants=( '(' -iname '*.iso' -o -name 'netboot-*' -o "${EXTENSIONS[@]}" ')' )
+	variants=$(find 20* "${find_variants[@]}" -printf '%f\n' | sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort -u)
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do
-		variant_path=$(find 20* -iname "${v}-20*" \( "${EXTENSIONS[@]}" -o -iname '*.iso' \) -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		variant_path=$(find 20* -iname "${v}-20*" "${find_variants[@]}" -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
 		if [ -z "${variant_path}" -o ! -e "${variant_path}" ]; then
 			echo "$ARCH: Variant ${v} is missing" 1>&2
 			continue

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     060a341e103a3e2801d8f0f6f3b69dad2378072b
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Fri May  6 18:31:55 2016 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Fri May  6 18:31:55 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=060a341e

copy_buildsync: squelch errors.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index a917b9d..2b5c1ef 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -109,8 +109,8 @@ process_arch() {
 	cd "${outdir}"
 	# %T@
 
-	iso_list="$(find 20* -name '*.iso' -printf '%h %f %h/%f\n' |grep -v hardened | sort -n)"
-	stage3_list=$(find 20* -name "stage3*" -a "${EXTENSIONS[@]}" -printf '%h %f %h/%f\n' | grep -v hardened | sort -n)
+	iso_list="$(find 20* -name '*.iso' -printf '%h %f %h/%f\n' 2>/dev/null |grep -v hardened | sort -n)"
+	stage3_list=$(find 20* -name "stage3*" -a "${EXTENSIONS[@]}" -printf '%h %f %h/%f\n' 2>/dev/null| grep -v hardened | sort -n)
 	latest_iso_date="$(echo -e "${iso_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
 	latest_stage3_date="$(echo -e "${stage3_list}" |awk '{print $1}' |cut -d/ -f1 | tail -n1)"
 	header="$(echo -e "# Latest as of $(date -uR)\n# ts=$(date -u +%s)")"
@@ -154,10 +154,10 @@ process_arch() {
 
 	# New variant preserve code
 	find_variants=( '(' -iname '*.iso' -o -name 'netboot-*' -o "${EXTENSIONS[@]}" ')' )
-	variants=$(find 20* "${find_variants[@]}" -printf '%f\n' | sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort -u)
+	variants=$(find 20* "${find_variants[@]}" -printf '%f\n' 2>/dev/null | sed  -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort -u)
 	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
 	for v in $variants ; do
-		variant_path=$(find 20* -iname "${v}-20*" "${find_variants[@]}" -print | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
+		variant_path=$(find 20* -iname "${v}-20*" "${find_variants[@]}" -print 2>/dev/null | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
 		if [ -z "${variant_path}" -o ! -e "${variant_path}" ]; then
 			echo "$ARCH: Variant ${v} is missing" 1>&2
 			continue

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     69b94063ec91bddce79573c5b776c05efece51fd
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Fri May 27 20:55:55 2016 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Fri May 27 20:55:55 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=69b94063

emerge must use repos.conf now, instead of PORTDIR.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/update_auto_tree     | 14 +++++++++++++-
 scripts/update_official_tree | 14 +++++++++++++-
 scripts/update_snapshot_tree | 14 +++++++++++++-
 3 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/scripts/update_auto_tree b/scripts/update_auto_tree
index 08909e7..a68d561 100755
--- a/scripts/update_auto_tree
+++ b/scripts/update_auto_tree
@@ -1,2 +1,14 @@
 #!/bin/bash
-PORTDIR="/release/trees/portage-auto/" FEATURES="$FEATURES -news" emerge --sync -q
+PORTDIR="/release/trees/portage-auto/"
+TMPREPO=$(mktemp)
+trap "rm -f $TMPREPO" EXIT
+cat >$f <<EOF
+[DEFAULT]
+main-repo = gentoo
+
+[gentoo]
+location = $PORTDIR
+sync-type = rsync
+sync-uri = rsync://rsync.gentoo.org/gentoo-portage
+EOF
+PORTAGE_REPOSITORIES="$(cat $TMPREPO)" FEATURES="$FEATURES -news" emerge --sync -q

diff --git a/scripts/update_official_tree b/scripts/update_official_tree
index 250e905..3424cb6 100755
--- a/scripts/update_official_tree
+++ b/scripts/update_official_tree
@@ -1,2 +1,14 @@
 #!/bin/bash
-PORTDIR="/release/trees/portage-official/" emerge --sync
+PORTDIR="/release/trees/portage-official/"
+TMPREPO=$(mktemp)
+trap "rm -f $TMPREPO" EXIT
+cat >$f <<EOF
+[DEFAULT]
+main-repo = gentoo
+
+[gentoo]
+location = $PORTDIR
+sync-type = rsync
+sync-uri = rsync://rsync.gentoo.org/gentoo-portage
+EOF
+PORTAGE_REPOSITORIES="$(cat $TMPREPO)" FEATURES="$FEATURES -news" emerge --sync -q

diff --git a/scripts/update_snapshot_tree b/scripts/update_snapshot_tree
index f64742d..3a5284a 100755
--- a/scripts/update_snapshot_tree
+++ b/scripts/update_snapshot_tree
@@ -1,2 +1,14 @@
 #!/bin/bash
-PORTDIR="/release/trees/portage-snapshot/" emerge --sync
+PORTDIR="/release/trees/portage-snapshot/"
+TMPREPO=$(mktemp)
+trap "rm -f $TMPREPO" EXIT
+cat >$f <<EOF
+[DEFAULT]
+main-repo = gentoo
+
+[gentoo]
+location = $PORTDIR
+sync-type = rsync
+sync-uri = rsync://rsync.gentoo.org/gentoo-portage
+EOF
+PORTAGE_REPOSITORIES="$(cat $TMPREPO)" FEATURES="$FEATURES -news" emerge --sync -q

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     17b064f0b9e4e6a2e86a17ec7155e9616da2965c
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Fri May 27 21:01:39 2016 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Fri May 27 21:01:39 2016 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=17b064f0

update*tree: fix variable.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/update_auto_tree     | 2 +-
 scripts/update_official_tree | 2 +-
 scripts/update_snapshot_tree | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/update_auto_tree b/scripts/update_auto_tree
index a68d561..029ef70 100755
--- a/scripts/update_auto_tree
+++ b/scripts/update_auto_tree
@@ -2,7 +2,7 @@
 PORTDIR="/release/trees/portage-auto/"
 TMPREPO=$(mktemp)
 trap "rm -f $TMPREPO" EXIT
-cat >$f <<EOF
+cat >$TMPREPO <<EOF
 [DEFAULT]
 main-repo = gentoo
 

diff --git a/scripts/update_official_tree b/scripts/update_official_tree
index 3424cb6..55ecea0 100755
--- a/scripts/update_official_tree
+++ b/scripts/update_official_tree
@@ -2,7 +2,7 @@
 PORTDIR="/release/trees/portage-official/"
 TMPREPO=$(mktemp)
 trap "rm -f $TMPREPO" EXIT
-cat >$f <<EOF
+cat >$TMPREPO <<EOF
 [DEFAULT]
 main-repo = gentoo
 

diff --git a/scripts/update_snapshot_tree b/scripts/update_snapshot_tree
index 3a5284a..76cd00e 100755
--- a/scripts/update_snapshot_tree
+++ b/scripts/update_snapshot_tree
@@ -2,7 +2,7 @@
 PORTDIR="/release/trees/portage-snapshot/"
 TMPREPO=$(mktemp)
 trap "rm -f $TMPREPO" EXIT
-cat >$f <<EOF
+cat >$TMPREPO <<EOF
 [DEFAULT]
 main-repo = gentoo
 

[gentoo-commits] proj/releng:master commit in: scripts/

[Matt Turner]

commit:     c9ad265f80178b8786974e0d1e250daa42e6a742
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 21 06:43:38 2018 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Nov 21 19:07:26 2018 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=c9ad265f

scripts/copy_buildsync: try to exclude rsync tempfile

In-progress uploads are named .<filename>.<rsynctemp> and were
mistakenly recognized by copy_buildsync.sh as targets to sweep to the
mirrors. When it did, it deleted the file causing the in-progress upload
to fail.

Since the script is run by cron every 15 minutes, this would lead to a
large portion of the generated installation media to fail to upload, and
worse there is no pattern to it -- it just happens whenever an upload
spans the 15 minute boundary.

The fix is to ignore hidden files and to tell rsync (via the filter
rules) to exclude them from its sweep to the mirrors.

Closes: https://bugs.gentoo.org/670458
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index d67cf6ee..6c6085ff 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -70,14 +70,14 @@ copy_arch_to_outgoing() {
 	fi
 
 	# Copying
-	for i in $(find ${indir} -type f | egrep -- '-20[0123][0-9]{5}(([0-9]{6})|(T[0-9]{6}Z))?' | sed -e 's:^.*-\(20[^.]\+\).*$:\1:' | sort -ur); do
+	for i in $(find ${indir} -not -path '*/\.*' -type f | egrep -- '-20[0123][0-9]{5}(([0-9]{6})|(T[0-9]{6}Z))?' | sed -e 's:^.*-\(20[^.]\+\).*$:\1:' | sort -ur); do
 		#echo "Doing $i"
 		t="${outdir}/${i}"
 		mkdir -p ${t} 2>/dev/null
-		rsync "${RSYNC_OPTS[@]}" --temp-dir=${tmpdir} --partial-dir=${tmpdir} ${indir}/ --filter "S *${i}*" --filter 'S **/' --filter 'H *' ${t}
+		rsync "${RSYNC_OPTS[@]}" --temp-dir=${tmpdir} --partial-dir=${tmpdir} ${indir}/ --filter '- **/.*' --filter "S *${i}*" --filter 'S **/' --filter 'H *' ${t}
 		rc=$?
 		if [ $rc -eq 0 ]; then
-			find ${indir} -type f -name "*${i}*" -print0 | xargs -0 --no-run-if-empty $DEBUGP rm $VERBOSEP -f
+			find ${indir} -not -path '*/\.*' -type f -name "*${i}*" -print0 | xargs -0 --no-run-if-empty $DEBUGP rm $VERBOSEP -f
 		else
 			echo "Not deleting ${indir}/*${i}*, rsync failed!" 1>&2
 			fail=1

[gentoo-commits] proj/releng:master commit in: scripts/

[Matt Turner]

commit:     9fa4c8f32c637c88bf5c0b7b3259781aba1f68a6
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 21 06:55:35 2018 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Nov 21 19:13:02 2018 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=9fa4c8f3

scripts/copy_buildsync: cleanup excludes of tempfile

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 37 ++++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 6c6085ff..815d21de 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -62,7 +62,7 @@ EOF
 # Copy artifacts for an arch to the outgoing directory.
 copy_arch_to_outgoing() {
 	local ARCH=$1 indir=$2 outdir=$3 tmpdir=$4
-	local i t rc
+	local i t rc timestamps
 
 	if [[ ! -d ${indir} ]]; then
 		# Nothing to do for this arch.
@@ -70,21 +70,48 @@ copy_arch_to_outgoing() {
 	fi
 
 	# Copying
-	for i in $(find ${indir} -not -path '*/\.*' -type f | egrep -- '-20[0123][0-9]{5}(([0-9]{6})|(T[0-9]{6}Z))?' | sed -e 's:^.*-\(20[^.]\+\).*$:\1:' | sort -ur); do
+	timestamps=( $(
+		find "${indir}" \
+				-regextype posix-egrep \
+				-type f \
+				-regex '.*-20[0123][0-9]{5}(([0-9]{6})|(T[0-9]{6}Z))?.*' \
+				\( -not -path '*/\.*' \) \
+		| sed -e 's:^.*-\(20[^.]\+\).*$:\1:' \
+		| sort -ur
+		) )
+
+	for i in "${timestamps[@]}" ; do
 		#echo "Doing $i"
 		t="${outdir}/${i}"
 		mkdir -p ${t} 2>/dev/null
-		rsync "${RSYNC_OPTS[@]}" --temp-dir=${tmpdir} --partial-dir=${tmpdir} ${indir}/ --filter '- **/.*' --filter "S *${i}*" --filter 'S **/' --filter 'H *' ${t}
+		rsync \
+			"${RSYNC_OPTS[@]}" \
+			--temp-dir="${tmpdir}" \
+			--partial-dir="${tmpdir}" \
+			--filter '- **/.*' \
+			--filter "S *${i}*" \
+			--filter 'S **/' \
+			--filter 'H *' \
+			${indir}/ \
+			${t}
 		rc=$?
 		if [ $rc -eq 0 ]; then
-			find ${indir} -not -path '*/\.*' -type f -name "*${i}*" -print0 | xargs -0 --no-run-if-empty $DEBUGP rm $VERBOSEP -f
+			find "${indir}" \
+				-type f \
+				-name "*${i}*" \
+				\( -not -path '*/\.*' \) \
+				-print0 \
+				| xargs -0 --no-run-if-empty \
+				"$DEBUGP" rm "$VERBOSEP" -f
 		else
 			echo "Not deleting ${indir}/*${i}*, rsync failed!" 1>&2
 			fail=1
 		fi
 	done
 	find "${outdir}" \
-		-depth -mindepth 1 -type d \
+		-depth \
+		-mindepth 1 \
+		-type d \
 		-exec rmdir --ignore-fail-on-non-empty {} +
 }
 

[gentoo-commits] proj/releng:master commit in: scripts/

[Matt Turner]

commit:     a4b2ec7de2e43454a969a41f1a3ad22bb9cb18aa
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 21 07:06:48 2018 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Wed Nov 21 19:13:05 2018 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=a4b2ec7d

scripts/copy_buildsync: shellcheck pass

[mattst88]: Small fixes
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 46 +++++++++++++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 815d21de..5e4046c4 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -56,7 +56,7 @@ Options:
   -v, --verbose    Run in verbose mode
   -d, --debug      Run in debug mode
 EOF
-	exit ${1:-1}
+	exit "${1:-1}"
 }
 
 # Copy artifacts for an arch to the outgoing directory.
@@ -83,7 +83,7 @@ copy_arch_to_outgoing() {
 	for i in "${timestamps[@]}" ; do
 		#echo "Doing $i"
 		t="${outdir}/${i}"
-		mkdir -p ${t} 2>/dev/null
+		mkdir -p "${t}" 2>/dev/null
 		rsync \
 			"${RSYNC_OPTS[@]}" \
 			--temp-dir="${tmpdir}" \
@@ -92,8 +92,8 @@ copy_arch_to_outgoing() {
 			--filter "S *${i}*" \
 			--filter 'S **/' \
 			--filter 'H *' \
-			${indir}/ \
-			${t}
+			"${indir}"/ \
+			"${t}"
 		rc=$?
 		if [ $rc -eq 0 ]; then
 			find "${indir}" \
@@ -102,7 +102,7 @@ copy_arch_to_outgoing() {
 				\( -not -path '*/\.*' \) \
 				-print0 \
 				| xargs -0 --no-run-if-empty \
-				"$DEBUGP" rm "$VERBOSEP" -f
+				$DEBUGP rm $VERBOSEP -f
 		else
 			echo "Not deleting ${indir}/*${i}*, rsync failed!" 1>&2
 			fail=1
@@ -124,7 +124,7 @@ process_arch() {
 	outdir="${OUTGOING_BASE}/${ARCH}"
 	tmpdir="${TMPDIR_BASE}/${ARCH}"
 
-	mkdir -p ${tmpdir} 2>/dev/null
+	mkdir -p "${tmpdir}" 2>/dev/null
 
 	# Sync incoming->outgoing first.
 	copy_arch_to_outgoing "${ARCH}" "${indir}" "${outdir}" "${tmpdir}"
@@ -182,23 +182,31 @@ process_arch() {
 	# New variant preserve code
 	find_variants=( '(' -iname '*.iso' -o -name 'netboot-*' -o "${EXTENSIONS[@]}" ')' )
 	variants=$(find 20* "${find_variants[@]}" -printf '%f\n' 2>/dev/null | sed -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort -u)
-	echo -n '' >"${tmpdir}"/.keep.${ARCH}.txt
+	keepfile="${tmpdir}/.keep.${ARCH}.txt"
+	keepfile_tmp=$(mktemp -p "${tmpdir}" -t ".keep.${ARCH}.txt.XXXXXX")
+	echo -n '' >"${keepfile_tmp}"
+	chmod 644 "${keepfile_tmp}"
 	for v in $variants ; do
+		# FIXME: trace the $a variable in this!
 		variant_path=$(find 20* -iname "${v}-20*" "${find_variants[@]}" -print 2>/dev/null | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
-		if [ -z "${variant_path}" -o ! -e "${variant_path}" ]; then
+		if [ -z "${variant_path}" ] || [ ! -e "${variant_path}" ]; then
 			echo "$ARCH: Variant ${v} is missing" 1>&2
 			continue
 		fi
-		size=$(stat --format=%s ${variant_path})
+		size=$(stat --format='%s' "${variant_path}")
 		f="latest-${v}.txt"
-		echo -e "${header}" >"${f}"
-		echo -e "${variant_path} ${size}" >>${f}
-		[[ ${variant_path} =~ tar.*$ ]] && echo -e "${variant_path} ${size}" >>${OUT_STAGE3}
-		[[ ${variant_path} =~ iso$ ]] && echo -e "${variant_path} ${size}" >>${OUT_ISO}
+		f_tmp=$(mktemp -p . -t ".${f}.XXXXXX")
+		chmod 644 "${f_tmp}"
+		echo -e "${header}" >"${f_tmp}"
+		echo -e "${variant_path} ${size}" >>"${f_tmp}"
+		[[ ${variant_path} =~ tar.*$ ]] && echo -e "${variant_path} ${size}" >>"${OUT_STAGE3}" # FIXME: tempfile
+		[[ ${variant_path} =~ iso$ ]] && echo -e "${variant_path} ${size}" >>"${OUT_ISO}" # FIXME: tempfile
 		rm -f "current-$v"
 		ln -sf "${variant_path%/*}" "current-$v"
-		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${tmpdir}"/.keep.${ARCH}.txt
+		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${keepfile_tmp}"
+		mv -f "${f_tmp}" "${f}"
 	done
+	mv -f "${keepfile_tmp}" "${keepfile}"
 
 	# ================================================================
 	# Cleanup
@@ -206,17 +214,17 @@ process_arch() {
 
 		# Clean up all but latest 4 from mirror dir
 		cd "${outdir}"
-		for i in $(find -regextype posix-basic -mindepth 1 -maxdepth 1 -type d -regex '.*20[012][0-9]\{5\}.*' \
+		for i in $(find . -regextype posix-basic -mindepth 1 -maxdepth 1 -type d -regex '.*20[012][0-9]\{5\}.*' \
 				| sed -e 's:^.*-\(20[^.]\+\).*$:\1:' \
 				| sort -ur \
-				| egrep -v "/${latest_iso_date}\$|/${latest_stage3_date}\$" \
-				| egrep -v -f "${tmpdir}"/.keep.${ARCH}.txt \
+				| grep -E -v -e "/${latest_iso_date}\$|/${latest_stage3_date}\$" \
+				| grep -E -v -f "${keepfile}" \
 				| tail -n +5); do
 
-			$DEBUGP rm $VERBOSEP -rf $(pwd)/${i}
+			$DEBUGP rm $VERBOSEP -rf "$(pwd)"/"${i}"
 		done
 
-		$DEBUGP rm $VERBOSEP -rf ${tmpdir}
+		$DEBUGP rm $VERBOSEP -rf "${tmpdir}"
 
 	else
 		echo "There was some failure for $ARCH during the weekly sync. Not doing cleanup for fear of dataloss." 1>&2

[gentoo-commits] proj/releng:master commit in: scripts/

[Matt Turner]

commit:     e022f988665aa9d2697c8a84630afd469ae7849b
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Fri Aug 14 19:21:11 2020 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 14 19:21:11 2020 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=e022f988

scripts/copy_buildsync: Add arm64

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 5e4046c4..f7c0c6a3 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -11,6 +11,7 @@ ARCHES=(
 	alpha
 	amd64
 	arm
+	arm64
 	hppa
 	ia64
 	#mips

[gentoo-commits] proj/releng:master commit in: scripts/

[Anthony G. Basile]

commit:     c6a9395dbb74ac92292e17b5087d2787c03a7f8c
Author:     Anthony G. Basile <blueness <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 19 20:18:24 2020 +0000
Commit:     Anthony G. Basile <blueness <AT> gentoo <DOT> org>
CommitDate: Sat Dec 19 20:18:35 2020 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=c6a9395d

scripts/{cp-musl.sh,cp-uclibc.sh}: move from nightheron:/home/blueness

Signed-off-by: Anthony G. Basile <blueness <AT> gentoo.org>

 scripts/cp-musl.sh   | 21 +++++++++++++++++++++
 scripts/cp-uclibc.sh | 19 +++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/scripts/cp-musl.sh b/scripts/cp-musl.sh
new file mode 100755
index 00000000..93f51675
--- /dev/null
+++ b/scripts/cp-musl.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+CATDIR="/release/buildroot/alt-dev/builds/musl"
+SERVER="amd64@dipper:~/musl"
+LIST=$(find $CATDIR -maxdepth 3 -iname 'stage3*amd64*' -type l)
+#COMMAND="rsync -e ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no -a --omit-dir-times --delay-updates "
+COMMAND="rsync"
+for f in $LIST; do
+	$COMMAND $(realpath $f)             $SERVER
+	$COMMAND $(realpath $f).CONTENTS.gz $SERVER
+	$COMMAND $(realpath $f).DIGESTS     $SERVER
+done
+
+#SERVER="/release/weekly/builds/x86/musl"
+SERVER="x86@dipper:~/musl"
+LIST=$(find $CATDIR -maxdepth 3 -iname 'stage3*i686*' -type l)
+for f in $LIST; do
+	$COMMAND $(realpath $f)             $SERVER
+	$COMMAND $(realpath $f).CONTENTS.gz $SERVER
+	$COMMAND $(realpath $f).DIGESTS     $SERVER
+done

diff --git a/scripts/cp-uclibc.sh b/scripts/cp-uclibc.sh
new file mode 100755
index 00000000..0bfd24a7
--- /dev/null
+++ b/scripts/cp-uclibc.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+CATDIR="/release/buildroot/alt-dev/builds/uclibc"
+SERVER="amd64@dipper:~/uclibc"
+LIST=$(find $CATDIR -maxdepth 3 -iname 'stage3*amd64*' -type l)
+COMMAND="rsync"
+for f in $LIST; do
+	$COMMAND $(realpath $f)             $SERVER
+	$COMMAND $(realpath $f).CONTENTS.gz $SERVER
+	$COMMAND $(realpath $f).DIGESTS     $SERVER
+done
+
+SERVER="x86@dipper:~/uclibc"
+LIST=$(find $CATDIR -maxdepth 3 -iname 'stage3*i686*' -type l)
+for f in $LIST; do
+	$COMMAND $(realpath $f)             $SERVER
+	$COMMAND $(realpath $f).CONTENTS.gz $SERVER
+	$COMMAND $(realpath $f).DIGESTS     $SERVER
+done

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     8fa82be9c49fb997ffe23f622066bb20135666d3
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Sat Jan 23 14:53:37 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Sat Jan 23 14:53:37 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=8fa82be9

Add riscv to list of uploading arches

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index f7c0c6a3..f9637770 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -16,6 +16,7 @@ ARCHES=(
 	ia64
 	#mips
 	ppc
+	riscv
 	s390
 	sh
 	sparc

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     f28b31c5f04b57a294095e66b3b686984a692836
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Sun May  9 21:23:55 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Sun May  9 21:23:55 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=f28b31c5

Add script to pull and update musl overlay

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/update_musl_overlay | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/scripts/update_musl_overlay b/scripts/update_musl_overlay
new file mode 100755
index 00000000..4d62aef0
--- /dev/null
+++ b/scripts/update_musl_overlay
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+MUSLDIR="/release/trees/musl-auto"
+MUSLURI="https://anongit.gentoo.org/git/proj/musl.git"
+
+if [ ! -d "${MUSLDIR}" ] ; then
+	echo musl directory not present yet or not a directory, fixing
+	rm -rf "${MUSLDIR}"
+	git clone "${MUSLURI}" "${MUSLDIR}"
+else
+	cd "${MUSLDIR}"
+	git pull
+fi

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     5e6a94495ac4622e98c9c72c9d9077fbd78216e1
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 31 16:00:13 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Sat Jul 31 16:00:13 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=5e6a9449

Drop more musl special sauce

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/cp-musl.sh          | 21 ---------------------
 scripts/update_musl_overlay | 13 -------------
 2 files changed, 34 deletions(-)

diff --git a/scripts/cp-musl.sh b/scripts/cp-musl.sh
deleted file mode 100755
index 93f51675..00000000
--- a/scripts/cp-musl.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-CATDIR="/release/buildroot/alt-dev/builds/musl"
-SERVER="amd64@dipper:~/musl"
-LIST=$(find $CATDIR -maxdepth 3 -iname 'stage3*amd64*' -type l)
-#COMMAND="rsync -e ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no -a --omit-dir-times --delay-updates "
-COMMAND="rsync"
-for f in $LIST; do
-	$COMMAND $(realpath $f)             $SERVER
-	$COMMAND $(realpath $f).CONTENTS.gz $SERVER
-	$COMMAND $(realpath $f).DIGESTS     $SERVER
-done
-
-#SERVER="/release/weekly/builds/x86/musl"
-SERVER="x86@dipper:~/musl"
-LIST=$(find $CATDIR -maxdepth 3 -iname 'stage3*i686*' -type l)
-for f in $LIST; do
-	$COMMAND $(realpath $f)             $SERVER
-	$COMMAND $(realpath $f).CONTENTS.gz $SERVER
-	$COMMAND $(realpath $f).DIGESTS     $SERVER
-done

diff --git a/scripts/update_musl_overlay b/scripts/update_musl_overlay
deleted file mode 100755
index 4d62aef0..00000000
--- a/scripts/update_musl_overlay
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-MUSLDIR="/release/trees/musl-auto"
-MUSLURI="https://anongit.gentoo.org/git/proj/musl.git"
-
-if [ ! -d "${MUSLDIR}" ] ; then
-	echo musl directory not present yet or not a directory, fixing
-	rm -rf "${MUSLDIR}"
-	git clone "${MUSLURI}" "${MUSLDIR}"
-else
-	cd "${MUSLDIR}"
-	git pull
-fi

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     c55494ef41ec2002a7326caa90704f03eb0dfd7a
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 31 16:04:01 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Sat Jul 31 16:04:01 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=c55494ef

Remove uclibc files

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/cp-uclibc.sh | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/scripts/cp-uclibc.sh b/scripts/cp-uclibc.sh
deleted file mode 100755
index 0bfd24a7..00000000
--- a/scripts/cp-uclibc.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-CATDIR="/release/buildroot/alt-dev/builds/uclibc"
-SERVER="amd64@dipper:~/uclibc"
-LIST=$(find $CATDIR -maxdepth 3 -iname 'stage3*amd64*' -type l)
-COMMAND="rsync"
-for f in $LIST; do
-	$COMMAND $(realpath $f)             $SERVER
-	$COMMAND $(realpath $f).CONTENTS.gz $SERVER
-	$COMMAND $(realpath $f).DIGESTS     $SERVER
-done
-
-SERVER="x86@dipper:~/uclibc"
-LIST=$(find $CATDIR -maxdepth 3 -iname 'stage3*i686*' -type l)
-for f in $LIST; do
-	$COMMAND $(realpath $f)             $SERVER
-	$COMMAND $(realpath $f).CONTENTS.gz $SERVER
-	$COMMAND $(realpath $f).DIGESTS     $SERVER
-done

[gentoo-commits] proj/releng:master commit in: scripts/

[Matt Turner]

commit:     36db6e7c8748e2d346e1ef92362807b0c242df76
Author:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 31 01:31:21 2021 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Sat Jul 31 19:41:16 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=36db6e7c

scripts: Remove cruft

Presumably these were used once upon a time when catalyst builds were
done on infra-managed boxes (where puppet/chef made updating things
difficult).

Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>

 scripts/backup_snapshot_repo |  10 -
 scripts/cache-tools.py       | 700 -------------------------------------------
 scripts/run_catalyst         |   2 -
 scripts/run_official         |  39 ---
 scripts/run_snapshot         |   2 -
 scripts/stage_build.sh       | 162 ----------
 scripts/sudo_catalyst        |  28 --
 scripts/sudo_official        |  46 ---
 scripts/sudo_snapshot        |  20 --
 scripts/update_auto_tree     |  14 -
 scripts/update_official_tree |  14 -
 scripts/update_snapshot_tree |  14 -
 12 files changed, 1051 deletions(-)

diff --git a/scripts/backup_snapshot_repo b/scripts/backup_snapshot_repo
deleted file mode 100755
index 94b2828f..00000000
--- a/scripts/backup_snapshot_repo
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# Start our rsyncs
-RSYNC_OPTS="--archive --delete --sparse --whole-file"
-
-if [ -e /release/repos/snapshot-tree/hooks/post-commit ]
-then
-	echo "$(date): Starting rsync of trees from tmpfs to disk..." >> /var/log/snapshot-tree-backup.log
-	rsync ${RSYNC_OPTS} /release/repos/snapshot-tree/* /release/repos/snapshot-tree-disk 2>&1 >> /var/log/snapshot-tree-backup.log || echo "$(date): rsync failed!" >> /var/log/snapshot-tree-backup.log
-fi

diff --git a/scripts/cache-tools.py b/scripts/cache-tools.py
deleted file mode 100755
index 03644503..00000000
--- a/scripts/cache-tools.py
+++ /dev/null
@@ -1,700 +0,0 @@
-#!/usr/bin/env python
-# Copyright 1999-2006 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-# $Header: $
-#
-# Zac Medico <zmedico@gentoo.org>
-#
-
-import errno, fpformat, os, sys, time
-
-if not hasattr(__builtins__, "set"):
-	from sets import Set as set
-from itertools import chain
-
-def create_syncronized_func(myfunc, mylock):
-	def newfunc(*pargs, **kwargs):
-		mylock.acquire()
-		try:
-			myfunc(*pargs, **kwargs)
-		finally:
-			mylock.release()
-	return myfunc
-
-class ConsoleUpdate(object):
-
-	_synchronized_methods = ["append", "carriageReturn",
-		"newLine", "reset", "update"]
-
-	def __init__(self):
-		self.offset = 0
-		import sys
-		self.stream = sys.stdout
-		self.quiet = False
-		import threading
-		self._lock = threading.RLock()
-		for method_name in self._synchronized_methods:
-			myfunc = create_syncronized_func(
-				getattr(self, method_name), self._lock)
-			setattr(self, method_name, myfunc)
-		# ANSI code that clears from the cursor to the end of the line
-		self._CLEAR_EOL = None
-		try:
-			import curses
-			try:
-				curses.setupterm()
-				self._CLEAR_EOL = curses.tigetstr('el')
-			except curses.error:
-				pass
-		except ImportError:
-			pass
-		if not self._CLEAR_EOL:
-			self._CLEAR_EOL = '\x1b[K'
-
-	def acquire(self, **kwargs):
-		return self._lock.acquire(**kwargs)
-
-	def release(self):
-		self._lock.release()
-
-	def reset(self):
-		self.offset = 0
-
-	def carriageReturn(self):
-		if not self.quiet:
-			self.stream.write("\r")
-			self.stream.write(self._CLEAR_EOL)
-			self.offset = 0
-
-	def newLine(self):
-		if not self.quiet:
-			self.stream.write("\n")
-			self.stream.flush()
-			self.reset()
-
-	def update(self, msg):
-		if not self.quiet:
-			self.carriageReturn()
-			self.append(msg)
-
-	def append(self, msg):
-		if not self.quiet:
-			self.offset += len(msg)
-			self.stream.write(msg)
-			self.stream.flush()
-
-class ProgressCounter(object):
-	def __init__(self):
-		self.total = 0
-		self.current = 0
-
-class ProgressAnalyzer(ProgressCounter):
-	def __init__(self):
-		self.start_time = time.time()
-		self.currentTime = self.start_time
-		self._samples = []
-		self.sampleCount = 20
-	def percentage(self, digs=0):
-		if self.total > 0:
-			float_percent = 100 * float(self.current) / float(self.total)
-		else:
-			float_percent = 0.0
-		return fpformat.fix(float_percent,digs)
-	def totalTime(self):
-		self._samples.append((self.currentTime, self.current))
-		while len(self._samples) > self.sampleCount:
-			self._samples.pop(0)
-		prev_time, prev_count = self._samples[0]
-		time_delta = self.currentTime - prev_time
-		if time_delta > 0:
-			rate = (self.current - prev_count) / time_delta
-			if rate > 0:
-				return self.total / rate
-		return 0
-	def remaining_time(self):
-		return self.totalTime() - self.elapsed_time()
-	def elapsed_time(self):
-		return self.currentTime - self.start_time
-
-class ConsoleProgress(object):
-	def __init__(self, name="Progress", console=None):
-		self.name = name
-		self.analyzer = ProgressAnalyzer()
-		if console is None:
-			self.console = ConsoleUpdate()
-		else:
-			self.console = console
-		self.time_format="%H:%M:%S"
-		self.quiet = False
-		self.lastUpdate = 0
-		self.latency = 0.5
-
-	def formatTime(self, t):
-		return time.strftime(self.time_format, time.gmtime(t))
-
-	def displayProgress(self, current, total):
-		if self.quiet:
-			return
-
-		self.analyzer.currentTime = time.time()
-		if self.analyzer.currentTime - self.lastUpdate < self.latency:
-			return
-		self.lastUpdate = self.analyzer.currentTime
-		self.analyzer.current = current
-		self.analyzer.total = total
-
-		output = ((self.name, self.analyzer.percentage(1).rjust(4) + "%"),
-		("Elapsed", self.formatTime(self.analyzer.elapsed_time())),
-		("Remaining", self.formatTime(self.analyzer.remaining_time())),
-		("Total", self.formatTime(self.analyzer.totalTime())))
-		self.console.update(" ".join([ x[0] + ": " + x[1] for x in output ]))
-
-class ProgressHandler(object):
-	def __init__(self):
-		self.curval = 0
-		self.maxval = 0
-		self.last_update = 0
-		self.min_display_latency = 0.2
-
-	def onProgress(self, maxval, curval):
-		self.maxval = maxval
-		self.curval = curval
-		cur_time = time.time()
-		if cur_time - self.last_update >= self.min_display_latency:
-			self.last_update = cur_time
-			self.display()
-
-	def display(self):
-		raise NotImplementedError(self)
-
-def open_file(filename=None):
-	if filename is None:
-		f = sys.stderr
-	elif filename == "-":
-		f = sys.stdout
-	else:
-		try:
-			filename = os.path.expanduser(filename)
-			f = open(filename, "a")
-		except (IOError, OSError), e:
-			sys.stderr.write("%s\n" % e)
-			sys.exit(e.errno)
-	return f
-
-def create_log(name="", logfile=None, loglevel=0):
-	import logging
-	log = logging.getLogger(name)
-	log.setLevel(loglevel)
-	handler = logging.StreamHandler(open_file(logfile))
-	handler.setFormatter(logging.Formatter("%(levelname)s %(message)s"))
-	log.addHandler(handler)
-	return log
-
-def is_interrupt(e):
-	if isinstance(e, (SystemExit, KeyboardInterrupt)):
-		return True
-	return hasattr(e, "errno") and e.errno == errno.EINTR
-
-def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, log,
-	eclass_cache, cleanse_on_transfer_failure):
-
-	cleanse_candidates = set(trg_cache.iterkeys())
-	update_count = 0
-
-	# Since the loop below is mission critical, we continue after *any*
-	# exception that is not an interrupt.
-
-	for x in valid_nodes_iterable:
-		log.debug("%s mirroring" % x)
-		if not cleanse_on_transfer_failure:
-			cleanse_candidates.discard(x)
-
-		try:
-			entry = copy_dict(src_cache[x])
-		except KeyError, e:
-			log.error("%s missing source: %s" % (x, str(e)))
-			del e
-			continue
-		except Exception, e:
-			if is_interrupt(e):
-				raise
-			log.error("%s reading source: %s" % (x, str(e)))
-			del e
-			continue
-
-		write_it = True
-		trg = None
-
-		try:
-			trg = copy_dict(trg_cache[x])
-			if long(trg["_mtime_"]) == long(entry["_mtime_"]) and \
-				eclass_cache.is_eclass_data_valid(trg["_eclasses_"]) and \
-				set(trg["_eclasses_"]) == set(entry["_eclasses_"]):
-				write_it = False
-		except KeyError:
-			pass
-		except Exception, e:
-			if is_interrupt(e):
-				raise
-			log.error("%s reading target: %s" % (x, str(e)))
-			del e
-
-		if trg and not write_it:
-			""" We don't want to skip the write unless we're really sure that
-			the existing cache is identical, so don't trust _mtime_ and
-			_eclasses_ alone."""
-			for d in (entry, trg):
-				if "EAPI" in d and d["EAPI"] in ("", "0"):
-					del d["EAPI"]
-			for k in set(chain(entry, trg)).difference(
-				("_mtime_", "_eclasses_")):
-				if trg.get(k, "") != entry.get(k, ""):
-					write_it = True
-					break
-
-		if write_it:
-			update_count += 1
-			log.info("%s transferring" % x)
-			inherited = entry.get("INHERITED", None)
-			if inherited:
-				if src_cache.complete_eclass_entries:
-					if not "_eclasses_" in entry:
-						log.error("%s missing _eclasses_" % x)
-						continue
-					if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]):
-						log.error("%s stale _eclasses_" % x)
-						continue
-				else:
-					entry["_eclasses_"] = eclass_cache.get_eclass_data(entry["INHERITED"].split(), \
-						from_master_only=True)
-					if not entry["_eclasses_"]:
-						log.error("%s stale _eclasses_" % x)
-						continue
-			try:
-				trg_cache[x] = entry
-				cleanse_candidates.discard(x)
-			except Exception, e:
-				if is_interrupt(e):
-					raise
-				log.error("%s writing target: %s" % (x, str(e)))
-				del e
-		else:
-			cleanse_candidates.discard(x)
-
-	if not trg_cache.autocommits:
-		try:
-			trg_cache.commit()
-		except Exception, e:
-			if is_interrupt(e):
-				raise
-			log.error("committing target: %s" % str(e))
-			del e
-
-	return update_count, cleanse_candidates
-
-def copy_dict(src, dest=None):
-	"""Some cache implementations throw cache errors when accessing the values.
-	We grab all the values at once here so that we don't have to be concerned
-	about exceptions later."""
-	if dest is None:
-		dest = {}
-	for k, v in src.iteritems():
-		dest[k] = v
-	return dest
-
-class ListPackages(object):
-	def __init__(self, portdb, log, shuffle=False):
-		self._portdb = portdb
-		self._log = log
-		self._shuffle = shuffle
-
-	def run(self):
-		log = self._log
-		cp_list = self._portdb.cp_list
-		cp_all = self._portdb.cp_all()
-		if self._shuffle:
-			from random import shuffle
-			shuffle(cp_all)
-		else:
-			cp_all.sort()
-		cpv_all = []
-		# Since the loop below is mission critical, we continue after *any*
-		# exception that is not an interrupt.
-		for cp in cp_all:
-			log.debug("%s cp_list" % cp)
-			try:
-				cpv_all.extend(cp_list(cp))
-			except Exception, e:
-				if is_interrupt(e):
-					raise
-				self._log.error("%s cp_list: %s" % (cp, str(e)))
-
-		self.cpv_all = cpv_all
-
-class MetadataGenerate(object):
-	"""When cache generation fails for some reason, cleanse the stale cache
-	entry if it exists.  This prevents the master mirror from distributing
-	stale cache, and will allow clients to safely assume that all cache is
-	valid.  The mtime requirement is especially annoying due to bug #139134
-	(timestamps of cache entries don't change when an eclass changes) and the
-	interaction of timestamps with rsync."""
-	def __init__(self, portdb, cpv_all, log):
-		self._portdb = portdb
-		self._cpv_all = cpv_all
-		self._log = log
-
-	def run(self, onProgress=None):
-		log = self._log
-		portdb = self._portdb
-		cpv_all = self._cpv_all
-		auxdb = portdb.auxdb[portdb.porttree_root]
-		cleanse_candidates = set(auxdb.iterkeys())
-
-		# Since the loop below is mission critical, we continue after *any*
-		# exception that is not an interrupt.
-		maxval = len(cpv_all)
-		curval = 0
-		if onProgress:
-			onProgress(maxval, curval)
-		while cpv_all:
-			cpv = cpv_all.pop(0)
-			log.debug("%s generating" % cpv)
-			try:
-				portdb.aux_get(cpv, ["EAPI"])
-				# Cleanse if the above doesn't succeed (prevent clients from
-				# receiving stale cache, and let them assume it is valid).
-				cleanse_candidates.discard(cpv)
-			except Exception, e:
-				if is_interrupt(e):
-					raise
-				log.error("%s generating: %s" % (cpv, str(e)))
-				del e
-			curval += 1
-			if onProgress:
-				onProgress(maxval, curval)
-
-		self.target_cache = auxdb
-		self.dead_nodes = cleanse_candidates
-
-class MetadataTransfer(object):
-	def __init__(self, portdb, cpv_all, forward, cleanse_on_transfer_failure,
-		log):
-		self._portdb = portdb
-		self._cpv_all = cpv_all
-		self._log = log
-		self._forward = forward
-		self._cleanse_on_transfer_failure = cleanse_on_transfer_failure
-
-	def run(self, onProgress=None):
-		log = self._log
-		portdb = self._portdb
-		cpv_all = self._cpv_all
-		aux_cache = portdb.auxdb[portdb.porttree_root]
-		import portage
-		auxdbkeys = portage.auxdbkeys[:]
-		metadbmodule = portdb.mysettings.load_best_module("portdbapi.metadbmodule")
-		portdir_cache = metadbmodule(portdb.porttree_root, "metadata/cache",
-			auxdbkeys)
-
-		maxval = len(cpv_all)
-		curval = 0
-		if onProgress:
-			onProgress(maxval, curval)
-		class pkg_iter(object):
-			def __init__(self, pkg_list, onProgress=None):
-				self.pkg_list = pkg_list
-				self.maxval = len(pkg_list)
-				self.curval = 0
-				self.onProgress = onProgress
-			def __iter__(self):
-				while self.pkg_list:
-					yield self.pkg_list.pop()
-					self.curval += 1
-					if self.onProgress:
-						self.onProgress(self.maxval, self.curval)
-
-		if self._forward:
-			src_cache = portdir_cache
-			trg_cache = aux_cache
-		else:
-			src_cache = aux_cache
-			trg_cache = portdir_cache
-
-		""" This encapsulates validation of eclass timestamps and also fills in
-		missing data (mtimes and/or paths) as necessary for the given cache
-		format."""
-		eclass_cache = portage.eclass_cache.cache(portdb.porttree_root)
-
-		if not trg_cache.autocommits:
-			trg_cache.sync(100)
-
-		self.target_cache = trg_cache
-		self.update_count, self.dead_nodes = mirror_cache(
-			pkg_iter(cpv_all, onProgress=onProgress),
-			src_cache, trg_cache, log, eclass_cache,
-			self._cleanse_on_transfer_failure)
-
-class CacheCleanse(object):
-	def __init__(self, auxdb, dead_nodes, log):
-		self._auxdb = auxdb
-		self._dead_nodes = dead_nodes
-		self._log = log
-	def run(self):
-		auxdb = self._auxdb
-		log = self._log
-		for cpv in self._dead_nodes:
-			try:
-				log.info("%s cleansing" % cpv)
-				del auxdb[cpv]
-			except Exception, e:
-				if is_interrupt(e):
-					raise
-				log.error("%s cleansing: %s" % (cpv, str(e)))
-				del e
-
-def import_portage():
-	try:
-		from portage import data as portage_data
-	except ImportError:
-		import portage_data
-	# If we're not already root or in the portage group, we make the gid of the
-	# current process become portage_gid.
-	if os.getgid() != 0 and portage_data.portage_gid not in os.getgroups():
-		portage_data.portage_gid = os.getgid()
-		portage_data.secpass = 1
-
-	os.environ["PORTAGE_LEGACY_GLOBALS"] = "false"
-	import portage
-	del os.environ["PORTAGE_LEGACY_GLOBALS"]
-	return portage
-
-def create_portdb(portdir=None, cachedir=None, config_root=None,
-	target_root=None, profile=None, **kwargs):
-
-	if cachedir is not None:
-		os.environ["PORTAGE_DEPCACHEDIR"] = cachedir
-	if config_root is None:
-		config_root = os.environ.get("PORTAGE_CONFIGROOT", "/")
-	if target_root is None:
-		target_root = os.environ.get("ROOT", "/")
-	if profile is None:
-		profile = ""
-
-	portage = import_portage()
-	try:
-		from portage import const as portage_const
-	except ImportError:
-		import portage_const
-
-	# Disable overlays because we only generate metadata for the main repo.
-	os.environ["PORTDIR_OVERLAY"] = ""
-	conf = portage.config(config_profile_path=profile,
-		config_incrementals=portage_const.INCREMENTALS,
-		target_root=target_root,
-		config_root=config_root)
-
-	if portdir is None:
-		portdir = conf["PORTDIR"]
-
-	# The cannonical path is the key for portdb.auxdb.
-	portdir = os.path.realpath(portdir)
-	conf["PORTDIR"] = portdir
-	conf.backup_changes("PORTDIR")
-
-	portdb = portage.portdbapi(portdir,
-		mysettings=conf)
-
-	return portdb
-
-def parse_args(myargv):
-	description = "This program will ensure that the metadata cache is up to date for entire portage tree."
-	usage = "usage: cache-tools [options] --generate || --transfer"
-	from optparse import OptionParser
-	parser = OptionParser(description=description, usage=usage)
-	parser.add_option("--portdir",
-		help="location of the portage tree",
-		dest="portdir")
-	parser.add_option("--cachedir",
-		help="location of the metadata cache",
-		dest="cachedir")
-	parser.add_option("--profile",
-		help="location of the profile",
-		dest="profile")
-	parser.add_option("--generate",
-		help="generate metadata as necessary to ensure that the cache is fully populated",
-		action="store_true", dest="generate", default=False)
-	parser.add_option("--shuffle",
-		help="generate cache in random rather than sorted order (useful to prevent two separate instances from competing to generate metadata for the same packages simultaneously)",
-		action="store_true", dest="shuffle", default=False)
-	parser.add_option("--transfer",
-		help="transfer metadata from portdir to cachedir or vice versa",
-		action="store_true", dest="transfer", default=False)
-	parser.add_option("--cleanse-on-transfer-failure",
-		help="cleanse target cache when transfer fails for any reason (such as the source being unavailable)",
-		action="store_true", dest="cleanse_on_transfer_failure", default=False)
-	parser.add_option("--forward",
-		help="forward metadata transfer flows from portdir to cachedir (default)",
-		action="store_true", dest="forward", default=True)
-	parser.add_option("--reverse",
-		help="reverse metadata transfer flows from cachedir to portdir",
-		action="store_false", dest="forward", default=True)
-	parser.add_option("--logfile",
-		help="send status messages to a file (default is stderr)",
-		dest="logfile", default=None)
-	parser.add_option("--loglevel",
-		help="numeric log level (defauls to 0 and may range from 0 to 50 corresponding to the default levels of the python logging module)",
-		dest="loglevel", default="0")
-	parser.add_option("--reportfile",
-		help="send a report to a file",
-		dest="reportfile", default=None)
-	parser.add_option("--spawn-outfile",
-		help="redirect ouput of spawned processes to a file instead of stdout/stderr",
-		dest="spawn_outfile", default=None)
-	parser.add_option("--no-progress",
-		action="store_false", dest="progress", default=True,
-		help="disable progress output to tty")
-	options, args = parser.parse_args(args=myargv)
-
-	# Conversion to dict allows us to use **opts as function args later on.
-	opts = {}
-	all_options = ("portdir", "cachedir", "profile", "progress", "logfile",
-		"loglevel", "generate", "transfer", "forward", "shuffle",
-		"spawn_outfile", "reportfile", "cleanse_on_transfer_failure")
-	for opt_name in all_options:
-		v = getattr(options, opt_name)
-		opts[opt_name] = v
-	return opts
-
-def run_command(args):
-	opts = parse_args(sys.argv[1:])
-
-	if opts["spawn_outfile"]:
-		fd = os.dup(1)
-		sys.stdout = os.fdopen(fd, 'w')
-		fd = os.dup(2)
-		sys.stderr = os.fdopen(fd, 'w')
-		f = open_file(opts["spawn_outfile"])
-		os.dup2(f.fileno(), 1)
-		os.dup2(f.fileno(), 2)
-		del fd, f
-
-	console = ConsoleUpdate()
-	if not opts["progress"] or not sys.stdout.isatty():
-		console.quiet = True
-	job = None
-	import signal, thread, threading
-	shutdown_initiated = threading.Event()
-	shutdown_complete = threading.Event()
-	def shutdown_console():
-		console.acquire()
-		try:
-			console.update("Interrupted.")
-			console.newLine()
-			console.quiet = True
-			shutdown_complete.set()
-			# Kill the main thread if necessary.
-			# This causes the SIGINT signal handler to be invoked in the
-			# main thread.  The signal handler needs to be an actual
-			# callable object (rather than something like signal.SIG_DFL)
-			# in order to avoid TypeError: 'int' object is not callable.
-			thread.interrupt_main()
-			thread.exit()
-		finally:
-			console.release()
-
-	def handle_interrupt(*args):
-		if shutdown_complete.isSet():
-			sys.exit(1)
-		# Lock the console from a new thread so that the main thread is allowed
-		# to cleanly complete any console interaction that may have been in
-		# progress when this interrupt arrived.
-		if not shutdown_initiated.isSet():
-			thread.start_new_thread(shutdown_console, ())
-			shutdown_initiated.set()
-
-	signal.signal(signal.SIGINT, handle_interrupt)
-	signal.signal(signal.SIGTERM, handle_interrupt)
-
-	try:
-		import datetime
-		datestamp = str(datetime.datetime.now())
-		time_begin = time.time()
-		log = create_log(name="MetadataGenerate",
-			logfile=opts["logfile"], loglevel=int(opts["loglevel"]))
-		if opts["reportfile"]:
-			reportfile = open_file(opts["reportfile"])
-		portdb = create_portdb(**opts)
-		try:
-			os.nice(int(portdb.mysettings.get("PORTAGE_NICENESS", "0")))
-		except (OSError, ValueError), e:
-			log.error("PORTAGE_NICENESS failed: '%s'" % str(e))
-			del e
-
-		job = ListPackages(portdb, log, shuffle=opts["shuffle"])
-		console.update("Listing packages in repository...")
-		job.run()
-		cpv_all = job.cpv_all
-		total_count = len(cpv_all)
-		if opts["generate"]:
-			job = MetadataGenerate(portdb, cpv_all, log)
-			name = "Cache generation"
-			complete_msg = "Metadata generation is complete."
-		elif opts["transfer"]:
-			job = MetadataTransfer(portdb, cpv_all, opts["forward"],
-			opts["cleanse_on_transfer_failure"], log)
-			if opts["forward"]:
-				name = "Forward transfer"
-				complete_msg = "Forward metadata transfer is complete."
-			else:
-				name = "Reverse transfer"
-				complete_msg = "Reverse metadata transfer is complete."
-		else:
-			sys.stderr.write("required options: --generate || --transfer\n")
-			sys.exit(os.EX_USAGE)
-		job.opts = opts
-
-		onProgress = None
-		if not console.quiet:
-			ui = ConsoleProgress(name=name, console=console)
-			progressHandler = ProgressHandler()
-			onProgress = progressHandler.onProgress
-			def display():
-				ui.displayProgress(progressHandler.curval, progressHandler.maxval)
-			progressHandler.display = display
-
-		job.run(onProgress=onProgress)
-
-		if not console.quiet:
-			# make sure the final progress is displayed
-			progressHandler.display()
-
-		update_count = None
-		if opts["transfer"]:
-			update_count = job.update_count
-		target_cache = job.target_cache
-		dead_nodes = job.dead_nodes
-		cleanse_count = len(dead_nodes)
-		console.update("Cleansing cache...")
-		job = CacheCleanse(target_cache, dead_nodes, log)
-		job.run()
-		console.update(complete_msg)
-		console.newLine()
-		time_end = time.time()
-		if opts["reportfile"]:
-			width = 20
-			reportfile.write(name.ljust(width) + "%s\n" % datestamp)
-			reportfile.write("Elapsed seconds".ljust(width) + "%f\n" % (time_end - time_begin))
-			reportfile.write("Total packages".ljust(width) + "%i\n" % total_count)
-			if update_count is not None:
-				reportfile.write("Updated packages".ljust(width) + "%i\n" % update_count)
-			reportfile.write("Cleansed packages".ljust(width) + "%i\n" % cleanse_count)
-			reportfile.write(("-"*50)+"\n")
-	except Exception, e:
-		if not is_interrupt(e):
-			raise
-		del e
-		handle_interrupt()
-	sys.exit(0)
-
-if __name__ == "__main__":
-	run_command(sys.argv[1:])

diff --git a/scripts/run_catalyst b/scripts/run_catalyst
deleted file mode 100755
index 997f6520..00000000
--- a/scripts/run_catalyst
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/bash
-sudo /release/bin/sudo_catalyst "$@"

diff --git a/scripts/run_official b/scripts/run_official
deleted file mode 100755
index dfb29f25..00000000
--- a/scripts/run_official
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-
-email_from="auto"
-email_to="releng@gentoo.org"
-url="https://poseidon.amd64.dev.gentoo.org/snapshots"
-snapshot_uri="/release/webroot/snapshots"
-svn_repo="/release/repos/snapshot-tree"
-
-send_email() {
-  subject="[Snapshot] ${1}"
-
-  echo -e "From: ${email_from}\r\nTo: ${email_to}\r\nSubject: ${subject}\r\n\r\nA new snapshot has been built from revision `svnlook history ${svn_repo} | head -n 3 | tail -n 1 | sed -e 's:^ *::' -e 's: .*$::'` of ${svn_repo}.  You can find it at ${url}.\r\n\r\n$(cat /release/snapshots/portage-${1}.tar.bz2.DIGESTS)\r\n" | /usr/sbin/sendmail -f ${email_from} ${email_to}
-}
-
-if [ "${email_from}" == "auto" ]
-then
-	username="$(whoami)"
-	if [ "${username}" == "root" ]
-	then
-		email_from="catalyst@poseidon.amd64.dev.gentoo.org"
-	else
-		email_from="${username}@gentoo.org"
-	fi
-fi
-
-sudo /release/bin/sudo_official "$@" && \
-echo "Starting rsync from /release/snapshots/portage-${1}.tar.bz2* to ${snapshot_uri}" && \
-rsync --archive --stats --progress /release/snapshots/portage-${1}.tar.bz2* \
-	${snapshot_uri}
-ret=$?
-
-if [ "${email_from}" == "none" ]
-then
-	echo "Skipping email step as configured..."
-else
-	[ $ret -eq 0 ] && send_email ${1}
-fi
-
-exit $ret

diff --git a/scripts/run_snapshot b/scripts/run_snapshot
deleted file mode 100755
index 20cc4609..00000000
--- a/scripts/run_snapshot
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/bash
-sudo /release/bin/sudo_snapshot "$@"

diff --git a/scripts/stage_build.sh b/scripts/stage_build.sh
deleted file mode 100755
index 46902c1e..00000000
--- a/scripts/stage_build.sh
+++ /dev/null
@@ -1,162 +0,0 @@
-#!/bin/bash
-
-PID=$$
-
-profile=
-version_stamp=
-subarch=
-stage1_seed=
-snapshot=
-config=/etc/catalyst/catalyst.conf
-email_from="catalyst@localhost"
-email_to="root@localhost"
-verbose=0
-
-usage() {
-  msg=$1
-
-  if [ -n "${msg}" ]; then
-    echo -e "${msg}\n";
-  fi
-
-  cat <<EOH
-Usage:
-  stage_build [-p|--profile <profile>] [-v|--version-stamp <stamp>]
-              [-a|--arch <arch>] [-s|--stage1-seed <seed>] [--verbose]
-              [-f|--email-from <from>] [-t|--email-to <to>] [-h|--help]
-
-Options:
-  -p|--profile        Sets the portage profile (required)
-  -v|--version-stamp  Sets the version stamp (required)
-  -a|--arch           Sets the 'subarch' in the spec (required)
-  -s|--stage1-seed    Sets the seed for the stage1 (required)
-  -S|--snapshot       Sets the snapshot name (if not given defaults to today's
-                      date)
-  -c|--config         catalyst config to use, defaults to catalyst default
-  --verbose           Send output of commands to console as well as log
-  -f|--email-from     Sets the 'From' on emails sent from this script (defaults
-                      to catalyst@localhost)
-  -t|--email-to       Sets the 'To' on emails sent from this script (defaults
-                      to root@localhost)
-  -h|--help           Show this message and quit
-
-Example:
-  stage_build -p default-linux/x86/2006.1 -v 2007.0_pre -a i686 -s default/stage3-i686-2006.1
-EOH
-}
-
-send_email() {
-  subject="[${subarch}] $1"
-  body=$2
-
-  echo -e "From: ${email_from}\r\nTo: ${email_to}\r\nSubject: ${subject}\r\n\r\nArch: ${subarch}\r\nProfile: ${profile}\r\nVersion stamp: ${version_stamp}\r\nStage1 seed: ${stage1_seed}\r\nSnapshot: ${snapshot}\r\n\r\n${body}\r\n" | /usr/sbin/sendmail -f ${email_from} ${email_to}
-}
-
-run_cmd() {
-  cmd=$1
-  logfile=$2
-
-  if [ $verbose = 1 ]; then
-    ${cmd} 2>&1 | tee ${logfile}
-  else
-    ${cmd} &> ${logfile}
-  fi
-}
-
-# Parse args
-params=${#}
-while [ ${#} -gt 0 ]
-do
-  a=${1}
-  shift
-  case "${a}" in
-    -h|--help)
-      usage
-      exit 0
-      ;;
-    -p|--profile)
-      profile=$1
-      shift
-      ;;
-    -v|--version-stamp)
-      version_stamp=$1
-      shift
-      ;;
-    -a|--arch)
-      subarch=$1
-      shift
-      ;;
-    -f|--email-from)
-      email_from=$1
-      shift
-      ;;
-    -t|--email-to)
-      email_to=$1
-      shift
-      ;;
-    -s|--stage1-seed)
-      stage1_seed=$1
-      shift
-      ;;
-    -S|--snapshot)
-      snapshot=$1
-      shift
-      ;;
-    -c|--config)
-      config=$1
-      shift
-      ;;
-    --verbose)
-      verbose=1
-      ;;
-    -*)
-      echo "You have specified an invalid option: ${a}"
-      usage
-      exit 1
-      ;;
-    esac
-done
-
-# Make sure all required values were specified
-if [ -z "${profile}" ]; then
-  usage "You must specify a profile."
-  exit 1
-fi
-if [ -z "${version_stamp}" ]; then
-  usage "You must specify a version stamp."
-  exit 1
-fi
-if [ -z "${subarch}" ]; then
-  usage "You must specify an arch."
-  exit 1
-fi
-if [ -z "${stage1_seed}" ]; then
-  usage "You must specify a stage1 seed."
-  exit 1
-fi
-cd /tmp
-
-if [ -z "${snapshot}" ]; then
-  snapshot=`date +%Y%m%d`
-  run_cmd "catalyst -c ${config} -s '${snapshot}'" "/tmp/catalyst_build_snapshot.${PID}.log"
-  if [ $? != 0 ]; then
-    send_email "Catalyst build error - snapshot" "$(</tmp/catalyst_build_snapshot.${PID}.log)"
-    exit 1
-  fi
-fi
-
-for i in 1 2 3; do
-  echo -e "subarch: ${subarch}\ntarget: stage${i}\nversion_stamp: ${version_stamp}\nrel_type: default\nprofile: ${profile}\nsnapshot: ${snapshot}" > stage${i}.spec
-  if [ ${i} = 1 ]; then
-    echo "source_subpath: ${stage1_seed}" >> stage${i}.spec
-  else 
-    echo "source_subpath: default/stage$(expr ${i} - 1)-${subarch}-${version_stamp}" >> stage${i}.spec
-  fi
-  run_cmd "catalyst -a -c ${config} -f stage${i}.spec" "/tmp/catalyst_build_stage${i}.${PID}.log"
-  if [ $? != 0 ]; then
-    send_email "Catalyst build error - stage${i}" "$(tail -n 200 /tmp/catalyst_build_stage${i}.${PID}.log)\r\n\r\nFull build log at /tmp/catalyst_build_stage${i}.${PID}.log"
-    exit 1
-  fi
-done
-
-send_email "Catalyst build success" "Everything finished successfully."

diff --git a/scripts/sudo_catalyst b/scripts/sudo_catalyst
deleted file mode 100755
index 19ecc90a..00000000
--- a/scripts/sudo_catalyst
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-usage() {
-	echo "Usage: $(basename ${0}) <arch> <target> <spec>"
-	echo "Where arch is either amd64 or x86, target is default, dev, hardened,"
-	echo "or uclibc, and spec is your spec file."
-	echo
-}
-
-if [ -z "$1" -o -z "$2" -o -z "$3" ]
-then
-	usage
-else
-	target="$(grep target ${3} | cut -d' ' -f2)"
-	/usr/bin/catalyst -c /etc/catalyst/${1}-${2}.conf -f ${3} ${4} ${5}
-#	&& \
-#	case ${target} in
-#		stage*|grp*|livecd-stage2)
-#			echo "Cleaning out ${target} temp files"
-#			rel_type="$(grep rel_type ${3} | cut -d' ' -f2)"
-#			subarch="$(grep subarch ${3} | cut -d' ' -f2)"
-#			version="$(grep version ${3} | cut -d' ' -f2)"
-#			storedir="$(grep storedir /etc/catalyst/${1}-${2}.conf | cut -d\" -f2)"
-#			echo "Removing ${storedir}/tmp/${rel_type}/${target}-${subarch}-${version}"
-#			rm -rf ${storedir}/tmp/${rel_type}/${target}-${subarch}-${version}
-#		;;
-#	esac
-fi

diff --git a/scripts/sudo_official b/scripts/sudo_official
deleted file mode 100755
index 80e7ca04..00000000
--- a/scripts/sudo_official
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-tree="/release/trees/snapshot-tree"
-portdir="${tree}/${1/_beta2/}/portage"
-cache_args="--portdir=${portdir} --cachedir=/release/tmp/depcache"
-
-usage() {
-	echo "Usage: $(basename ${0}) <version>"
-}
-
-if [ -z "${1}" ]
-then
-	usage
-else
-	cd ${tree}
-	echo "Clearing out old metadata cache"
-	rm -rf ${portdir}/metadata/cache
-	echo "Performing a svn up on ${tree}"
-	svn up || exit 1
-	mkdir -p ${portdir}/metadata/cache
-	echo "Recreating portage metadata cache"
-	cache-tools.py ${cache_args} --generate || exit 1
-	cache-tools.py ${cache_args} --transfer --reverse \
-		--cleanse-on-transfer-failure || exit 1
-	if [ ! -d ${portdir}/metadata/cache/sys-kernel ]
-	then
-		echo "Metadata update failed! Bailing out!"
-		exit 1
-	fi
-	catalyst -c /etc/catalyst/snapshot-official.conf -s ${1} \
-		-C portdir="${portdir}" || exit 1
-	for i in amd64 x86
-	do
-		for j in default dev hardened uclibc
-		do
-			cd /release/buildroot/${i}-${j}/snapshots
-			rm -f portage-official.tar.bz2 portage-${1}.tar.bz2*
-			ln -sf /release/snapshots/portage-${1}.tar.bz2 \
-				portage-official.tar.bz2
-			ln -sf /release/snapshots/portage-${1}.tar.bz2 \
-				portage-${1}.tar.bz2
-			ln -sf /release/snapshots/portage-${1}.tar.bz2.DIGESTS \
-				portage-${1}.tar.bz2.DIGESTS
-		done
-	done
-fi

diff --git a/scripts/sudo_snapshot b/scripts/sudo_snapshot
deleted file mode 100755
index 1ba64855..00000000
--- a/scripts/sudo_snapshot
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-usage() {
-	echo "Usage: $(basename ${0}) <version>"
-}
-if [ -z "${1}" ]
-then
-	usage
-else
-	catalyst -c /etc/catalyst/snapshot.conf -s ${1}
-	for i in amd64 x86
-	do
-		for j in default dev hardened uclibc
-		do
-			cd /release/buildroot/${i}-${j}/snapshots
-			rm -f portage-${1}.tar.bz2
-			ln -sf /release/snapshots/portage-${1}.tar.bz2 \
-				portage-${1}.tar.bz2
-		done
-	done
-fi

diff --git a/scripts/update_auto_tree b/scripts/update_auto_tree
deleted file mode 100755
index 029ef703..00000000
--- a/scripts/update_auto_tree
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-PORTDIR="/release/trees/portage-auto/"
-TMPREPO=$(mktemp)
-trap "rm -f $TMPREPO" EXIT
-cat >$TMPREPO <<EOF
-[DEFAULT]
-main-repo = gentoo
-
-[gentoo]
-location = $PORTDIR
-sync-type = rsync
-sync-uri = rsync://rsync.gentoo.org/gentoo-portage
-EOF
-PORTAGE_REPOSITORIES="$(cat $TMPREPO)" FEATURES="$FEATURES -news" emerge --sync -q

diff --git a/scripts/update_official_tree b/scripts/update_official_tree
deleted file mode 100755
index 55ecea0b..00000000
--- a/scripts/update_official_tree
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-PORTDIR="/release/trees/portage-official/"
-TMPREPO=$(mktemp)
-trap "rm -f $TMPREPO" EXIT
-cat >$TMPREPO <<EOF
-[DEFAULT]
-main-repo = gentoo
-
-[gentoo]
-location = $PORTDIR
-sync-type = rsync
-sync-uri = rsync://rsync.gentoo.org/gentoo-portage
-EOF
-PORTAGE_REPOSITORIES="$(cat $TMPREPO)" FEATURES="$FEATURES -news" emerge --sync -q

diff --git a/scripts/update_snapshot_tree b/scripts/update_snapshot_tree
deleted file mode 100755
index 76cd00ea..00000000
--- a/scripts/update_snapshot_tree
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-PORTDIR="/release/trees/portage-snapshot/"
-TMPREPO=$(mktemp)
-trap "rm -f $TMPREPO" EXIT
-cat >$TMPREPO <<EOF
-[DEFAULT]
-main-repo = gentoo
-
-[gentoo]
-location = $PORTDIR
-sync-type = rsync
-sync-uri = rsync://rsync.gentoo.org/gentoo-portage
-EOF
-PORTAGE_REPOSITORIES="$(cat $TMPREPO)" FEATURES="$FEATURES -news" emerge --sync -q

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     087b1951049e8708d76794ab9f1b172fad15e6eb
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Thu Sep  2 21:50:40 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Thu Sep  2 21:50:40 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=087b1951

Add m68k to copy_buildsync arch list

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index f9637770..886d3819 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -15,6 +15,7 @@ ARCHES=(
 	hppa
 	ia64
 	#mips
+	m68k
 	ppc
 	riscv
 	s390

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     9e885d861f558574335a25a1d34f84c8722616f2
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 31 17:38:54 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Sun Oct 31 17:45:35 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=9e885d86

Do not advertise a "current stage3" or a "current iso" in general

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 29 ++++-------------------------
 1 file changed, 4 insertions(+), 25 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 886d3819..9f56f44a 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -152,34 +152,13 @@ process_arch() {
 	if [ -n "${iso_list}" ]; then
 		echo -e "${header}" >"${OUT_ISO}"
 		# Some arches produce more than one type of iso.
-		# Only apply the current-iso link logic to them.
-		# TODO: Should make this dynamic based on the iso list.
-		case ${ARCH} in
-		amd64|x86)
-			rm -f current-iso
-			;;
-		*)
-			echo -e "${iso_list}" |awk '{print $3}' | grep "$latest_iso_date" >>${OUT_ISO}
-			ln -sfT "$latest_iso_date" current-iso
-			;;
-		esac
+		# So let's not advertise a current one via a symlink in general.
+		rm -f current-iso
 	fi
 	if [ -n "${stage3_list}" ]; then
 		echo -e "${header}" >"${OUT_STAGE3}"
-
-		# In the new variant preserve code there is a better way to do this
-		#echo -e "${stage3_list}" |awk '{print $3}' |grep "$latest_stage3_date" >>${OUT_STAGE3}
-
-		# The "latest stage3" concept works for only a few arches -- ones that
-		# do not have more than one stage3 target per arch (i.e. multilib, etc...).
-		case ${ARCH} in
-		amd64|arm|hppa|ppc|s390|sh|x86)
-			rm -f current-stage3
-			;;
-		*)
-			ln -sfT "$latest_stage3_date" current-stage3
-			;;
-		esac
+		# Dito for stage3
+		rm -f current-stage3
 	fi
 
 	# New variant preserve code

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     f7764f5d60ccafd637df5b32098103181fce91b9
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Mon Nov  1 20:46:14 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Mon Nov  1 20:46:38 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=f7764f5d

Typo fix

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 9f56f44a..7b4dfbdc 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -157,7 +157,7 @@ process_arch() {
 	fi
 	if [ -n "${stage3_list}" ]; then
 		echo -e "${header}" >"${OUT_STAGE3}"
-		# Dito for stage3
+		# Ditto for stage3
 		rm -f current-stage3
 	fi
 

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     8eb8dd235dfeb80df3740369343177f3cb64dfa8
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 12 00:15:05 2021 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Fri Nov 12 00:15:05 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=8eb8dd23

scripts/copy_buildsync: try to find dead links and textfiles

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 7b4dfbdc..e224f75e 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -164,10 +164,12 @@ process_arch() {
 	# New variant preserve code
 	find_variants=( '(' -iname '*.iso' -o -name 'netboot-*' -o "${EXTENSIONS[@]}" ')' )
 	variants=$(find 20* "${find_variants[@]}" -printf '%f\n' 2>/dev/null | sed -e 's,-20[012][0-9]\{5\}.*,,g' -r | sort -u)
+	# This file specifies which variants are still in use.
 	keepfile="${tmpdir}/.keep.${ARCH}.txt"
 	keepfile_tmp=$(mktemp -p "${tmpdir}" -t ".keep.${ARCH}.txt.XXXXXX")
 	echo -n '' >"${keepfile_tmp}"
 	chmod 644 "${keepfile_tmp}"
+
 	for v in $variants ; do
 		# FIXME: trace the $a variable in this!
 		variant_path=$(find 20* -iname "${v}-20*" "${find_variants[@]}" -print 2>/dev/null | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
@@ -188,6 +190,8 @@ process_arch() {
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${keepfile_tmp}"
 		mv -f "${f_tmp}" "${f}"
 	done
+
+	# Refresh keepfile
 	mv -f "${keepfile_tmp}" "${keepfile}"
 
 	# ================================================================
@@ -206,6 +210,37 @@ process_arch() {
 			$DEBUGP rm $VERBOSEP -rf "$(pwd)"/"${i}"
 		done
 
+		# Preserve the keepfile for review
+		cp -lf "${keepfile}" "${outdir}"
+
+		# Find the dead links for cleanup
+		_dead="${tmpdir}"/dead-link
+		find -L $(pwd) -type l >"${_dead}"
+		if test -s "${_dead}"; then
+			echo "copy_buildsync: dead links to verify:" 1>&2
+			cat "${_dead}" 1>&2
+		fi
+
+		# Find the dead latest txt files
+		_dead="${tmpdir}"/dead-latest
+		for f in $(find "${outdir}" -name 'latest*txt' ) ; do
+			d=$(dirname $f)
+			find $d \
+				| fgrep -f <(awk '/^#/{next} {print $1}' $f) \
+				| sed  "s,${d}/,,g" \
+				| sort \
+				| comm -13 \
+						- \
+						<(sort $f |grep -v -e '^#' |awk '{print $1}') \
+				| fgrep -l -f - $f \
+				| xargs -n1 --no-ruN-if-empty readlink -f
+		done >"${_dead}"
+		if test -s "${_dead}"; then
+				echo "copy_buildsync: dead latest*txt files to verify:" 1>&2
+				cat "${_dead}" 1>&2
+		fi
+
+		# Cleanup tmpdir
 		$DEBUGP rm $VERBOSEP -rf "${tmpdir}"
 
 	else

[gentoo-commits] proj/releng:master commit in: scripts/

[Matt Turner]

commit:     502ed37683f36ad6c55a02a06561c02d974947c8
Author:     Matt Turner <mattst88 <AT> gmail <DOT> com>
AuthorDate: Fri Nov 12 03:32:10 2021 +0000
Commit:     Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Nov 12 03:32:10 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=502ed376

scripts/copy_buildsync: Fix typo

Signed-off-by: Matt Turner <mattst88 <AT> gmail.com>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index e224f75e..78cc1653 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -233,7 +233,7 @@ process_arch() {
 						- \
 						<(sort $f |grep -v -e '^#' |awk '{print $1}') \
 				| fgrep -l -f - $f \
-				| xargs -n1 --no-ruN-if-empty readlink -f
+				| xargs -n1 --no-run-if-empty readlink -f
 		done >"${_dead}"
 		if test -s "${_dead}"; then
 				echo "copy_buildsync: dead latest*txt files to verify:" 1>&2

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     ebb201a6ddfac32e10c9ffdf758f1a553462d9fc
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 12 17:45:42 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Fri Nov 12 17:45:42 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=ebb201a6

Actually remove dead txt files

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 78cc1653..f7f9a3df 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -236,8 +236,10 @@ process_arch() {
 				| xargs -n1 --no-run-if-empty readlink -f
 		done >"${_dead}"
 		if test -s "${_dead}"; then
-				echo "copy_buildsync: dead latest*txt files to verify:" 1>&2
-				cat "${_dead}" 1>&2
+				echo "copy_buildsync: removing dead latest*txt files:" 1>&2
+				foreach txtfil in $(cat "${_dead}") ; do
+					rm -v "${txtfil}" 1>&2
+				done
 		fi
 
 		# Cleanup tmpdir

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     8623ef538cf32aaf4d46982c5e22569ae72ed8b0
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 12 17:59:08 2021 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Fri Nov 12 18:00:31 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=8623ef53

scripts/copy_buildsync: dead detection works too well, send to files instead

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index f7f9a3df..0308da39 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -6,6 +6,8 @@ INCOMING_BASE="/release/weekly/builds"
 OUTGOING_BASE="/release/distfiles/weekly"
 # Scratch space used when moving files from incoming to outgoing.
 TMPDIR_BASE="/release/distfiles/tmp/buildsync/partial"
+# Keep some records
+LOGDIR_BASE="/release/distfiles/tmp/buildsync/logs"
 
 ARCHES=(
 	alpha
@@ -64,7 +66,7 @@ EOF
 
 # Copy artifacts for an arch to the outgoing directory.
 copy_arch_to_outgoing() {
-	local ARCH=$1 indir=$2 outdir=$3 tmpdir=$4
+	local ARCH=$1 indir=$2 outdir=$3 tmpdir=$4 logdir=$5
 	local i t rc timestamps
 
 	if [[ ! -d ${indir} ]]; then
@@ -91,6 +93,7 @@ copy_arch_to_outgoing() {
 			"${RSYNC_OPTS[@]}" \
 			--temp-dir="${tmpdir}" \
 			--partial-dir="${tmpdir}" \
+			--log-file="${logdir}/rsync.log" \
 			--filter '- **/.*' \
 			--filter "S *${i}*" \
 			--filter 'S **/' \
@@ -126,11 +129,12 @@ process_arch() {
 	indir="${INCOMING_BASE}/${ARCH}"
 	outdir="${OUTGOING_BASE}/${ARCH}"
 	tmpdir="${TMPDIR_BASE}/${ARCH}"
+	logdir="${LOGDIR_BASE}/${ARCH}"
 
-	mkdir -p "${tmpdir}" 2>/dev/null
+	mkdir -p "${tmpdir}" "${logdir}" 2>/dev/null
 
 	# Sync incoming->outgoing first.
-	copy_arch_to_outgoing "${ARCH}" "${indir}" "${outdir}" "${tmpdir}"
+	copy_arch_to_outgoing "${ARCH}" "${indir}" "${outdir}" "${tmpdir}" "${logdir}"
 
 	# ================================================================
 	# Build data for revealing latest:
@@ -216,10 +220,7 @@ process_arch() {
 		# Find the dead links for cleanup
 		_dead="${tmpdir}"/dead-link
 		find -L $(pwd) -type l >"${_dead}"
-		if test -s "${_dead}"; then
-			echo "copy_buildsync: dead links to verify:" 1>&2
-			cat "${_dead}" 1>&2
-		fi
+		mv -f "${_dead}" "${logdir}/dead-links.txt"
 
 		# Find the dead latest txt files
 		_dead="${tmpdir}"/dead-latest
@@ -235,6 +236,7 @@ process_arch() {
 				| fgrep -l -f - $f \
 				| xargs -n1 --no-run-if-empty readlink -f
 		done >"${_dead}"
+
 		if test -s "${_dead}"; then
 				echo "copy_buildsync: removing dead latest*txt files:" 1>&2
 				foreach txtfil in $(cat "${_dead}") ; do
@@ -242,11 +244,14 @@ process_arch() {
 				done
 		fi
 
+		mv -f "${_dead}" "${logdir}/dead-latest-txt.txt"
+
 		# Cleanup tmpdir
 		$DEBUGP rm $VERBOSEP -rf "${tmpdir}"
 
 	else
 		echo "There was some failure for $ARCH during the weekly sync. Not doing cleanup for fear of dataloss." 1>&2
+		echo "See logs in $logdir" 1>&2
 	fi
 }
 

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     724fbc1c2982f9f8279e67d5e121c9007e77992b
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 12 18:18:44 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Fri Nov 12 18:18:44 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=724fbc1c

This is bash. Prepare to be substituted.

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 0308da39..7053147f 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -239,7 +239,7 @@ process_arch() {
 
 		if test -s "${_dead}"; then
 				echo "copy_buildsync: removing dead latest*txt files:" 1>&2
-				foreach txtfil in $(cat "${_dead}") ; do
+				for txtfil in $(cat "${_dead}") ; do
 					rm -v "${txtfil}" 1>&2
 				done
 		fi

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     6cd81639e3c21e1e2ed8e259b6680d23658a2d5c
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 12 18:30:08 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Fri Nov 12 18:31:21 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=6cd81639

Revert "Actually remove dead txt files"

This reverts commit ebb201a6ddfac32e10c9ffdf758f1a553462d9fc.

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 0308da39..81137b43 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -238,10 +238,8 @@ process_arch() {
 		done >"${_dead}"
 
 		if test -s "${_dead}"; then
-				echo "copy_buildsync: removing dead latest*txt files:" 1>&2
-				foreach txtfil in $(cat "${_dead}") ; do
-					rm -v "${txtfil}" 1>&2
-				done
+				echo "copy_buildsync: dead latest*txt files to verify:" 1>&2
+				cat "${_dead}" 1>&2
 		fi
 
 		mv -f "${_dead}" "${logdir}/dead-latest-txt.txt"

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     f72b1445e2f9541714739d7fe166ab4d9a0a96a6
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 12 18:29:48 2021 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Fri Nov 12 18:31:11 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=f72b1445

Revert "This is bash. Prepare to be substituted."

This reverts commit 724fbc1c2982f9f8279e67d5e121c9007e77992b.

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 7053147f..0308da39 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -239,7 +239,7 @@ process_arch() {
 
 		if test -s "${_dead}"; then
 				echo "copy_buildsync: removing dead latest*txt files:" 1>&2
-				for txtfil in $(cat "${_dead}") ; do
+				foreach txtfil in $(cat "${_dead}") ; do
 					rm -v "${txtfil}" 1>&2
 				done
 		fi

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     b42481cdb6a428c661ad8f5cb8851793454bc817
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 12 18:42:47 2021 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Fri Nov 12 18:42:47 2021 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=b42481cd

scripts/copy_buildsync: disable dead spam

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 81137b43..193075ef 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -236,12 +236,6 @@ process_arch() {
 				| fgrep -l -f - $f \
 				| xargs -n1 --no-run-if-empty readlink -f
 		done >"${_dead}"
-
-		if test -s "${_dead}"; then
-				echo "copy_buildsync: dead latest*txt files to verify:" 1>&2
-				cat "${_dead}" 1>&2
-		fi
-
 		mv -f "${_dead}" "${logdir}/dead-latest-txt.txt"
 
 		# Cleanup tmpdir

[gentoo-commits] proj/releng:master commit in: scripts/

[Andreas K. Hüttel]

commit:     d4d4ff3d5eb79e2d874b40a4b60f31fea31f2b8d
Author:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Tue Mar  1 10:22:58 2022 +0000
Commit:     Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Tue Mar  1 10:23:16 2022 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=d4d4ff3d

Re-enable mips uploads

Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 193075ef..dbd0041a 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -16,7 +16,7 @@ ARCHES=(
 	arm64
 	hppa
 	ia64
-	#mips
+	mips
 	m68k
 	ppc
 	riscv

[gentoo-commits] proj/releng:master commit in: scripts/

[Ben Kohler]

commit:     1716e0f39b9e6a99f43c7cd51435d83224d3b5bc
Author:     Ben Kohler <bkohler <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 13 15:51:41 2022 +0000
Commit:     Ben Kohler <bkohler <AT> gentoo <DOT> org>
CommitDate: Thu Oct 13 15:52:19 2022 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=1716e0f3

scripts/copy_buildsync.sh: add loong to ARCHES

Signed-off-by: Ben Kohler <bkohler <AT> gentoo.org>

 scripts/copy_buildsync.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index dbd0041a..ac56f7f7 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -16,6 +16,7 @@ ARCHES=(
 	arm64
 	hppa
 	ia64
+	loong
 	mips
 	m68k
 	ppc

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     6c75b00db7184b652ff12086a036d5081617d470
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:06:13 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:06:13 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=6c75b00d

copy_buildsync: fix tempfile for OUT_ISO & OUT_STAGE, plus comments

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index ac56f7f7..851ccb57 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -154,14 +154,21 @@ process_arch() {
 	[ -z "${latest_iso_date}" ] && latest_iso_date="NONE-FOUND"
 	[ -z "${latest_stage3_date}" ] && latest_stage3_date="NONE-FOUND"
 
+
+	OUT_ISO_tmp=""
+	OUT_STAGE3_tmp=""
 	if [ -n "${iso_list}" ]; then
-		echo -e "${header}" >"${OUT_ISO}"
+		OUT_ISO_tmp=$(mktemp -p . -t ".${OUT_ISO}.XXXXXX")
+		chmod 644 "${OUT_ISO_tmp}"
+		echo -e "${header}" >"${OUT_ISO_tmp}"
 		# Some arches produce more than one type of iso.
 		# So let's not advertise a current one via a symlink in general.
 		rm -f current-iso
 	fi
 	if [ -n "${stage3_list}" ]; then
-		echo -e "${header}" >"${OUT_STAGE3}"
+		OUT_STAGE3_tmp=$(mktemp -p . -t ".${OUT_STAGE3}.XXXXXX")
+		chmod 644 "${OUT_STAGE3_tmp}"
+		echo -e "${header}" >"${OUT_STAGE3_tmp}"
 		# Ditto for stage3
 		rm -f current-stage3
 	fi
@@ -177,25 +184,37 @@ process_arch() {
 
 	for v in $variants ; do
 		# FIXME: trace the $a variable in this!
+		# example output 20230907T160230Z/install-alpha-minimal-20230907T160230Z.iso
 		variant_path=$(find 20* -iname "${v}-20*" "${find_variants[@]}" -print 2>/dev/null | sed -e "s,.*/$a/autobuilds/,,g" | sort -k1,1 -t/ | tail -n1 )
 		if [ -z "${variant_path}" ] || [ ! -e "${variant_path}" ]; then
 			echo "$ARCH: Variant ${v} is missing" 1>&2
 			continue
 		fi
+		variant_date="${variant_path%/*}"
+		variant_base="${variant_path#*/}"
 		size=$(stat --format='%s' "${variant_path}")
 		f="latest-${v}.txt"
 		f_tmp=$(mktemp -p . -t ".${f}.XXXXXX")
 		chmod 644 "${f_tmp}"
 		echo -e "${header}" >"${f_tmp}"
 		echo -e "${variant_path} ${size}" >>"${f_tmp}"
-		[[ ${variant_path} =~ tar.*$ ]] && echo -e "${variant_path} ${size}" >>"${OUT_STAGE3}" # FIXME: tempfile
-		[[ ${variant_path} =~ iso$ ]] && echo -e "${variant_path} ${size}" >>"${OUT_ISO}" # FIXME: tempfile
+		[[ ${variant_path} =~ tar.*$ ]] && echo -e "${variant_path} ${size}" >>"${OUT_STAGE3_tmp}"
+		[[ ${variant_path} =~ iso$ ]] && echo -e "${variant_path} ${size}" >>"${OUT_ISO_tmp}"
 		rm -f "current-$v"
 		ln -sf "${variant_path%/*}" "current-$v"
+
+		# Update keepfile
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${keepfile_tmp}"
+
+		# Place latest-*txt into place in the base arch dir.
 		mv -f "${f_tmp}" "${f}"
+
 	done
 
+	# Atomic move these files if created.
+	[[ -n "${OUT_ISO_tmp}" ]] && [[ -f "${OUT_ISO_tmp}" ]] && mv "${OUT_ISO_tmp}" "${OUT_ISO}"
+	[[ -n "${OUT_STAGE3_tmp}" ]] && [[ -f "${OUT_STAGE3_tmp}" ]] && mv "${OUT_STAGE3_tmp}" "${OUT_STAGE3}"
+
 	# Refresh keepfile
 	mv -f "${keepfile_tmp}" "${keepfile}"
 

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     9c04ea27acdff25c41efd343713268b8dbf6b091
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:07:20 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:07:20 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=9c04ea27

copy_buildsync: convert current-* to directory instead of symlink

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 851ccb57..9a13872a 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -200,8 +200,24 @@ process_arch() {
 		echo -e "${variant_path} ${size}" >>"${f_tmp}"
 		[[ ${variant_path} =~ tar.*$ ]] && echo -e "${variant_path} ${size}" >>"${OUT_STAGE3_tmp}"
 		[[ ${variant_path} =~ iso$ ]] && echo -e "${variant_path} ${size}" >>"${OUT_ISO_tmp}"
-		rm -f "current-$v"
-		ln -sf "${variant_path%/*}" "current-$v"
+
+		# Previously, current-${v}/ was a symlink to the timestamp directory.
+		# This was apparently confusing to some users because it had way too many files.
+		# So instead make current-${v}/ a directory, containing just symlinks to
+		# the selected build.
+		#
+		# Link the files for a given variant into a current-${v}/ directory.
+		# If it's an old link, remove to convert to directory.
+		if test -l "current-$v" ; then rm "current-$v" ; fi
+		mkdir -p "current-$v"
+
+		# Remove old links in the directory.
+		find "current-$v" -type l ! -name "$f" ! -name "${variant_date}*"
+
+		# install new links
+		# do NOT use -f here, we do not want to override the existing files.
+		# this will ensure the mtime of the links does not change in most cases.
+		ln -s --target-directory="current-$v"/ "../${variant_path}"*
 
 		# Update keepfile
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${keepfile_tmp}"
@@ -209,6 +225,11 @@ process_arch() {
 		# Place latest-*txt into place in the base arch dir.
 		mv -f "${f_tmp}" "${f}"
 
+		# current-${variant}/latest-${variant}.txt contains the RELATIVE filename
+		# So we have to strip the leading directory on the path.
+		# reuse the tmpfile from $f
+		sed "s,^${variant_date}/,,g" <"${f}" >"${f_tmp}" && mv -f "${f_tmp}" "current-$v"/"${f}"
+
 	done
 
 	# Atomic move these files if created.

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     5ef7b638f16c10b1e2b8831dbc16f79dbc90abc0
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:09:23 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:09:23 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=5ef7b638

copy_buildsync: disable sync for everything except alpha to test new symlink code

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 9a13872a..c313af72 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -11,20 +11,21 @@ LOGDIR_BASE="/release/distfiles/tmp/buildsync/logs"
 
 ARCHES=(
 	alpha
-	amd64
-	arm
-	arm64
-	hppa
-	ia64
-	loong
-	mips
-	m68k
-	ppc
-	riscv
-	s390
-	sh
-	sparc
-	x86
+	# Temporarily disable to reduce flap.
+	#amd64
+	#arm
+	#arm64
+	#hppa
+	#ia64
+	#loong
+	#mips
+	#m68k
+	#ppc
+	#riscv
+	#s390
+	#sh
+	#sparc
+	#x86
 )
 RSYNC_OPTS=(
 	-aO

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     ddd18bfa902e89b820b3fef1365984dcb641e5cb
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:10:42 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:10:42 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=ddd18bfa

copy_buildsync: fix check typo

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index c313af72..59f08f0a 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -209,7 +209,7 @@ process_arch() {
 		#
 		# Link the files for a given variant into a current-${v}/ directory.
 		# If it's an old link, remove to convert to directory.
-		if test -l "current-$v" ; then rm "current-$v" ; fi
+		if test -L "current-$v" ; then rm "current-$v" ; fi
 		mkdir -p "current-$v"
 
 		# Remove old links in the directory.

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     a4fdfd2a582485fcbb2f5deb9b3331e7175c8e8e
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:17:02 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:17:02 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=a4fdfd2a

copy_buildsync: fix relative globbing

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 59f08f0a..3d164504 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -218,7 +218,7 @@ process_arch() {
 		# install new links
 		# do NOT use -f here, we do not want to override the existing files.
 		# this will ensure the mtime of the links does not change in most cases.
-		ln -s --target-directory="current-$v"/ "../${variant_path}"*
+		( cd "$current_v" && ln -s --target-directory=. "../${variant_path}"* )
 
 		# Update keepfile
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${keepfile_tmp}"

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     2fd21d11ea1a3f30a6ac5805e80e09489935a88a
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:17:54 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:17:54 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=2fd21d11

copy_buildsync: fix relative globbing again

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 3d164504..0f0779ce 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -218,7 +218,7 @@ process_arch() {
 		# install new links
 		# do NOT use -f here, we do not want to override the existing files.
 		# this will ensure the mtime of the links does not change in most cases.
-		( cd "$current_v" && ln -s --target-directory=. "../${variant_path}"* )
+		( cd "current-$v" && ln -s --target-directory=. "../${variant_path}"* )
 
 		# Update keepfile
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${keepfile_tmp}"

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     8b0e4c14c1a508a864ca47cb47b5c76c39cee858
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:20:17 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:20:26 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=8b0e4c14

Revert "copy_buildsync: disable sync for everything except alpha to test new symlink code"

This reverts commit 5ef7b638f16c10b1e2b8831dbc16f79dbc90abc0.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 0f0779ce..9da5893d 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -11,21 +11,20 @@ LOGDIR_BASE="/release/distfiles/tmp/buildsync/logs"
 
 ARCHES=(
 	alpha
-	# Temporarily disable to reduce flap.
-	#amd64
-	#arm
-	#arm64
-	#hppa
-	#ia64
-	#loong
-	#mips
-	#m68k
-	#ppc
-	#riscv
-	#s390
-	#sh
-	#sparc
-	#x86
+	amd64
+	arm
+	arm64
+	hppa
+	ia64
+	loong
+	mips
+	m68k
+	ppc
+	riscv
+	s390
+	sh
+	sparc
+	x86
 )
 RSYNC_OPTS=(
 	-aO

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     04bc16e0867d0a19ba803570852610b63743c3f6
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:32:15 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:32:15 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=04bc16e0

copy_buildsync: fix corner cases for targets changing

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 9da5893d..71e890d9 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -209,15 +209,42 @@ process_arch() {
 		# Link the files for a given variant into a current-${v}/ directory.
 		# If it's an old link, remove to convert to directory.
 		if test -L "current-$v" ; then rm "current-$v" ; fi
-		mkdir -p "current-$v"
+
+		# If there is a file here, something went wrong.
+		if ! mkdir -p "current-$v" ; then
+			echo "$ARCH: could not mkdir current-${v}" 1>&2
+			continue
+		fi
 
 		# Remove old links in the directory.
 		find "current-$v" -type l ! -name "$f" ! -name "${variant_date}*"
 
 		# install new links
-		# do NOT use -f here, we do not want to override the existing files.
-		# this will ensure the mtime of the links does not change in most cases.
-		( cd "current-$v" && ln -s --target-directory=. "../${variant_path}"* )
+		# do NOT unconditionally use -f here, we do not want to override the
+		# existing files.  this will ensure the mtime of the links does not change
+		# in most cases.
+		(
+				# shellcheck disable=SC2164 # error-checked above
+				cd "current-$v"
+				for variant_file in "../${variant_path}"* ; do
+				  doit=0
+					# If it doesn't exist, add it.
+					if [[ ! -e "$variant_file" ]]; then
+						doit=1
+						ln -s -t . "../${variant_path}"
+					else
+						# If it does exist, check carefully to see if anything is different
+						# Does it point to somewhere else?
+						# Is the target newer?
+						# If those are true, also bump the symlink.
+					  vfb=$(basename "$variant_file")
+					  vft=$(readlink -f "$vfb")
+						[[ "$vft" != "$(readlink -f "$variant_file")" ]] && doit=1
+						[[ "$vfb" -nt "$vft" ]] && doit=1
+					fi
+					[[ $doit -eq 1 ]] && ln -sf -t . "../${variant_file}"
+				done
+		)
 
 		# Update keepfile
 		echo "${variant_path}" | sed -e 's,/.*,,g' -e 's,^,/,g' -e 's,$,$,g' >>"${keepfile_tmp}"

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     9a3f920c9554173481d79acccedfed74357776ea
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sun Oct  8 23:35:51 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sun Oct  8 23:35:51 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=9a3f920c

copy_buildsync: ensure cleanup of old symlinks fires

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 71e890d9..7016fa26 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -217,7 +217,11 @@ process_arch() {
 		fi
 
 		# Remove old links in the directory.
-		find "current-$v" -type l ! -name "$f" ! -name "${variant_date}*"
+		find "current-$v" \
+			-type l \
+			! -name "$f" \
+			! -name "*${variant_date}*" \
+			-delete
 
 		# install new links
 		# do NOT unconditionally use -f here, we do not want to override the

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     5dedbbbbadf325136f0b6d575b50fd95dc01c0de
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Mon Oct  9 00:04:24 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Mon Oct  9 00:04:24 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=5dedbbbb

copy_buildsync: fix double ../..

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 7016fa26..68aac55c 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -235,7 +235,7 @@ process_arch() {
 					# If it doesn't exist, add it.
 					if [[ ! -e "$variant_file" ]]; then
 						doit=1
-						ln -s -t . "../${variant_path}"
+						ln -s -t . "${variant_file}"
 					else
 						# If it does exist, check carefully to see if anything is different
 						# Does it point to somewhere else?

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     477b9842eab60e258902238b648bc96462b68465
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Mon Oct  9 00:05:44 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Mon Oct  9 00:05:44 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=477b9842

copy_buildsync: fix double ln

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 68aac55c..95e3215a 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -235,7 +235,6 @@ process_arch() {
 					# If it doesn't exist, add it.
 					if [[ ! -e "$variant_file" ]]; then
 						doit=1
-						ln -s -t . "${variant_file}"
 					else
 						# If it does exist, check carefully to see if anything is different
 						# Does it point to somewhere else?
@@ -246,7 +245,7 @@ process_arch() {
 						[[ "$vft" != "$(readlink -f "$variant_file")" ]] && doit=1
 						[[ "$vfb" -nt "$vft" ]] && doit=1
 					fi
-					[[ $doit -eq 1 ]] && ln -sf -t . "../${variant_file}"
+					[[ $doit -eq 1 ]] && ln -sf -t . "${variant_file}"
 				done
 		)
 

[gentoo-commits] proj/releng:master commit in: scripts/

[Robin H. Johnson]

commit:     c5b7bfb6ecb63958ae99ed64144f0a508010b645
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Mon Oct  9 00:08:03 2023 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Mon Oct  9 00:08:03 2023 +0000
URL:        https://gitweb.gentoo.org/proj/releng.git/commit/?id=c5b7bfb6

copy_buildsync: refactor check for link create

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 scripts/copy_buildsync.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/copy_buildsync.sh b/scripts/copy_buildsync.sh
index 95e3215a..df971e0a 100755
--- a/scripts/copy_buildsync.sh
+++ b/scripts/copy_buildsync.sh
@@ -232,15 +232,15 @@ process_arch() {
 				cd "current-$v"
 				for variant_file in "../${variant_path}"* ; do
 				  doit=0
+					vfb=$(basename "$variant_file")
 					# If it doesn't exist, add it.
-					if [[ ! -e "$variant_file" ]]; then
+					if [[ ! -e "$vfb" ]]; then
 						doit=1
 					else
 						# If it does exist, check carefully to see if anything is different
 						# Does it point to somewhere else?
 						# Is the target newer?
 						# If those are true, also bump the symlink.
-					  vfb=$(basename "$variant_file")
 					  vft=$(readlink -f "$vfb")
 						[[ "$vft" != "$(readlink -f "$variant_file")" ]] && doit=1
 						[[ "$vfb" -nt "$vft" ]] && doit=1