[gentoo-commits] proj/toolchain:master commit in: scripts/

["Mike Frysinger" <vapier@gentoo.org>]

commit:     cb91c820d26023cb0bcaa20dae4ddcd6db914064
Author:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 13 03:49:55 2015 +0000
Commit:     Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Mon Apr 13 04:09:02 2015 +0000
URL:        https://gitweb.gentoo.org/proj/toolchain.git/commit/?id=cb91c820

scripts: rewrite in python

The bash scripts are getting hard to maintain.

 scripts/.gitignore |   1 +
 scripts/.pylintrc  |  24 ++++++++++
 scripts/common.py  | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 scripts/update-gcc | 122 ++++++++++++++++++++++++++------------------------
 scripts/update-gdb | 127 ++++++++++++++++++++++++++++++-----------------------
 5 files changed, 286 insertions(+), 114 deletions(-)

diff --git a/scripts/.gitignore b/scripts/.gitignore
index 913d141..57e2a4e 100644
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@@ -1 +1,2 @@
 cronjob.log
+*.pyc

diff --git a/scripts/.pylintrc b/scripts/.pylintrc
new file mode 100644
index 0000000..d91e935
--- /dev/null
+++ b/scripts/.pylintrc
@@ -0,0 +1,24 @@
+[MESSAGES CONTROL]
+disable=C0103,C0111,C0302,E1103,I0011,R0201,R0902,R0903,R0911,R0912,R0913,R0914,R0915,W0122,W0141,W0142,W0403,W0511,W0703,R0904,R0921,R0922,bad-continuation
+
+[REPORTS]
+reports=no
+
+[FORMAT]
+indent-string='	'
+
+[TYPECHECK]
+ignored-classes=hashlib,numpy
+
+[BASIC]
+function-rgx=([A-Z_][a-zA-Z0-9]{2,30}|main)$
+method-rgx=((_|test)?[A-Z][a-zA-Z0-9]{2,30}|__[a-z]+__|setUp|tearDown)$
+
+[SIMILARITIES]
+min-similarity-lines=20
+
+[VARIABLES]
+dummy-variables-rgx=_|unused_
+
+[DESIGN]
+max-parents=10

diff --git a/scripts/common.py b/scripts/common.py
new file mode 100644
index 0000000..2da9139
--- /dev/null
+++ b/scripts/common.py
@@ -0,0 +1,126 @@
+#!/usr/bin/python
+
+"""Utility funcs"""
+
+from __future__ import print_function
+
+import argparse
+import distutils.version
+import ftplib
+import locale
+import logging
+import os
+import re
+import subprocess
+import sys
+import time
+import urlparse
+
+
+dry_run = False
+
+
+def cd(cat, pn):
+	"""Change to the $CATAGORY/$PN subdir for this package"""
+	path = os.path.join(os.path.dirname(os.path.dirname(
+		os.path.realpath(__file__))), cat, pn)
+	logging.info('processing %s', path)
+	os.chdir(path)
+	assert os.path.exists('metadata.xml')
+
+
+def list_snaps(url, debug=False):
+	"""Get a listing of all the snapshots for this package"""
+	if debug:
+		if os.path.exists('.listing'):
+			return open('.listing').read().splitlines()
+	o = urlparse.urlparse(url)
+	logging.info('listing %s', url)
+	ftp = ftplib.FTP(o.netloc)
+	ftp.login()
+	ftp.cwd(o.path)
+	nlst = ftp.nlst()
+	if debug:
+		with open('.listing', 'w') as f:
+			f.write('\n'.join(nlst))
+	return nlst
+
+
+ver_sort = lambda x: sorted(x, key=lambda v: distutils.version.LooseVersion(v))
+
+
+def run(cmd, **kwargs):
+	logging.info('running: %s', ' '.join(cmd))
+	if not dry_run:
+		subprocess.check_call(cmd, **kwargs)
+
+
+def run_ebuild(ebuild, command):
+	"""Run `ebuild |ebuild| |command|`"""
+	env = os.environ.copy()
+	env.update({
+		'FEATURES': 'assume-digests -strict digest',
+		'GENTOO_MIRRORS': ' ',
+	})
+	run(['ebuild', ebuild, command], env=env)
+
+
+def git(args):
+	"""Run `git |args|`"""
+	run(['git'] + args)
+
+
+def setup_logging(debug=False):
+	"""Setup the logging module"""
+	fmt = '%(asctime)s: %(levelname)-7s: '
+	if debug:
+		fmt += '%(filename)s:%(funcName)s: '
+	fmt += '%(message)s'
+	datefmt = '%a, %d %b %Y %H:%M:%S ' + time.tzname[0]
+
+	level = logging.DEBUG if debug else logging.INFO
+
+	handler = logging.StreamHandler(stream=sys.stdout)
+	formatter = logging.Formatter(fmt, datefmt)
+	handler.setFormatter(formatter)
+
+	logger = logging.getLogger()
+	logger.addHandler(handler)
+	logger.setLevel(level)
+
+
+def get_ver(ebuild):
+	"""Given an ebuild name, return the version"""
+	m = re.match(r'[a-z-]+-(([0-9]+\.?)+)(_alpha([0-9]+))?\.ebuild', ebuild)
+	if not m:
+		raise ValueError('could not parse %s' % ebuild)
+	dots = m.group(1)
+	stamp = m.group(4)
+	return distutils.version.LooseVersion('%s-%s' % (dots, stamp))
+
+
+def parse_args(argv):
+	parser = argparse.ArgumentParser()
+	parser.add_argument('-d', '--debug', default=False, action='store_true')
+	parser.add_argument('-n', '--dry-run', default=False, action='store_true')
+	return parser.parse_args(argv)
+
+
+def common_main(argv, cat, pn):
+	locale.setlocale(locale.LC_ALL, 'C')
+
+	opts = parse_args(argv)
+	global dry_run
+	dry_run = opts.dry_run
+
+	setup_logging(debug=opts.debug)
+	logging.info('running %s', pn)
+	cd(cat, pn)
+
+	# Clean the paths.
+	git(['reset', 'HEAD', '.'])
+	git(['checkout', '-f', '.'])
+	git(['clean', '-q', '-f', '.'])
+	git(['status', '.'])
+
+	return opts

diff --git a/scripts/update-gcc b/scripts/update-gcc
index 2fa5412..da82ec6 100755
--- a/scripts/update-gcc
+++ b/scripts/update-gcc
@@ -1,69 +1,75 @@
-#!/bin/bash -e
+#!/usr/bin/python
 
-export LC_ALL=C
-PN="gcc"
+"""Update gcc snapshots"""
 
-sudo=
+from __future__ import print_function
 
-[ -d sys-devel ] && cd sys-devel/${PN}
-[ -d ../sys-devel ] && cd ../sys-devel/${PN}
-if [ ! -e metadata.xml ] ; then
-	echo "Run this in the ${PN} dir"
-	exit 1
-fi
+import distutils.version
+import glob
+import logging
+import re
+import shutil
+import sys
 
-export FEATURES="assume-digests -strict digest"
-fetch() {
-	${sudo} env GENTOO_MIRRORS=" " FEATURES="${FEATURES}" ebuild $1 fetch
-}
-manifest() {
-	${sudo} ebuild $1 manifest
-}
+from common import * # pylint: disable=wildcard-import,unused-wildcard-import
 
-date
 
-wget -nv --no-remove-listing -O /dev/null ftp://gcc.gnu.org/pub/gcc/snapshots/
-l=".listing"
-sed -i 's/\r$//' ${l}
+CATEGORY = 'sys-devel'
+PN = 'gcc'
+URL = 'ftp://gcc.gnu.org/pub/gcc/snapshots/'
 
-# First unload ones that no longer exist.
-ebuilds=$(echo ${PN}-*_alpha*.ebuild)
-for e in ${ebuilds} ; do
-	v=$(echo "${e}" | sed -e 's:^gcc-::g' -e 's:.ebuild::g' -e 's:.._alpha:-:g')
-	if grep -q "\<${v}$" ${l} ; then
-		continue
-	fi
-	git rm -f ${e}
-done
 
-# Then load new ones.
-majors=$(awk '$(NF-2) ~ /^LATEST-/ { print gensub("LATEST-", "", "", $(NF-2)) }' ${l} | sort -V)
-echo "${PN} majors:" ${majors}
+def main(argv):
+	opts = common_main(argv, CATEGORY, PN)
 
-lm=
-for m in ${majors} ; do
-	snaps=$(awk '$NF ~ /^'${m}'-/ { print gensub("'${m}'-", "", "", $NF) }' ${l})
-	curr=$(ls ${PN}-${m}.0_alpha* | sort | tail -n 1)
-	if [[ -z ${curr} ]] ; then
-		curr=$(ls ${PN}-${lm}.0_alpha* | sort | tail -n 1)
-		if [[ -z ${curr} ]] ; then
-			echo "no current ebuild for major ${m}"
-			exit 1
-		fi
-	fi
-	# Pad out to 2 versions if the listing doesn't have it.
-	[[ ${m} != *.* ]] && m+=".0"
-	echo "### ${m}: ${curr}"
-	for s in ${snaps} ; do
-		s="${PN}-${m}.0_alpha${s}.ebuild"
-		[[ -e ${s} ]] && continue
-		echo " ${s}"
-		cp ${curr} ${s}
-		fetch ${s}
-	done
-	lm=${m}
-done
+	remote_list = ver_sort(
+		x for x in list_snaps(URL, debug=opts.debug)
+		if not x.startswith('LATEST-') and '-' in x)
 
-rm -f ${l}
+	# Create the lists of curr/new versions.
+	old_pkgs = set(glob.glob('%s-*.ebuild' % PN))
+	new_pkgs = set()
+	for snap in remote_list:
+		m = re.match(r'([0-9.]+)-([0-9.]+)$', snap)
+		if m:
+			# Turn "5" into "5.0.0" and "4.3" into "4.3.0".
+			dots = '.'.join((m.group(1).split('.') + (['0'] * 3))[0:3])
+			ebuild = '%s-%s_alpha%s.ebuild' % (PN, dots, m.group(2))
+			new_pkgs.add(ebuild)
+			logging.debug('found remote %s', ebuild)
+		else:
+			logging.warning('skipping reomte %s', snap)
 
-manifest ${s}
+	# Create ebuilds for the new versions we found.
+	closest_ver = distutils.version.LooseVersion('0')
+	for pkg in new_pkgs - old_pkgs:
+		logging.info('adding %s', pkg)
+		ver = get_ver(pkg)
+		for opkg in old_pkgs:
+			if '_alpha' not in opkg:
+				continue
+			over = get_ver(opkg)
+			if over < ver and over > closest_ver:
+				closest_ver = over
+		logging.info(' copying from %s', closest_ver)
+		dots, stamp = str(closest_ver).split('-')
+		ebuild = '%s-%s_alpha%s.ebuild' % (PN, dots, stamp)
+		if not opts.dry_run:
+			shutil.copy(ebuild, pkg)
+		git(['add', pkg])
+		run_ebuild(pkg, 'fetch')
+		#run_ebuild(pkg, 'manifest')
+
+	# Clean out the old snapshots.
+	for pkg in ver_sort(old_pkgs - new_pkgs):
+		if '_alpha' not in pkg:
+			continue
+		logging.info('cleaning old %s', pkg)
+		git(['rm', '-f', pkg])
+
+	run(['repoman', 'manifest'])
+	git(['add', 'Manifest'])
+
+
+if __name__ == '__main__':
+	main(sys.argv[1:])

diff --git a/scripts/update-gdb b/scripts/update-gdb
index a6d64c2..08cfcb2 100755
--- a/scripts/update-gdb
+++ b/scripts/update-gdb
@@ -1,56 +1,71 @@
-#!/bin/bash -e
-
-export LC_ALL=C
-PN="gdb"
-
-sudo=
-
-[ -d sys-devel ] && cd sys-devel/${PN}
-[ -d ../sys-devel ] && cd ../sys-devel/${PN}
-if [ ! -e metadata.xml ] ; then
-	echo "Run this in the ${PN} dir"
-	exit 1
-fi
-
-export FEATURES="assume-digests -strict digest"
-fetch() {
-	${sudo} env GENTOO_MIRRORS=" " FEATURES="${FEATURES}" ebuild $1 fetch
-}
-manifest() {
-	${sudo} ebuild $1 manifest
-}
-
-date
-
-wget -nv --no-remove-listing -O /dev/null ftp://sources.redhat.com/pub/${PN}/snapshots/current/
-l=".listing"
-sed -i 's/\r$//' ${l}
-
-# First unload ones that no longer exist.
-ebuilds=$(echo ${PN}-?.?.50.*.ebuild)
-for e in ${ebuilds} ; do
-	f=$(echo "${e}" | sed -e 's:^gdb-:gdb-weekly-:g' -e 's:.ebuild:.tar.(bz2|xz):g')
-	if grep -qE "\<${f}$" ${l} ; then
-		continue
-	fi
-	git rm -f ${e}
-done
-
-# Then load new ones.
-snaps=$(grep -o ${PN}-weekly-'[[:digit:]]'.*.tar.* ${l} | \
-	sed -e 's:.*-::' -e 's:.tar.*::')
-
-src=$(ls ${PN}-*.ebuild | sort | tail -n 1)
-for s in ${snaps} ; do
-	dst="${PN}-${s}.ebuild"
-	[[ ! -e ${dst} ]] || continue
-	echo "### found ${s}"
-	cp ${src} ${dst}
-	fetch ${dst}
-done
-
-rm -f ${l}
-
-if [[ -n ${src} ]] ; then
-	manifest ${src}
-fi
+#!/usr/bin/python
+
+"""Update gdb snapshots"""
+
+from __future__ import print_function
+
+import distutils.version
+import glob
+import logging
+import re
+import shutil
+import sys
+
+from common import * # pylint: disable=wildcard-import,unused-wildcard-import
+
+
+CATEGORY = 'sys-devel'
+PN = 'gdb'
+URL = 'ftp://sources.redhat.com/pub/gdb/snapshots/current/'
+
+
+def main(argv):
+	opts = common_main(argv, CATEGORY, PN)
+
+	remote_list = ver_sort(
+		x for x in list_snaps(URL, debug=opts.debug)
+		if x.startswith('%s-weekly-' % PN) and '.tar' in x)
+
+	# Create the lists of curr/new versions.
+	old_pkgs = set(glob.glob('%s-*.ebuild' % PN))
+	new_pkgs = set()
+	for snap in remote_list:
+		m = re.match(r'%s-weekly-([0-9.]+)\.tar' % PN, snap)
+		if m:
+			ebuild = '%s-%s.ebuild' % (PN, m.group(1))
+			new_pkgs.add(ebuild)
+			logging.debug('found remote %s', ebuild)
+		else:
+			logging.warning('skipping reomte %s', snap)
+
+	# Create ebuilds for the new versions we found.
+	closest_ver = distutils.version.LooseVersion('0')
+	for pkg in new_pkgs - old_pkgs:
+		logging.info('adding %s', pkg)
+		ver = get_ver(pkg)
+		for opkg in old_pkgs:
+			if '.50.' not in opkg:
+				continue
+			over = get_ver(opkg)
+			if over < ver and over > closest_ver:
+				closest_ver = over
+		logging.info(' copying from %s', closest_ver)
+		ebuild = '%s-%s.ebuild' % (PN, closest_ver)
+		shutil.copy(ebuild, pkg)
+		git(['add', pkg])
+		run_ebuild(pkg, 'fetch')
+		#run_ebuild(pkg, 'manifest')
+
+	# Clean out the old snapshots.
+	for pkg in ver_sort(old_pkgs - new_pkgs):
+		if '.50.' not in pkg:
+			continue
+		logging.info('cleaning old %s', pkg)
+		git(['rm', '-f', pkg])
+
+	run(['repoman', 'manifest'])
+	git(['add', 'Manifest'])
+
+
+if __name__ == '__main__':
+	main(sys.argv[1:])