[gentoo-commits] proj/arch-tools:master commit in: /

["Paweł Hajdan" <phajdan.jr@gentoo.org>]

commit:     842f3d251d81ffc274fa01a1d9ba9b5794a2d714
Author:     William Hubbs <w.d.hubbs <AT> gmail <DOT> com>
AuthorDate: Sat Dec 13 19:33:09 2014 +0000
Commit:     Paweł Hajdan <phajdan.jr <AT> gentoo <DOT> org>
CommitDate: Mon Dec 29 14:52:27 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/arch-tools.git;a=commit;h=842f3d25

Port most tools to python 3

This commit ports the following tools to python 3.

- common code
- batch-stabilize
- file-stabilization-bugs
- maintainer-timeout
- reverse-dependencies
- stabilization-candidates

This is a list of the changes:

- Remove the dependency on pybugz; these tools now use xmlrpc.client
  directly. This is needed because the next release of pybugz does the
  same thing; we no longer have a bugzilla module since it was just a
  wrapper.

Handle tokens correctly:

- When we log into Bugzilla, we are given a
  token we must pass back to it with every command we issue.

- Log out at the end of every session to expire the token.

Fix the state file processing:

- The pickle module handles binary files, not text files, so read and
  write the state file in binary mode.

- Add fix_imports=True when reading the state file so a python-2
  compatible state file can be read.

- Do not set the URL for a stabilization request

X-Gentoo-Bug: 532054
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532054
X-Gentoo-Bug: 532368
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532368

---
 batch-stabilize.py          | 43 ++++++++++++++++++++----------------
 common.py                   |  4 ++--
 file-stabilization-bugs.py  | 42 ++++++++++++++++++-----------------
 maintainer-timeout.py       | 31 ++++++++++++++++++--------
 reverse-dependencies.py     |  6 ++---
 stabilization-candidates.py | 54 ++++++++++++++++++++++++++-------------------
 6 files changed, 104 insertions(+), 76 deletions(-)

diff --git a/batch-stabilize.py b/batch-stabilize.py
index 053cd19..155af38 100755
--- a/batch-stabilize.py
+++ b/batch-stabilize.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # Copyright 2011 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
@@ -11,8 +11,8 @@ import re
 import shutil
 import subprocess
 import sys
+import xmlrpc.client
 
-from bugz.bugzilla import BugzillaProxy
 from common import login
 import portage.versions
 
@@ -20,7 +20,7 @@ BUG_REGEX = re.compile("[Bb]ug #?(\d+)")
 
 def print_and_log(message, log):
 	try:
-		print message
+		print(message)
 		log.write(message + '\n')
 	finally:
 		log.flush()
@@ -40,7 +40,7 @@ def run_command(args, cwd, log):
 		log.flush()
 
 def save_state(done_bugs):
-	with open('batch-stabilize.state', 'w') as state_file:
+	with open('batch-stabilize.state', 'wb') as state_file:
 		pickle.dump(done_bugs, state_file)
 
 if __name__ == "__main__":
@@ -62,12 +62,12 @@ if __name__ == "__main__":
 
 	done_bugs = []
 	if os.path.exists('batch-stabilize.state'):
-		with open('batch-stabilize.state', 'r') as state_file:
-			done_bugs = pickle.load(state_file)
+		with open('batch-stabilize.state', 'rb') as state_file:
+			done_bugs = pickle.load(state_file, fix_imports=True)
 
 	url = 'https://bugs.gentoo.org/xmlrpc.cgi'
-	print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url
-	bugzilla = BugzillaProxy(url)
+	print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url)
+	bugzilla = xmlrpc.client.ServerProxy(url)
 	user, login_data = login(bugzilla)
 	
 	with open(options.input_filename, "r") as input_file:
@@ -83,14 +83,14 @@ if __name__ == "__main__":
 			if line.startswith("#"):
 				match = BUG_REGEX.search(line, re.IGNORECASE)
 				if not match:
-					print 'Ignoring comment line [%s]...' % line
+					print('Ignoring comment line [%s]...' % line)
 					continue
 				else:
 					bug_id = int(match.group(1))
 				continue
 
 			if bug_id == -1:
-				print 'Could not recognize bug id'
+				print('Could not recognize bug id')
 				sys.exit(1)
 
 			# Drop the leading '='.
@@ -109,10 +109,10 @@ if __name__ == "__main__":
 			for (pn, ebuild_name) in stabilization_dict[bug_id]:
 				ebuild_path = os.path.join(options.repo, pn, ebuild_name)
 				if not os.path.exists(ebuild_path):
-					print '%s: file does not exist' % ebuild_path
+					print('%s: file does not exist' % ebuild_path)
 					success = False
 		if not success:
-			print 'Sanity check failed. Please make sure your CVS repo is up to date (cvs up).'
+			print('Sanity check failed. Please make sure your CVS repo is up to date (cvs up).')
 			sys.exit(1)
 
 		with open('batch-stabilize.log', 'a') as log_file:
@@ -132,20 +132,20 @@ if __name__ == "__main__":
 						try:
 							shutil.rmtree(cvs_path)
 						except OSError:
-							print '!!! rmtree %s failed' % cvs_path
+							print('!!! rmtree %s failed' % cvs_path)
 							sys.exit(1)
 
 					if run_command(["cvs", "up", pn], options.repo, log_file)[0] != 0:
-						print '!!! cvs up failed'
+						print('!!! cvs up failed')
 						sys.exit(1)
 				for (pn, ebuild_name) in stabilization_dict[bug_id]:
 					cvs_path = os.path.join(options.repo, pn)
 					print_and_log('Working in %s...' % cvs_path, log_file)
 					if run_command(["ekeyword", options.arch, ebuild_name], cvs_path, log_file)[0] != 0:
-						print '!!! ekeyword failed'
+						print('!!! ekeyword failed')
 						sys.exit(1)
 					if run_command(["repoman", "manifest"], cvs_path, log_file)[0] != 0:
-						print '!!! repoman manifest failed'
+						print('!!! repoman manifest failed')
 						sys.exit(1)
 				for (pn, ebuild_name) in stabilization_dict[bug_id]:
 					cvs_path = os.path.join(options.repo, pn)
@@ -160,12 +160,13 @@ if __name__ == "__main__":
 					if run_command(["echangelog", commit_message], cvs_path, log_file)[0] != 0:
 						print_and_log('echangelog failed, maybe just the Manifest is being updated; continuing', log_file)
 					if run_command(["repoman", "manifest"], cvs_path, log_file)[0] != 0:
-						print '!!! repoman manifest failed'
+						print('!!! repoman manifest failed')
 						sys.exit(1)
 					if run_command(["repoman", "commit", "--ignore-arches", "-m", commit_message], cvs_path, log_file)[0] != 0:
-						print '!!! repoman commit failed'
+						print('!!! repoman commit failed')
 						sys.exit(1)
 				params = {}
+				params['Bugzilla_token'] = login_data['token']
 				params['ids'] = [bug_id]
 				bug_xml = bugzilla.Bug.get(params)['bugs'][0]
 				has_my_arch = False
@@ -188,7 +189,7 @@ if __name__ == "__main__":
 				# We don't close bugs which still have other arches for obvious reasons,
 				# and security bugs because stabilization is not the last step for them.
 				params = {}
-				params['token'] = login_data['token']
+				params['Bugzilla_token'] = login_data['token']
 				params['ids'] = [bug_id]
 				params['cc'] = {}
 				params['cc']['remove'] = ['%s@gentoo.org' % options.arch]
@@ -209,3 +210,7 @@ if __name__ == "__main__":
 	
 	if os.path.exists('batch-stabilize.state'):
 		os.remove('batch-stabilize.state')
+
+		params = {}
+		params['Bugzilla_token'] = login_data['token']
+		bugzilla.User.logout(params)

diff --git a/common.py b/common.py
index 578b0e0..89e4996 100644
--- a/common.py
+++ b/common.py
@@ -1,7 +1,7 @@
 # Copyright 2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
-import cStringIO
+import io
 import datetime
 import getpass
 import re
@@ -53,7 +53,7 @@ def detect_cpvs(text):
 
 
 def get_input(prompt):
-	return raw_input(prompt)
+	return input(prompt)
 
 
 def login(bugzilla):

diff --git a/file-stabilization-bugs.py b/file-stabilization-bugs.py
index 1c5bbe9..f9b170c 100755
--- a/file-stabilization-bugs.py
+++ b/file-stabilization-bugs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # Copyright 2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
@@ -11,16 +11,14 @@ import re
 import shutil
 import subprocess
 import sys
-import urllib
-import xmlrpclib
+import xmlrpc.client
 
-from bugz.bugzilla import BugzillaProxy
 from common import login
 import portage.versions
 from portage.xml.metadata import MetaDataXML
 
 def save_state(done_cpvs):
-	with open('file-stabilization-bugs.state', 'w') as state_file:
+	with open('file-stabilization-bugs.state', 'wb') as state_file:
 		pickle.dump(done_cpvs, state_file)
 
 if __name__ == "__main__":
@@ -40,13 +38,13 @@ if __name__ == "__main__":
 
 	done_cpvs = []
 	if os.path.exists('file-stabilization-bugs.state'):
-		with open('file-stabilization-bugs.state', 'r') as state_file:
-			done_cpvs = pickle.load(state_file)
+		with open('file-stabilization-bugs.state', 'rb') as state_file:
+			done_cpvs = pickle.load(state_file, fix_imports=True)
 
 	url = 'https://bugs.gentoo.org/xmlrpc.cgi'
-	print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url
-	bugzilla = BugzillaProxy(url)
-	login(bugzilla)
+	print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url)
+	bugzilla = xmlrpc.client.ServerProxy(url)
+	user, login_data = login(bugzilla)
 	
 	with open(options.input_filename, "r") as input_file:
 		for line in input_file:
@@ -58,7 +56,7 @@ if __name__ == "__main__":
 
 			cpv = line
 			if cpv in done_cpvs:
-				print 'Skipping %s because it\'s marked as done' % cpv
+				print('Skipping %s because it\'s marked as done' % cpv)
 				continue
 
 			cp = portage.versions.pkgsplit(cpv)[0]
@@ -75,37 +73,41 @@ if __name__ == "__main__":
 			description = ('Is it OK to stabilize =%s ?\n' % cpv +
 				       'If so, please CC all arches which have stable keywords\n' +
 				       'for older versions of this package.')
-			url = 'http://packages.gentoo.org/package/%s?arches=linux' % urllib.quote(cp)
 			params = {}
+			params['Bugzilla_token'] = login_data['token']
 			params['product'] = 'Gentoo Linux'
 			params['version'] = 'unspecified'
 			params['component'] = 'Keywording and Stabilization'
 			params['summary'] = '%s: stabilization request' % cpv
 			params['description'] = description
-			params['url'] = url
 			params['assigned_to'] = maintainer
 			params['cc'] = other_maintainers
 			params['severity'] = 'enhancement'
 			try:
 				bug_id = bugzilla.Bug.create(params)['id']
-				print 'Submitted bug #%d for %s. ;-)' % (bug_id, cpv)
+				print('Submitted bug #%d for %s. ;-)' % (bug_id, cpv))
 				done_cpvs += cpv
 				save_state(done_cpvs)
 				try:
 					params = {}
+					params['Bugzilla_token'] = login_data['token']
 					params['ids'] = [bug_id]
 					params['keywords'] = {'set': 'STABLEREQ'}
 					bugzilla.Bug.update(params)
-				except xmlrpclib.Fault, f:
+				except xmlrpc.client.Fault as f:
 					exit_code = 1
-					print f
-					print 'Failed to add STABLEREQ keyword for %s. :-/' % cpv
-			except xmlrpclib.Fault, f:
+					print(f)
+					print('Failed to add STABLEREQ keyword for %s. :-/' % cpv)
+			except xmlrpc.client.Fault as f:
 				exit_code = 1
-				print f
-				print 'Failed to submit bug for %s. :-(' % cpv
+				print(f)
+				print('Failed to submit bug for %s. :-(' % cpv)
 	
 	if exit_code == 0 and os.path.exists('file-stabilization-bugs.state'):
 		os.remove('file-stabilization-bugs.state')
 	
+	params = {}
+	params['Bugzilla_token'] = login_data['token']
+	bugzilla.User.logout(params)
+
 	sys.exit(exit_code)

diff --git a/maintainer-timeout.py b/maintainer-timeout.py
index 76c7e59..54b4044 100755
--- a/maintainer-timeout.py
+++ b/maintainer-timeout.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # Copyright 2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
@@ -6,11 +6,11 @@ import datetime
 import optparse
 import os.path
 import sys
+import xmlrpc.client
 
-from bugz.bugzilla import BugzillaProxy
 import portage.versions
 
-from common import Bug, chunks, login, detect_cpvs
+from common import login, detect_cpvs
 
 
 def keyword_to_email(keyword):
@@ -27,12 +27,20 @@ if __name__ == "__main__":
 		parser.error("unrecognized command-line args")
 
 	url = 'https://bugs.gentoo.org/xmlrpc.cgi'
-	print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url
-	bugzilla = BugzillaProxy(url)
+	print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url)
+	bugzilla = xmlrpc.client.ServerProxy(url)
 	user, login_data = login(bugzilla)
 	
-	bugs = bugzilla.Bug.search({'reporter': user, 'summary': ['stabilize', 'stabilization', 'stable'], 'resolution': ''})['bugs']
-	comments = bugzilla.Bug.comments({'ids': [bug['id'] for bug in bugs]})
+	params = {}
+	params['Bugzilla_token'] = login_data['token']
+	params['reporter'] = user
+	params['summary'] = ['stabilize', 'stabilization', 'stable']
+	params['resolution'] = ''
+	bugs = bugzilla.Bug.search(params)['bugs']
+	params = {}
+	params['Bugzilla_token'] = login_data['token']
+	params['ids'] = [bug['id'] for bug in bugs]
+	comments = bugzilla.Bug.comments(params)
 	for bug in bugs:
 		# Skip bugs where stabilization seems to be already in progress.
 		arch_found = False
@@ -68,6 +76,7 @@ if __name__ == "__main__":
 					target_keywords.add(keyword)
 
 		params = {}
+		params['Bugzilla_token'] = login_data['token']
 		params['ids'] = [bug['id']]
 		params['cc'] = {}
 		params['cc']['add'] = list(set(keyword_to_email(k) for k in target_keywords))
@@ -75,7 +84,11 @@ if __name__ == "__main__":
 		params['comment']['body'] = 'Maintainer timeout (%d days). Arches please go ahead.' % options.days
 
 		bugzilla.Bug.update(params)
-		print 'Updated bug #%s (%s). Target KEYWORDS: %s ;-)' % (
+		print('Updated bug #%s (%s). Target KEYWORDS: %s ;-)' % (
 			bug['id'],
 			bug['summary'],
-			', '.join(list(target_keywords)))
+			', '.join(list(target_keywords))))
+
+	params = {}
+	params['Bugzilla_token'] = login_data['token']
+	bugzilla.User.logout(params)

diff --git a/reverse-dependencies.py b/reverse-dependencies.py
index 96233ea..624b668 100755
--- a/reverse-dependencies.py
+++ b/reverse-dependencies.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # Copyright 2011 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
@@ -16,7 +16,7 @@ def process_deps(deps, use=None):
 		use = []
 	result = []
 	for index, x in enumerate(deps):
-		if type(x) == types.ListType:
+		if type(x) == list:
 			continue
 		elif x == "||":  # Any-of.
 			result.extend(process_deps(deps[index + 1], use))
@@ -82,4 +82,4 @@ if __name__ == "__main__":
 								output_file.write("#\t%s\n" % ",".join(use_combination))
 						output_file.write("%s\n" % cp)
 				elif options.verbose:
-					print 'No reverse dependencies for %s' % pn
+					print('No reverse dependencies for %s' % pn)

diff --git a/stabilization-candidates.py b/stabilization-candidates.py
index 52c88f1..d324bdb 100755
--- a/stabilization-candidates.py
+++ b/stabilization-candidates.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # Copyright 2011 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
@@ -8,10 +8,8 @@ import os.path
 import random
 import re
 import subprocess
-import xmlrpclib
-import urllib
+import xmlrpc.client
 
-from bugz.bugzilla import BugzillaProxy
 from portage.package.ebuild.getmaskingstatus import getmaskingstatus
 from portage.xml.metadata import MetaDataXML
 import portage.versions
@@ -36,9 +34,9 @@ if __name__ == "__main__":
 		parser.error("unrecognized command-line args")
 
 	url = 'https://bugs.gentoo.org/xmlrpc.cgi'
-	print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url
-	bugzilla = BugzillaProxy(url)
-	login(bugzilla)
+	print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url)
+	bugzilla = xmlrpc.client.ServerProxy(url)
+	user, login_data = login(bugzilla)
 	
 	final_candidates = []
 	now = datetime.datetime.now()
@@ -77,7 +75,7 @@ if __name__ == "__main__":
 		best_stable = portage.versions.best(portage.portdb.match(cp))
 		if not best_stable:
 			continue
-		print 'Working on %s...' % cp,
+		print('Working on %s...' % cp, end=' ')
 		candidates = []
 		for cpv in portage.portdb.cp_list(cp):
 			# Only consider higher versions than best stable.
@@ -100,12 +98,12 @@ if __name__ == "__main__":
 				continue
 
 			# Eliminate hard masked packages among others.
-			if getmaskingstatus(cpv) not in [[u'~%s keyword' % arch] for arch in options.arch]:
+			if getmaskingstatus(cpv) not in [['~%s keyword' % arch] for arch in options.arch]:
 				continue
 
 			candidates.append(cpv)
 		if not candidates:
-			print 'no candidates'
+			print('no candidates')
 			continue
 
 		candidates.sort(key=portage.versions.cpv_sort_key())
@@ -122,14 +120,14 @@ if __name__ == "__main__":
 				regex = '\*%s \((.*)\)' % re.escape(pv)
 				match = re.search(regex, changelog_file.read())
 				if not match:
-					print 'error parsing ChangeLog'
+					print('error parsing ChangeLog')
 					continue
 				changelog_date = datetime.datetime.strptime(match.group(1), '%d %b %Y')
 				if now - changelog_date < datetime.timedelta(days=options.days):
-					print 'not old enough'
+					print('not old enough')
 					continue
-		except IOError, e:
-			print e
+		except IOError as e:
+			print(e)
 			continue
 
 		keywords = portage.db["/"]["porttree"].dbapi.aux_get(best_candidate, ['KEYWORDS'])[0]
@@ -139,19 +137,25 @@ if __name__ == "__main__":
 				missing_arch = True
 				break
 		if missing_arch:
-			print 'not keyworded ~arch'
+			print('not keyworded ~arch')
 			continue
 
 		# Do not risk trying to stabilize a package with known bugs.
-		bugs = [x for x in bugzilla.Bug.search({'summary': cp})['bugs'] if x['is_open'] and x['severity'] not in ['enhancement', 'QA']]
+		params = {}
+		params['Bugzilla_token'] = login_data['token']
+		params['summary'] = cp
+		bugs = [x for x in bugzilla.Bug.search(params)['bugs'] if x['is_open'] and x['severity'] not in ['enhancement', 'QA']]
 		if bugs:
-			print 'has bugs'
+			print('has bugs')
 			continue
 
 		# Protection against filing a stabilization bug twice.
-		bugs = bugzilla.Bug.search({'summary': best_candidate})['bugs']
+		params = {}
+		params['Bugzilla_token'] = login_data['token']
+		params['summary'] = best_candidate
+		bugs = bugzilla.Bug.search(params)['bugs']
 		if bugs:
-			print 'version has bugs'
+			print('version has bugs')
 			continue
 
 		ebuild_name = portage.versions.catsplit(best_candidate)[1] + ".ebuild"
@@ -160,8 +164,8 @@ if __name__ == "__main__":
 		try:
 			original_contents = open(ebuild_path).read()
 			manifest_contents = open(manifest_path).read()
-		except IOError, e:
-			print e
+		except IOError as e:
+			print(e)
 			continue
 		try:
 			for arch in options.arch:
@@ -169,7 +173,7 @@ if __name__ == "__main__":
 			subprocess.check_output(["repoman", "manifest"], cwd=cvs_path)
 			subprocess.check_output(["repoman", "full"], cwd=cvs_path)
 		except subprocess.CalledProcessError:
-			print 'repoman error'
+			print('repoman error')
 			continue
 		finally:
 			f = open(ebuild_path, "w")
@@ -182,4 +186,8 @@ if __name__ == "__main__":
 		with open(options.output_filename, 'a') as f:
 			f.write('# %s %s\n' % (maintainer, ', '.join(other_maintainers)))
 			f.write('%s\n' % best_candidate)
-		print (best_candidate, maintainer, other_maintainers)
+		print((best_candidate, maintainer, other_maintainers))
+
+	params = {}
+	params['Bugzilla_token'] = login_data['token']
+	bugzilla.User.logout(params)