From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1Q2W5b-0003Jk-9m for garchives@archives.gentoo.org; Wed, 23 Mar 2011 21:59:39 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id CE35A1C029; Wed, 23 Mar 2011 21:59:22 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 8D5EA1C029 for ; Wed, 23 Mar 2011 21:59:22 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-CAMELLIA256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id DB53A1BC0D0 for ; Wed, 23 Mar 2011 21:59:21 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id 456A98006E for ; Wed, 23 Mar 2011 21:59:21 +0000 (UTC) From: "Nirbheek Chauhan" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Nirbheek Chauhan" Message-ID: <2af0175847b6b8d097e7e3dbbe11228c724eaaf4.nirbheek@gentoo> Subject: [gentoo-commits] proj/gnome:master commit in: scripts/ X-VCS-Repository: proj/gnome X-VCS-Files: scripts/slot_rindex.py X-VCS-Directories: scripts/ X-VCS-Committer: nirbheek X-VCS-Committer-Name: Nirbheek Chauhan X-VCS-Revision: 2af0175847b6b8d097e7e3dbbe11228c724eaaf4 Date: Wed, 23 Mar 2011 21:59:21 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: d5da834b1439be0759ed1af127b87bcd commit: 2af0175847b6b8d097e7e3dbbe11228c724eaaf4 Author: Nirbheek Chauhan gentoo org> AuthorDate: Wed Mar 23 12:54:16 2011 +0000 Commit: Nirbheek Chauhan gentoo org> CommitDate: Wed Mar 23 21:59:03 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/gnome.git;a=3D= commit;h=3D2af01758 slot_rindex.py: add slower portage-only mode, reorganize code * Set PORTAGE_ONLY to True to use the slower code * It's about 5x slower on hot cache --- scripts/slot_rindex.py | 123 ++++++++++++++++++++++++++++++++----------= ----- 1 files changed, 84 insertions(+), 39 deletions(-) diff --git a/scripts/slot_rindex.py b/scripts/slot_rindex.py index bf5f834..3610813 100755 --- a/scripts/slot_rindex.py +++ b/scripts/slot_rindex.py @@ -11,26 +11,31 @@ # Currently prints out a list of revdeps which *don't* use a slot in the # dependency atom containing the given library # -# TODO: Add a slower portage-only mode which calculates the required rin= dex -# =20 import sys -import urllib2 import os.path as osp =20 import portage from portage.xml.metadata import MetaDataXML =20 +portdb =3D portage.portdb +portdb.porttrees =3D [portage.settings['PORTDIR']] +PORTDIR =3D portage.settings['PORTDIR'] +DEPVARS =3D ['RDEPEND', 'PDEPEND', 'DEPEND'] + +##################### +### Configuration ### +##################### if len(sys.argv) < 2: print "Usage: %s " % sys.argv[0] sys.exit(1) =20 -portage.portdb.porttrees =3D [portage.settings['PORTDIR']] -PORTDIR =3D portage.settings['PORTDIR'] -RINDEX =3D "http://tinderbox.dev.gentoo.org/misc/rindex" -DEPSTR =3D ['RDEPEND', 'PDEPEND', 'DEPEND'] KEY =3D sys.argv[1] +PORTAGE_ONLY =3D False =20 +######################## +### Output Functions ### +######################## def get_herds(): return osp.join(PORTDIR, 'metadata', 'herds.xml') =20 @@ -50,7 +55,9 @@ def rdeps_with_slot(slot_rdeps, slot=3DNone): if not slot_rdeps.has_key(slot): # No rdeps using the given slot return + print "-------------------------------" print "All packages:" + print "-------------------------------" for pkg in slot_rdeps[slot]: pkg_md =3D MetaDataXML(get_md_path(pkg), get_herds()) for herd in pkg_md.herds(): @@ -69,54 +76,92 @@ def rdeps_with_slot(slot_rdeps, slot=3DNone): print '%s' % i.email, print =20 + print "-------------------------------" print "Herd packages:" + print "-------------------------------" for (herd, pkgs) in pkg_herds.iteritems(): print 'Herd: %s' % herd for pkg in pkgs: print '\t%s' % pkg =20 + print "-------------------------------" print "Maintainer packages:" + print "-------------------------------" for (maint, pkgs) in pkg_maints.iteritems(): print 'Maintainer: %s' % maint for pkg in pkgs: print '\t%s' % pkg =20 +############################# +### Portage API Functions ### +############################# +def get_deps_both(cpv, depvars=3DDEPVARS): + """ + Parses the dependency variables listed in depvars for cpv =20 -vrdeps =3D urllib2.urlopen('/'.join([RINDEX, KEY])).read().split() -rdeps =3D [] -for i in vrdeps: - rdeps.append(i.split(':')[0]) - -slot_rdeps =3D {} -failed_rdeps =3D [] -for rdep in rdeps: - rdep =3D rdep.split(':')[0] - if not portage.isvalidatom('=3D'+rdep): - print 'Invalid atom: ' + rdep - continue + returns (set(dep_cps), set(dep_strs)) + """ + dep_cps =3D set() + dep_strs =3D set() + raw_deps =3D [] try: - temp =3D portage.portdb.aux_get(rdep, DEPSTR)[0].split() + raw_deps =3D portdb.aux_get(cpv, depvars)[0].split() except KeyError: - failed_rdeps.append(rdep) + return (dep_cps, dep_strs) + for dep in portage.dep.use_reduce(' '.join(raw_deps), + matchall=3DTrue, flat=3DTrue): + # Ignore blockers, etc + if portage.isvalidatom(dep): + dep_strs.add(dep) + dep_cps.add(portage.dep.dep_getkey(dep)) + return (dep_cps, dep_strs) + +def get_revdeps_rindex(key): + """ + Given a key, returns a reverse-dependency list of that key using the= tinderbox rindex + """ + import urllib2 + RINDEX =3D "http://tinderbox.dev.gentoo.org/misc/rindex" + revdeps =3D [] + rdeps_raw =3D urllib2.urlopen('/'.join([RINDEX, key])).read().split(= ) + for i in rdeps_raw: + cpv =3D i.split(':')[0] + if portage.isvalidatom('=3D'+cpv): + revdeps.append(cpv) + return revdeps + +def get_revdeps_portage(key): + """ + Given a key, returns a reverse-dependency list of that key using por= tage API + """ + revdeps =3D [] + for cp in portdb.cp_all(): + cpvrs =3D portdb.xmatch('match-all', cp) + for cpvr in cpvrs: + if key in get_deps_both(cpvr)[0]: + revdeps.append(cpvr) + return revdeps + +################### +### Actual Work ### +################### +slot_rdeps =3D {} +revdeps =3D [] +if PORTAGE_ONLY: + revdeps =3D get_revdeps_portage(KEY) +else: + revdeps =3D get_revdeps_rindex(KEY) + +for rdep in revdeps: + (cps, deps) =3D get_deps_both(rdep) + if KEY not in cps: continue - for dep in temp: - # Ignore ||, (, ), etc. - if not portage.isvalidatom(dep): + for cpv in deps: + if cpv.find(KEY) =3D=3D -1: continue - # Categorize the dep into the slot it uses - if portage.dep.dep_getkey(dep) =3D=3D KEY: - slot =3D portage.dep.dep_getslot(dep) - if not slot_rdeps.has_key(slot): - # We use a set here because atoms often get repeated - slot_rdeps[slot] =3D set() - slot_rdeps[slot].add(rdep) - -# Convert back to list, and sort the atoms -for slot in slot_rdeps.keys(): - slot_rdeps[slot] =3D list(slot_rdeps[slot]) - slot_rdeps[slot].sort() - -if failed_rdeps: - print 'Failed: ' + str(failed_rdeps) + slot =3D portage.dep.dep_getslot(cpv) + if not slot_rdeps.has_key(slot): + slot_rdeps[slot] =3D [] + slot_rdeps[slot].append(rdep) =20 rdeps_with_slot(slot_rdeps)