From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from smtp.gentoo.org (smtp.gentoo.org [134.68.220.30]) by robin.gentoo.org (8.13.4/8.13.4) with ESMTP id j3SGeNSP029420 for ; Thu, 28 Apr 2005 16:40:24 GMT Received: from [195.37.161.2] (helo=cojobo.bonn.de) by smtp.gentoo.org with esmtp (Exim 4.43) id 1DRC3y-0005u2-G1 for gentoo-dev@lists.gentoo.org; Thu, 28 Apr 2005 16:40:30 +0000 Received: from localhost (localhost [127.0.0.1]) by cojobo.bonn.de (Postfix) with ESMTP id 1E0933B941 for ; Thu, 28 Apr 2005 18:40:30 +0200 (CEST) Received: from cojobo.bonn.de ([127.0.0.1]) by localhost (korinth [127.0.0.1]) (amavisd-new, port 10024) with SMTP id 22622-06 for ; Thu, 28 Apr 2005 18:40:25 +0200 (CEST) Received: from heino (p50884BD1.dip.t-dialin.net [80.136.75.209]) (using TLSv1 with cipher RC4-MD5 (128/128 bits)) (No client certificate requested) by cojobo.bonn.de (Postfix) with ESMTP id 6408939BAE for ; Thu, 28 Apr 2005 18:40:24 +0200 (CEST) From: Heinrich Wendel To: gentoo-dev@lists.gentoo.org Subject: [gentoo-dev] Portage ebuild cruft User-Agent: KMail/1.8 Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-dev@gentoo.org Reply-to: gentoo-dev@lists.gentoo.org MIME-Version: 1.0 Organization: Gentoo Foundation Date: Thu, 28 Apr 2005 18:40:23 +0200 Content-Type: Multipart/Mixed; boundary="Boundary-00=_3HRcCZOjjx1ZmIH" Message-Id: <200504281840.23788.lanius@gentoo.org> X-Virus-Scanned: by amavisd-new-20030616-p10 (Debian) at cojobo.net X-Archives-Salt: e1c03355-9a42-46e4-b9fe-349ce492be47 X-Archives-Hash: 2e64846b97d8b929fedbeb435dc1c31c --Boundary-00=_3HRcCZOjjx1ZmIH Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Content-Disposition: inline Hi, Portage is slow? How to make it faster? By removing unused ebuilds! I wrote a little script to check which ebuilds in portage aren't used anylonger, here the result: Total packages checked: 9076 Total ebuilds checked: 18662 Total ebuilds to remove: 4643 Of course the script can't detect every ebuild situation, so take the numbers with care. But still it shows that 1/4 of all ebuilds could be removed. This would improve portage performance by at least 1/4, so developers go ahead. The script is attached, just run it as if it was repoman, it won't do anything, just show the orphaned packages. mfg, heinrich :-) --Boundary-00=_3HRcCZOjjx1ZmIH Content-Type: application/x-python; name="cleanup.py" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="cleanup.py" #!/usr/bin/python ################################################### # Remove unneeded ebuilds/files # # (c) 2005 by Heinrich Wendel # # Version 0.1 # # Todo: # # - check FILESDIR # # - check if other ebuilds depend on this version # ################################################### import os, sys, re def check(portdir, category, package): global count_ebuilds, count_packages, count_remove # get list/dictionary of ebuilds ebuilds = {} for entry in os.listdir(os.path.join(portdir, category, package)): if entry.find(".ebuild") > 0: count_ebuilds += 1 # entry entry = entry.replace(".ebuild", "") # slot slot = portage.db["/"]["porttree"].getslot(category + "/" + entry) # keywords keyword = "" for line in file(os.path.join(portdir, category, package , entry) + ".ebuild", "r"): if line.find("KEYWORDS") == 0: try: tmp = line.split('"') keyword = tmp[1].split(" ") except IndexError: tmp = line.split("'") keyword = tmp[1].split(" ") break if not keyword: return # status status = portage.getmaskingstatus(category + "/" + entry) # files #files = [] #command = "grep '${FILESDIR}' %s.ebuild" % entry #f = os.popen(command) #for line in f.readlines(): # if line == "": # continue # for datei in re.findall('\$\{FILESDIR\}[^ ]*', line): # files.append(datei.strip()) #f.close() #print files ## replace P, PV, PN, PF # create entry if ebuilds.has_key(slot): ebuilds[slot].append([entry,keyword,status]) else: ebuilds[slot] = [ [entry,keyword,status] ] if len(ebuilds) == 0: return # sort ebuilds according to priority def sort_ebuilds(pkg1, pkg2): pkg1 = portage.pkgsplit(pkg1[0]) pkg2 = portage.pkgsplit(pkg2[0]) return portage.pkgcmp(pkg1,pkg2) # check which packages are still needed needed = [] for slot, builds in ebuilds.iteritems(): builds.sort(sort_ebuilds) builds.reverse() # check for the newest stable version on earch arch for arch in portage.archlist: for ebuild in builds: if arch in ebuild[1]: # check if package is masked if not "package.mask" in ebuild[2]: if not ebuild in needed: needed.append(ebuild) break # get package.masked pkg's and -* packages for ebuild in builds: if "package.mask" in ebuild[2] or ebuild[2] == ["-* keyword"]: if not ebuild in needed: needed.append(ebuild) # all packages not in needed can be removed remove = [] for slot, builds in ebuilds.iteritems(): for ebuild in builds: if ebuild not in needed: remove.append(ebuild[0] + ".ebuild") # check files dir # check_files(portdir,category,package,"files") # print collected information if len(ebuilds) > 0: print "Checking Package: " + category + "/" + package for package in remove: print " * " + package count_remove += 1 count_packages += 1 #def check_files(portdir,category,package,curdir): # global count_files # for file in os.listdir(os.path.join(portdir,category,package,curdir)): # count_files += 1 # if file == "CVS": # pass # elif file.find("digest") == 0: # pass # elif os.path.isdir(os.path.join(portdir,category,package,curdir,file)): # check_files(portdir,category,package,os.path.join(curdir,file)) # else: # pass # where are we? portdir = "" package = "" category = "" count_remove = 0 count_ebuilds = 0 count_packages = 0 count_files = 0 if os.path.isdir("profiles"): portdir = os.getcwd() elif os.path.isdir("files"): (portdir, category, package) = os.getcwd().rsplit("/", 2) elif os.path.isdir("../profiles"): (portdir, category) = os.getcwd().rsplit("/", 1) else: print "Not in a portage directory" sys.exit(1) os.environ["PORTDIR"] = portdir import portage # loop trough packages if category and package: check(portdir, category, package) elif category: packages = os.listdir(os.path.join(portdir, category)) packages.sort() for package in packages: if os.path.isdir(package): check(portdir, category, package) else: categories = os.listdir(portdir) categories.sort() for category in categories: if os.path.isdir(category): packages = os.listdir(os.path.join(portdir, category)) packages.sort() for package in packages: if os.path.isdir(os.path.join(category, package)): check(portdir, category, package) print "" print "Total packages checked: " + str(count_packages) print "Total ebuilds checked: " + str(count_ebuilds) print "Total ebuilds to remove: " + str(count_remove) #print "Total files in ${FILESDIR}: " + str(count_files) --Boundary-00=_3HRcCZOjjx1ZmIH-- -- gentoo-dev@gentoo.org mailing list