From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 18303 invoked by uid 1002); 24 Jun 2003 21:46:34 -0000 Mailing-List: contact gentoo-dev-help@gentoo.org; run by ezmlm Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-dev@gentoo.org Received: (qmail 32596 invoked from network); 24 Jun 2003 21:46:34 -0000 X-Authentication-Warning: sam.unet.brandeis.edu: rossgir owned process doing -bs Date: Tue, 24 Jun 2003 17:46:33 -0400 (EDT) From: ross b girshick X-X-Sender: rossgir@sam.unet.brandeis.edu To: gentoo-dev@gentoo.org In-Reply-To: <20030624090847.6cfbdd51.seemant@gentoo.org> Message-ID: MIME-Version: 1.0 Content-Type: TEXT/PLAIN; charset=US-ASCII Subject: Re: [gentoo-dev] "Updating Portage Cache" optimizations X-Archives-Salt: 64f13e7d-80c6-4660-8cf5-e4f8be319fd7 X-Archives-Hash: e644b3183b408ee44b5dd3f0055af53f On Tue, 24 Jun 2003, Seemant Kulleen wrote: > no I meant rsync on the local filesystem Ah, I see what you mean. Here's a patch to do that. The first time it runs, it takes my machine ~ 3 minutes (compared to 4.5 in past) and after that usually <1 minute (30 seconds if there aren't many updates to to the tree). If people are interested in testing this out and/or providing feedback please do. I've been using it all day without anything breaking (i wish you luck too) ;). Ross Girshick --- emerge.orig 2003-06-24 15:51:10.000000000 -0400 +++ emerge 2003-06-24 17:40:26.000000000 -0400 @@ -1656,31 +1656,55 @@ sys.exit(1) if os.path.exists(myportdir+"/metadata/cache"): print "\n>>> Updating Portage cache... ", - os.umask(0002) - if os.path.exists(portage.dbcachedir): - portage.spawn("rm -Rf "+portage.dbcachedir,free=1) - try: - os.mkdir(portage.dbcachedir) - os.chown(portage.dbcachedir, os.getuid(), portage.portage_gid) - os.chmod(portage.dbcachedir, 06775) - os.umask(002) - except: - pass - mynodes=portage.portdb.cp_all() + sys.stdout.flush() + # We shouldn't have to worry about this because when portage is imported dbcachedir is created if it's missing + if not os.path.exists(portage.dbcachedir): + print "!!! Cache Directory " + portage.dbcachedir + " does not exist. Re-running emerge should fix this" + sys.exit(1) + # XXX If we don't --delete, then we don't have to regenerate the cache files...what danger does this create? + # maybe it's sufficient to use --delete only every N syncs??? XXX + #update_cache_command = "/usr/bin/rsync -rlptD --delete --delete-after "... + update_cache_command = "/usr/bin/rsync -rlptD " + myportdir + "/metadata/cache/ " + portage.dbcachedir + exitcode = portage.spawn(update_cache_command, free=1) + # print update_cache_command + if (exitcode > 0): + ## more error info might be good + print "!!! local rsync error (cache update failed): " + exitcode + "\n" + sys.exit(1) + mynodes = portage.portdb.cp_all() for x in mynodes: - myxsplit=x.split("/") - if not os.path.exists(portage.dbcachedir+"/"+myxsplit[0]): - os.mkdir(portage.dbcachedir+"/"+myxsplit[0]) - os.chown(portage.dbcachedir+"/"+myxsplit[0], os.getuid(), portage.portage_gid) - os.chmod(portage.dbcachedir+"/"+myxsplit[0], 06775) - mymatches=portage.portdb.xmatch("match-all",x) + mymatches = portage.portdb.xmatch("match-all", x) for y in mymatches: update_spinner() + mydbkey = portage.dbcachedir+y + myebuild, in_overlay = portage.portdb.findname2(y) + myebuild_mtime = os.stat(myebuild)[ST_MTIME] try: - ignored=portage.portdb.aux_get(y,[],metacachedir=myportdir+"/metadata/cache") - except: - pass - portage.spawn("chmod -R g+rw "+portage.dbcachedir, free=1) + mydbkeystat = os.stat(mydbkey) + if mydbkeystat[ST_SIZE] == 0 or myebuild_mtime != mydbkeystat[ST_MTIME]: + doregen = 1 + else: + doregen = 0 + except OSError: + doregen = 1 + + if doregen: + ##print "doregen " + mydbkey + "\n" + try: + os.unlink(mydbkey) + except: + pass + # regenerate the dep cache file using doebuild interface + if portage.doebuild(myebuild, "depend", "/"): + #depend returned non-zero exit code... + sys.stderr.write(str(red("\nemerge sync:")+" (0) Error in "+y+" ebuild.\n" + " Check for syntax error or corruption in the ebuild. (--debug)\n\n")) + try: + os.utime(mydbkey, (myebuild_mtime, myebuild_mtime)) + except (IOError, OSError): + sys.stderr.write(str(red("\nemerge sync:")+" (1) Error in "+y+" ebuild.\n" + " Check for syntax error or corruption in the ebuild. (--debug)\n\n")) + ##portage.spawn("chmod -R g+rw "+portage.dbcachedir, free=1) sys.stdout.write("\b\b ...done!\n\n") sys.stdout.flush() -- gentoo-dev@gentoo.org mailing list