From mboxrd@z Thu Jan 1 00:00:00 1970
Received: from lists.gentoo.org ([140.105.134.102] helo=robin.gentoo.org)
by nuthatch.gentoo.org with esmtp (Exim 4.50)
id 1EWH0K-0004qm-WC
for garchives@archives.gentoo.org; Sun, 30 Oct 2005 17:30:02 +0000
Received: from robin.gentoo.org (localhost [127.0.0.1])
by robin.gentoo.org (8.13.5/8.13.5) with SMTP id j9UHSBii021757;
Sun, 30 Oct 2005 17:28:11 GMT
Received: from smtp.gentoo.org (smtp.gentoo.org [134.68.220.30])
by robin.gentoo.org (8.13.5/8.13.5) with ESMTP id j9UHS9Bw020438
for <gentoo-portage-dev@lists.gentoo.org>; Sun, 30 Oct 2005 17:28:09 GMT
Received: from cpe-65-26-255-237.wi.res.rr.com ([65.26.255.237] helo=nightcrawler)
by smtp.gentoo.org with esmtpa (Exim 4.43)
id 1EWGyX-0002g1-40
for gentoo-portage-dev@lists.gentoo.org; Sun, 30 Oct 2005 17:28:09 +0000
Date: Sun, 30 Oct 2005 11:27:57 -0600
From: Brian Harring <ferringb@gentoo.org>
To: gentoo-portage-dev@lists.gentoo.org
Subject: [gentoo-portage-dev] [3/3] Cache subsystem rewrite
Message-ID: <20051030172756.GJ6443@nightcrawler>
References: <20051030170526.GG6443@nightcrawler>
Precedence: bulk
List-Post: <mailto:gentoo-portage-dev@lists.gentoo.org>
List-Help: <mailto:gentoo-portage-dev+help@gentoo.org>
List-Unsubscribe: <mailto:gentoo-portage-dev+unsubscribe@gentoo.org>
List-Subscribe: <mailto:gentoo-portage-dev+subscribe@gentoo.org>
List-Id: Gentoo Linux mail <gentoo-portage-dev.gentoo.org>
X-BeenThere: gentoo-portage-dev@gentoo.org
Reply-to: gentoo-portage-dev@lists.gentoo.org
Mime-Version: 1.0
Content-Type: multipart/signed; micalg=pgp-sha1;
protocol="application/pgp-signature"; boundary="cs5saTBZh7UZl2eX"
Content-Disposition: inline
In-Reply-To: <20051030170526.GG6443@nightcrawler>
User-Agent: Mutt/1.5.8i
X-Archives-Salt: 15c35f3a-baae-48bc-8653-f9e6b5b489ef
X-Archives-Hash: 6c6f60e314cdfb978a9dd4d5633ea5a7
--cs5saTBZh7UZl2eX
Content-Type: multipart/mixed; boundary="kunpHVz1op/+13PW"
Content-Disposition: inline
--kunpHVz1op/+13PW
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
removal of the old cache modules.
'nuff said.
~harring
--kunpHVz1op/+13PW
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="remove-old-cache.patch"
Content-Transfer-Encoding: quoted-printable
Index: pym/portage_db_cpickle.py
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- pym/portage_db_cpickle.py (revision 2121)
+++ pym/portage_db_cpickle.py (working copy)
@@ -1,79 +0,0 @@
-# Copyright 2004 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-# $Id: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_cpickle.py,v 1=
=2E9.2.2 2005/04/23 07:26:04 jstubbs Exp $
-
-
-import anydbm,cPickle,types
-from os import chown,access,R_OK,unlink
-import os
-
-import portage_db_template
-
-class database(portage_db_template.database):
- def module_init(self):
- self.modified =3D False
- =09
- prevmask=3Dos.umask(0)
- if not os.path.exists(self.path):
- os.makedirs(self.path, 02775)
-
- self.filename =3D self.path + "/" + self.category + ".cpickle"
- =09
- if access(self.filename, R_OK):
- try:
- mypickle=3DcPickle.Unpickler(open(self.filename,"r"))
- mypickle.find_global=3DNone
- self.db =3D mypickle.load()
- except SystemExit, e:
- raise
- except:
- self.db =3D {}
- else:
- self.db =3D {}
-
- os.umask(prevmask)
-
- def has_key(self,key):
- self.check_key(key)
- if self.db.has_key(key):
- return 1
- return 0
- =09
- def keys(self):
- return self.db.keys()
-=09
- def get_values(self,key):
- self.check_key(key)
- if self.db.has_key(key):
- return self.db[key]
- return None
-=09
- def set_values(self,key,val):
- self.modified =3D True
- self.check_key(key)
- self.db[key] =3D val
-=09
- def del_key(self,key):
- if self.has_key(key):
- del self.db[key]
- self.modified =3D True
- return True
- return False
- =09
- def sync(self):
- if self.modified:
- try:
- if os.path.exists(self.filename):
- unlink(self.filename)
- cPickle.dump(self.db, open(self.filename,"w"), -1)
- os.chown(self.filename,self.uid,self.gid)
- os.chmod(self.filename, 0664)
- except SystemExit, e:
- raise
- except:
- pass
-=09
- def close(self):
- self.sync()
- self.db =3D None;
-=09
Index: pym/portage_db_anydbm.py
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- pym/portage_db_anydbm.py (revision 2121)
+++ pym/portage_db_anydbm.py (working copy)
@@ -1,64 +0,0 @@
-# Copyright 2004 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-# $Id: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_anydbm.py,v 1.=
11.2.1 2005/01/16 02:35:33 carpaski Exp $
-
-
-import anydbm,cPickle,types,os
-
-import portage_db_template
-
-class database(portage_db_template.database):
- def module_init(self):
- prevmask=3Dos.umask(0)
- if not os.path.exists(self.path):
- current_path=3D"/"
- for mydir in self.path.split("/"):
- current_path +=3D "/"+mydir
- if not os.path.exists(current_path):
- os.mkdir(current_path)
-
- self.filename =3D self.path + "/" + self.category + ".anydbm"
- =09
- try:
- # open it read/write
- self.db =3D anydbm.open(self.filename, "c", 0664)
- except SystemExit, e:
- raise
- except:
- # Create a new db... DB type not supported anymore?
- self.db =3D anydbm.open(self.filename, "n", 0664)
-
- os.umask(prevmask)
-
- def has_key(self,key):
- self.check_key(key)
- if self.db.has_key(key):
- return 1
- return 0
- =09
- def keys(self):
- return self.db.keys()
-=09
- def get_values(self,key):
- self.check_key(key)
- if self.db.has_key(key):
- myval =3D cPickle.loads(self.db[key])
- return myval
- return None
-=09
- def set_values(self,key,val):
- self.check_key(key)
- self.db[key] =3D cPickle.dumps(val,cPickle.HIGHEST_PROTOCOL)
-=09
- def del_key(self,key):
- if self.has_key(key):
- del self.db[key]
- return True
- return False
- =09
- def sync(self):
- self.db.sync()
-=09
- def close(self):
- self.db.close()
-=09
Index: pym/portage_db_template.py
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- pym/portage_db_template.py (revision 2121)
+++ pym/portage_db_template.py (working copy)
@@ -1,174 +0,0 @@
-# Copyright 2004 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-# $Id: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_template.py,v =
1.11.2.1 2005/01/16 02:35:33 carpaski Exp $
-
-
-import os.path,string
-from portage_util import getconfig, ReadOnlyConfig
-from portage_exception import CorruptionError
-
-class database:
- def __init__(self,path,category,dbkeys,uid,gid,config_path=3D"/etc/portag=
e/module_configs/"):
- self.__cacheArray =3D [None, None, None]
- self.__cacheKeyArray =3D [None, None, None]
- self.__template_init_called =3D True
- self.path =3D path
- self.category =3D category
- self.dbkeys =3D dbkeys
- self.uid =3D uid
- self.gid =3D gid
-
- self.config =3D None
- self.__load_config(config_path)
-
- self.module_init()
-=09
- def getModuleName(self):
- return self.__module__+"."+self.__class__.__name__[:]
-
- def __load_config(self,config_path):
- config_file =3D config_path + "/" + self.getModuleName()
- self.config =3D ReadOnlyConfig(config_file)
-
- def __check_init(self):
- try:
- if self.__template_init_called:
- pass
- except SystemExit, e:
- raise
- except:
- raise NotImplementedError("db_template.__init__ was overridden")
-
- def check_key(self,key):
- if (not key) or not isinstance(key, str):
- raise KeyError, "No key provided. key: %s" % (key)
-=09
- def clear(self):
- for x in self.keys():
- self.del_key(x)
-
- def __addCache(self,key,val):
- del self.__cacheArray[2]
- self.__cacheArray.insert(0,val)
- del self.__cacheKeyArray[2]
- self.__cacheKeyArray.insert(0,key)
-
- def __delCache(self,key):
- i =3D self.__cacheKeyArray.index(key)
- self.__cacheArray[i] =3D None
- self.__cacheKeyArray[i] =3D None
-
- def flushCache(self):
- self.__cacheArray =3D [None, None, None]
- self.__cacheKeyArray =3D [None, None, None]
-
- def __getitem__(self,key):
- if key in self.__cacheKeyArray:
- i =3D self.__cacheKeyArray.index(key)
- return self.__cacheArray[i]
-
- self.check_key(key)
- if self.has_key(key):
- try:
- values =3D self.get_values(key)
- self.__addCache(key,values)
- return values
- except SystemExit, e:
- raise
- except Exception, e:
- raise CorruptionError("Corruption detected when reading key '%s': %s" =
% (key,str(e)))
- raise KeyError("Key not in db: '%s'" % (key))
-=09
- def __setitem__(self,key,values):
- self.check_key(key)
- self.__addCache(key,values)
- return self.set_values(key,values)
-
- def __delitem__(self,key):
- self.__delCache(key)
- return self.del_key(key)
-
- def has_key(self,key):
- raise NotImplementedError("Method not defined")
-=09
- def keys(self):
- raise NotImplementedError("Method not defined")
-
- def get_values(self,key):
- raise NotImplementedError("Method not defined")
-=09
- def set_values(self,key,val):
- raise NotImplementedError("Method not defined")
-
- def del_key(self,key):
- raise NotImplementedError("Method not defined")
- =09
- def sync(self):
- raise NotImplementedError("Method not defined")
-=09
- def close(self):
- raise NotImplementedError("Method not defined")
-
-
-=09
-def test_database(db_class,path,category,dbkeys,uid,gid):
- if "_mtime_" not in dbkeys:
- dbkeys+=3D["_mtime_"]
- d =3D db_class(path,category,dbkeys,uid,gid)
-
- print "Module: "+str(d.__module__)
-
- # XXX: Need a way to do this that actually works.
- for x in dir(database):
- if x not in dir(d):
- print "FUNCTION MISSING:",str(x)
-
- list =3D d.keys()
- if(len(list) =3D=3D 0):
- values =3D {}
- for x in dbkeys:
- values[x] =3D x[:]
- values["_mtime_"] =3D "1079903037"
- d.set_values("test-2.2.3-r1", values)
- d.set_values("test-2.2.3-r2", values)
- d.set_values("test-2.2.3-r3", values)
- d.set_values("test-2.2.3-r4", values)
-
- list =3D d.keys()
- print "Key count:",len(list)
-
- values =3D d.get_values(list[0])
- print "value count:",len(values)
-=09
- mykey =3D "foobar-1.2.3-r4"
-=09
- d.check_key(mykey)
- d.set_values(mykey, values)
- d.sync()
- del d
-
- d =3D db_class(path,category,dbkeys,uid,gid)
- new_vals =3D d.get_values(mykey)
-
- if dbkeys and new_vals:
- for x in dbkeys:
- if x not in new_vals.keys():
- print "---",x
- for x in new_vals.keys():
- if x not in dbkeys:
- print "+++",x
- else:
- print "Mismatched:",dbkeys,new_vals
-=09
- d.del_key(mykey)
-=09
- print "Should be None:",d.get_values(mykey)
-
- d.clear()
-
- d.sync
- d.close
-=09
- del d
-=09
- print "Done."
Index: pym/portage_db_test.py
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- pym/portage_db_test.py (revision 2121)
+++ pym/portage_db_test.py (working copy)
@@ -1,21 +0,0 @@
-#!/usr/bin/python -O
-# Copyright 2004 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-# $Id: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_test.py,v 1.3.=
2.1 2005/01/16 02:35:33 carpaski Exp $
-
-
-import portage
-import portage_db_template
-import portage_db_anydbm
-import portage_db_flat
-import portage_db_cpickle
-
-import os
-
-uid =3D os.getuid()
-gid =3D os.getgid()
-
-portage_db_template.test_database(portage_db_flat.database,"/var/cache/edb=
/dep", "sys-apps",portage.auxdbkeys,uid,gid)
-portage_db_template.test_database(portage_db_cpickle.database,"/var/cache/=
edb/dep","sys-apps",portage.auxdbkeys,uid,gid)
-portage_db_template.test_database(portage_db_anydbm.database,"/var/cache/e=
db/dep", "sys-apps",portage.auxdbkeys,uid,gid)
-
Index: pym/portage_db_metadata.py
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- pym/portage_db_metadata.py (revision 2121)
+++ pym/portage_db_metadata.py (working copy)
@@ -1,49 +0,0 @@
-# Copyright 2004 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_flat.py,v =
1.13.2.6 2005/04/19 07:14:17 ferringb Exp $
-cvs_id_string=3D"$Id: portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 fe=
rringb Exp $"[5:-2]
-
-import os, portage_db_flat_hash, portage_db_flat
-
-class database(portage_db_flat_hash.database):
-=09
- def get_values(self, key):
- if not key:
- raise KeyError("key is not valid")
- =09
- try:
- myf =3D open(self.fullpath + key, "r")
- except OSError:
- raise KeyError("key is not valid")
- mtime =3D os.fstat(myf.fileno()).st_mtime
- data =3D myf.read().splitlines()
- =09
- # easy attempt first.
- if len(data) !=3D portage_db_flat.magic_line_count:
- d =3D dict(map(lambda x: x.split("=3D",1), data))
- d["_mtime_"] =3D mtime
- return portage_db_flat_hash.database.get_values(self, key, d)
- # this one's interesting.
- d =3D {}
-
- for line in data:
- # yes, meant to iterate over a string.
- hashed =3D False
- for idx, c in enumerate(line):
- if not c.isalpha():
- if c =3D=3D "=3D" and idx > 0:
- hashed =3D True
- d[line[:idx]] =3D line[idx + 1:]
- elif c =3D=3D "_" or c.isdigit():
- continue
- break
- elif not c.isupper():
- break
-
- if not hashed:
- # non hashed.
- data.append(mtime)
- return portage_db_flat.database.get_values(self, key, data=3Ddata)
-
- d["_mtime_"] =3D mtime
- return portage_db_flat_hash.database.get_values(self, key, data=3Dd)
Index: pym/portage_db_flat.py
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- pym/portage_db_flat.py (revision 2121)
+++ pym/portage_db_flat.py (working copy)
@@ -1,124 +0,0 @@
-# Copyright 2004 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-# $Id: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_flat.py,v 1.13=
=2E2.6 2005/04/19 07:14:17 ferringb Exp $
-
-
-import types
-import os
-import stat
-
-import portage_db_template
-
-# since this format is massively deprecated,=20
-# we're hardcoding the previously weird line count
-magic_line_count =3D 22
-
-class database(portage_db_template.database):
- def module_init(self):
- self.lastkey =3D None # Cache
- self.lastval =3D None # Cache
-
- self.fullpath =3D self.path + "/" + self.category + "/"
-
- if not os.path.exists(self.fullpath):
- prevmask=3Dos.umask(0)
- os.makedirs(self.fullpath, 02775)
- os.umask(prevmask)
- try:
- os.chown(self.fullpath, self.uid, self.gid)
- os.chmod(self.fullpath, 02775)
- except SystemExit, e:
- raise
- except:
- pass
- =09
- def has_key(self,key):
- if os.path.exists(self.fullpath+key):
- return 1
- return 0
-=09
- def keys(self):
- # XXX: NEED TOOLS SEPERATED
- # return portage.listdir(self.fullpath,filesonly=3D1)
- mykeys =3D []
- for x in os.listdir(self.fullpath):
- if os.path.isfile(self.fullpath+x) and not x.startswith(".update."):
- mykeys +=3D [x]
- return mykeys
-
- def get_values(self,key, data=3DNone):
- """ do not use data unless you know what it does."""
-
- if not key:
- raise KeyError, "key is not set to a valid value"
-
- mydict =3D {}
- if data =3D=3D None:
- try:
- # give buffering a hint of the pretty much maximal cache size we deal =
with
- myf =3D open(self.fullpath+key, "r", 8192)
- except OSError:
- # either the file didn't exist, or it was removed under our feet.
- raise KeyError("failed reading key")
-
- # nuke the newlines right off the batt.
- data =3D myf.read().splitlines()
- mydict["_mtime_"] =3D os.fstat(myf.fileno()).st_mtime
- myf.close()
- else:
- mydict["_mtime_"] =3D data.pop(-1)
-
- # rely on exceptions to note differing line counts.
- try:
- for x in range(magic_line_count):
- mydict[self.dbkeys[x]] =3D data[x]
-
- except IndexError:
- raise ValueError, "Key count mistmatch"
-
- return mydict
-=09
- def set_values(self,key, val, raw=3DFalse):
- if not key:
- raise KeyError, "No key provided. key:%s val:%s" % (key,val)
- if not val:
- raise ValueError, "No value provided. key:%s val:%s" % (key,val)
- =09
- # XXX threaded cache updates won't play nice with this.
- # need a synchronization primitive, or locking (of the fileno, not a sep=
erate file)
- # to correctly handle threading.
-
- update_fp =3D self.fullpath + ".update." + str(os.getpid()) + "." + key
- myf =3D open(update_fp,"w")
- if not raw:
- myf.writelines( [ str(val[x]) +"\n" for x in self.dbkeys] )
- if len(self.dbkeys) !=3D magic_line_count:
- myf.writelines(["\n"] * len(self.dbkeys) - magic_line_count)
- mtime =3D val["_mtime_"]
- else:
- mtime =3D val.pop(-1)
- myf.writelines(val)
- myf.close()
- =09
- os.chown(update_fp, self.uid, self.gid)
- os.chmod(update_fp, 0664)
- os.utime(update_fp, (-1,long(mtime)))
- os.rename(update_fp, self.fullpath+key)
-
- def del_key(self,key):
- try:
- os.unlink(self.fullpath+key)
- except OSError, oe:
- # just attempt it without checking, due to the fact that
- # a cache update could be in progress.
- self.lastkey =3D None
- self.lastval =3D None
- return 0
- return 1
- =09
- def sync(self):
- return
-=09
- def close(self):
- return
-=09
--kunpHVz1op/+13PW--
--cs5saTBZh7UZl2eX
Content-Type: application/pgp-signature
Content-Disposition: inline
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.1 (GNU/Linux)
iD8DBQFDZQKcvdBxRoA3VU0RAv7GAKC35ENyUD9RFO4g/J127hZ8AqKXXgCgqkDb
f8Z1s1O1/5ZhMzCy5D4LOZ8=
=s2xh
-----END PGP SIGNATURE-----
--cs5saTBZh7UZl2eX--
--
gentoo-portage-dev@gentoo.org mailing list