[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     33b8ae45109ebbbd62a560690ccbaf2c009772b1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Feb  7 00:13:22 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb  7 00:13:22 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=33b8ae45

cache.sqlite: handle sqlite ImportError

This will fix bug #353836.

---
 pym/portage/cache/sqlite.py |   23 +++++++++++++++--------
 1 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 2e13be3..d15b6ec 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2010 Gentoo Foundation
+# Copyright 1999-2011 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import sys
@@ -8,11 +8,6 @@ from portage import os
 from portage import _unicode_decode
 from portage.util import writemsg
 from portage.localization import _
-try:
-	import sqlite3 as db_module # sqlite3 is optional with >=python-2.5
-except ImportError:
-	from pysqlite2 import dbapi2 as db_module
-DBError = db_module.Error
 
 if sys.hexversion >= 0x3000000:
 	basestring = str
@@ -25,12 +20,11 @@ class database(fs_template.FsBased):
 	# to calculate the number of pages requested, according to the following
 	# equation: cache_bytes = page_bytes * page_count
 	cache_bytes = 1024 * 1024 * 10
-	_db_module = db_module
-	_db_error = DBError
 	_db_table = None
 
 	def __init__(self, *args, **config):
 		super(database, self).__init__(*args, **config)
+		self._import_sqlite()
 		self._allowed_keys = ["_mtime_", "_eclasses_"]
 		self._allowed_keys.extend(self._known_keys)
 		self._allowed_keys.sort()
@@ -49,6 +43,19 @@ class database(fs_template.FsBased):
 		self._db_init_connection(config)
 		self._db_init_structures()
 
+	def _import_sqlite(self):
+		# sqlite3 is optional with >=python-2.5
+		try:
+			import sqlite3 as db_module
+		except ImportError:
+			try:
+				from pysqlite2 import dbapi2 as db_module
+			except ImportError as e:
+				raise cache_errors.InitializationError(self.__class__, e)
+
+		self._db_module = db_module
+		self._db_error = db_module.Error
+
 	def _db_escape_string(self, s):
 		"""meta escaping, returns quoted string for use in sql statements"""
 		if not isinstance(s, basestring):

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     ad382da72d24dbd0df1ee3696aacd638171a1cda
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb  8 06:35:14 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb  8 06:35:14 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ad382da7

cache.ebuild_xattr: don't import ENODATA globally

ENODATA is undefined on FreeBSD. This will fix bug #354057.

---
 pym/portage/cache/ebuild_xattr.py |   13 +++++++------
 1 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py
index 1d7e26f..6b388fa 100644
--- a/pym/portage/cache/ebuild_xattr.py
+++ b/pym/portage/cache/ebuild_xattr.py
@@ -4,6 +4,8 @@
 
 __all__ = ['database']
 
+import errno
+
 import portage
 from portage.cache import fs_template
 from portage.versions import catsplit
@@ -11,7 +13,6 @@ from portage import cpv_getkey
 from portage import os
 from portage import _encodings
 from portage import _unicode_decode
-from errno import ENODATA,ENOSPC,E2BIG
 portage.proxy.lazyimport.lazyimport(globals(),
 	'xattr')
 
@@ -58,16 +59,16 @@ class database(fs_template.FsBased):
 		except IOError as e:
 			# ext based give wrong errno
 			# http://bugzilla.kernel.org/show_bug.cgi?id=12793
-			if e.errno in (E2BIG,ENOSPC):
+			if e.errno in (errno.E2BIG, errno.ENOSPC):
 				result = len(s)-100
 			else:
-				raise e
+				raise
 
 		try:
 			self.__remove(path,'test_max')
 		except IOError as e:
-			if e.errno is not ENODATA:
-				raise e
+			if e.errno != errno.ENODATA:
+				raise
 
 		return result
 
@@ -87,7 +88,7 @@ class database(fs_template.FsBased):
 		try:
 			return xattr.get(path,key,namespace=self.ns)
 		except IOError as e:
-			if not default is None and ENODATA == e.errno:
+			if not default is None and errno.ENODATA == e.errno:
 				return default
 			else:
 				raise NoValueException()

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     34b54ba9e854b13f0413383979824ac3f90a2686
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 12 18:42:08 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 12 18:42:08 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=34b54ba9

cache.sqlite: handle readonly

---
 pym/portage/cache/sqlite.py |    5 +++--
 1 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index d15b6ec..4eb340b 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -31,7 +31,7 @@ class database(fs_template.FsBased):
 		self.location = os.path.join(self.location, 
 			self.label.lstrip(os.path.sep).rstrip(os.path.sep))
 
-		if not os.path.exists(self.location):
+		if not self.readonly and not os.path.exists(self.location):
 			self._ensure_dirs()
 
 		config.setdefault("autocommit", self.autocommits)
@@ -72,7 +72,8 @@ class database(fs_template.FsBased):
 		connection_kwargs = {}
 		connection_kwargs["timeout"] = config["timeout"]
 		try:
-			self._ensure_dirs()
+			if not self.readonly:
+				self._ensure_dirs()
 			self._db_connection = self._db_module.connect(
 				database=_unicode_decode(self._dbpath), **connection_kwargs)
 			self._db_cursor = self._db_connection.cursor()

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     d4d1e7145554a4dd3bdd7094b73243aef75d79f4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 12 18:58:27 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 12 18:58:27 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d4d1e714

cache.volatile: discard "perms" param

---
 pym/portage/cache/volatile.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/cache/volatile.py b/pym/portage/cache/volatile.py
index e671904..0bf6bab 100644
--- a/pym/portage/cache/volatile.py
+++ b/pym/portage/cache/volatile.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2009 Gentoo Foundation
+# Copyright 1999-2011 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import copy
@@ -11,6 +11,7 @@ class database(template.database):
 
 	def __init__(self, *args, **config):
 		config.pop("gid", None)
+		config.pop("perms", None)
 		super(database, self).__init__(*args, **config)
 		self._data = {}
 		self.__iter__ = self._data.__iter__

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     9d3564ac4479ac6367cf5f12f936edd61a67b114
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 12 19:04:54 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 12 19:04:54 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9d3564ac

cache.sqlite: handle readonly more

---
 pym/portage/cache/sqlite.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 4eb340b..fcc62ff 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -78,7 +78,7 @@ class database(fs_template.FsBased):
 				database=_unicode_decode(self._dbpath), **connection_kwargs)
 			self._db_cursor = self._db_connection.cursor()
 			self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8"))
-			if not self._ensure_access(self._dbpath):
+			if not self.readonly and not self._ensure_access(self._dbpath):
 				raise cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % self._dbpath)
 			self._db_init_cache_size(config["cache_bytes"])
 			self._db_init_synchronous(config["synchronous"])

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     f55f1714c7e2855ca4cbb45f9d8ae6cee126af17
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Sep  7 15:56:15 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Sep  7 15:56:15 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f55f1714

cache/volatile: fix __iter__ and __contains__

Special methods can't be assigned in the constructor.

---
 pym/portage/cache/volatile.py |    8 ++++++--
 1 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/volatile.py b/pym/portage/cache/volatile.py
index 0bf6bab..18049dd 100644
--- a/pym/portage/cache/volatile.py
+++ b/pym/portage/cache/volatile.py
@@ -14,12 +14,16 @@ class database(template.database):
 		config.pop("perms", None)
 		super(database, self).__init__(*args, **config)
 		self._data = {}
-		self.__iter__ = self._data.__iter__
 		self._delitem = self._data.__delitem__
-		self.__contains__ = self._data.__contains__
 
 	def _setitem(self, name, values):
 		self._data[name] = copy.deepcopy(values)
 
 	def _getitem(self, cpv):
 		return copy.deepcopy(self._data[cpv])
+
+	def __iter__(self):
+		return iter(self._data)
+
+	def __contains__(self, key):
+		return key in self._data

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     41f95b75ceb77d33f0335455788a93c68c58aa69
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 14 15:30:17 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Oct 14 15:30:17 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=41f95b75

Revert "Use plain ascii encoding for this file"

This reverts commit 2a4b07c8d0105ea7418ea3afc9e777a7a396fc46.
Current python ebuilds no longer remove the encodings directory when
USE=ebuild is enabled, so now it's possible to use UTF8 encoding in
python source files without triggering unsightly SyntaxError messages
when python_mod_optimize is compiling byte-code files.

---
 pym/portage/cache/ebuild_xattr.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py
index 6b388fa..0086e40 100644
--- a/pym/portage/cache/ebuild_xattr.py
+++ b/pym/portage/cache/ebuild_xattr.py
@@ -1,5 +1,6 @@
+# -*- coding: UTF8 -*-
 # Copyright: 2009-2011 Gentoo Foundation
-# Author(s): Petteri R&#228;ty (betelgeuse@gentoo.org)
+# Author(s): Petteri Räty (betelgeuse@gentoo.org)
 # License: GPL2
 
 __all__ = ['database']

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     095a648f4726a467934b819e214a2468308ca351
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 18 05:26:27 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct 18 05:26:27 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=095a648f

Fix serialize_eclasses = False in cache template.

This fixes volatile cache validation that did not get fixed in commit
a63c163a50bf8a4e5ca8673cd8bebae29c36643a. Now all volatile cache
problems that have been introduced since commit
2ed1cb53cc4158af08c22d466b15b9a9a7767212 should be fixed.

---
 pym/portage/cache/template.py |   34 ++++++++++++++++++++++++++++------
 pym/portage/cache/volatile.py |    2 +-
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index a76a5f5..515ba02 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -64,24 +64,46 @@ class database(object):
 		override this in derived classess"""
 		raise NotImplementedError
 
+	@staticmethod
+	def _internal_eclasses(extern_ec_dict, chf_type, paths):
+		"""
+		When serialize_eclasses is False, we have to convert an external
+		eclass dict containing hashed_path objects into an appropriate
+		internal dict containing values of chf_type (and eclass dirs
+		if store_eclass_paths is True).
+		"""
+		if not extern_ec_dict:
+			return extern_ec_dict
+		chf_getter = operator.attrgetter(chf_type)
+		if paths:
+			intern_ec_dict = dict((k, (v.eclass_dir, chf_getter(v)))
+				for k, v in extern_ec_dict.items())
+		else:
+			intern_ec_dict = dict((k, chf_getter(v))
+				for k, v in extern_ec_dict.items())
+		return intern_ec_dict
+
 	def __setitem__(self, cpv, values):
 		"""set a cpv to values
 		This shouldn't be overriden in derived classes since it handles the readonly checks"""
 		if self.readonly:
 			raise cache_errors.ReadOnlyRestriction()
+		d = None
 		if self.cleanse_keys:
 			d=ProtectedDict(values)
 			for k, v in list(d.items()):
 				if not v:
 					del d[k]
-			if self.serialize_eclasses and "_eclasses_" in values:
+		if "_eclasses_" in values:
+			if d is None:
+				d = ProtectedDict(values)
+			if self.serialize_eclasses:
 				d["_eclasses_"] = serialize_eclasses(d["_eclasses_"],
 					self.validation_chf, paths=self.store_eclass_paths)
-		elif self.serialize_eclasses and "_eclasses_" in values:
-			d = ProtectedDict(values)
-			d["_eclasses_"] = serialize_eclasses(d["_eclasses_"],
-				self.validation_chf, paths=self.store_eclass_paths)
-		else:
+			else:
+				d["_eclasses_"] = self._internal_eclasses(d["_eclasses_"],
+					self.validation_chf, self.store_eclass_paths)
+		elif d is None:
 			d = values
 		self._setitem(cpv, d)
 		if not self.autocommits:

diff --git a/pym/portage/cache/volatile.py b/pym/portage/cache/volatile.py
index a3c57f5..5516745 100644
--- a/pym/portage/cache/volatile.py
+++ b/pym/portage/cache/volatile.py
@@ -17,7 +17,7 @@ class database(template.database):
 		self._data = {}
 		self._delitem = self._data.__delitem__
 
-	def __setitem__(self, name, values):
+	def _setitem(self, name, values):
 		self._data[name] = copy.deepcopy(values)
 
 	def __getitem__(self, cpv):

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     2c1ea6faef52448d0836f070997d5d20f0b53103
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 29 23:10:13 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Oct 29 23:10:13 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2c1ea6fa

cache/template.__getitem__: filter INHERITED

Never return INHERITED, since portdbapi.aux_get() will generate it
automatically from _eclasses_, and we want to omit it in comparisons
between cache entries like those that egencache uses to avoid redundant
writes (see commit 0e120da008c9d0d41c9372c81145c6e153028a6d).

---
 pym/portage/cache/metadata.py |    1 -
 pym/portage/cache/template.py |    5 +++++
 2 files changed, 5 insertions(+), 1 deletions(-)

diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py
index 07ec20e..9d2c3a5 100644
--- a/pym/portage/cache/metadata.py
+++ b/pym/portage/cache/metadata.py
@@ -72,7 +72,6 @@ class database(flat_hash.database):
 				except KeyError as e:
 					# INHERITED contains a non-existent eclass.
 					raise cache_errors.CacheCorruption(cpv, e)
-				del d["INHERITED"]
 			else:
 				d["_eclasses_"] = {}
 		elif isinstance(d["_eclasses_"], basestring):

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index 515ba02..0af6c20 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -47,6 +47,11 @@ class database(object):
 				self.validation_chf, paths=self.store_eclass_paths)
 		elif "_eclasses_" not in d:
 			d["_eclasses_"] = {}
+		# Never return INHERITED, since portdbapi.aux_get() will
+		# generate it automatically from _eclasses_, and we want
+		# to omit it in comparisons between cache entries like
+		# those that egencache uses to avoid redundant writes.
+		d.pop("INHERITED", None)
 		mtime = d.get('_mtime_')
 		if mtime is None:
 			raise cache_errors.CacheCorruption(cpv,

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     1b367f9121b7c9cdceab644539b329c501e539c2
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed May 23 18:59:45 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed May 23 18:59:45 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1b367f91

validate_entry: handle KeyError for bug #417253

This error is triggered by corrupt cache entries.

---
 pym/portage/cache/template.py |    7 ++++++-
 1 files changed, 6 insertions(+), 1 deletions(-)

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index 0af6c20..0ab6e0a 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -194,8 +194,13 @@ class database(object):
 
 	def validate_entry(self, entry, ebuild_hash, eclass_db):
 		hash_key = '_%s_' % self.validation_chf
-		if entry[hash_key] != getattr(ebuild_hash, self.validation_chf):
+		try:
+			entry_hash = entry[hash_key]
+		except KeyError:
 			return False
+		else:
+			if entry_hash != getattr(ebuild_hash, self.validation_chf):
+				return False
 		update = eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], self.validation_chf,
 			self.store_eclass_paths)
 		if update is None:

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     cd0285b567b307ae04435aa342f40714d2ed6ac3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 24 19:06:11 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu May 24 19:06:11 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cd0285b5

reconstruct_eclasses: unicode hashes for python2

---
 pym/portage/cache/template.py |    7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index 0ab6e0a..cf1e8ae 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -1,4 +1,4 @@
-# Copyright: 2005 Gentoo Foundation
+# Copyright: 2005-2012 Gentoo Foundation
 # Author(s): Brian Harring (ferringb@gentoo.org)
 # License: GPL2
 
@@ -10,8 +10,11 @@ import warnings
 import operator
 
 if sys.hexversion >= 0x3000000:
+	_unicode = str
 	basestring = str
 	long = int
+else:
+	_unicode = unicode
 
 class database(object):
 	# this is for metadata/cache transfer.
@@ -273,7 +276,7 @@ def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True):
 		# occasionally this occurs in the fs backends.  they suck.
 		return {}
 
-	converter = str
+	converter = _unicode
 	if chf_type == 'mtime':
 		converter = long
 

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     2b47483b406a40974f7e2ce4df4163dd08e657d2
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 10 08:24:59 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun 10 08:24:59 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2b47483b

cache/sqlite.py: dynamically add columns to table

---
 pym/portage/cache/sqlite.py |   42 ++++++++++++++++++++++++++++++++++++++++--
 1 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index fcc62ff..7ca5a03 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -1,6 +1,7 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import re
 import sys
 from portage.cache import fs_template
 from portage.cache import cache_errors
@@ -117,7 +118,13 @@ class database(fs_template.FsBased):
 		for k, v in self._db_table.items():
 			if self._db_table_exists(v["table_name"]):
 				create_statement = self._db_table_get_create(v["table_name"])
-				if create_statement != v["create"]:
+				table_ok, missing_keys = self._db_validate_create_statement(create_statement)
+				if table_ok:
+					if missing_keys:
+						for k in sorted(missing_keys):
+							cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" %
+								(self._db_table["packages"]["table_name"], k))
+				else:
 					writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"])
 					cursor.execute("DROP TABLE %s" % v["table_name"])
 					cursor.execute(v["create"])
@@ -138,6 +145,37 @@ class database(fs_template.FsBased):
 			self._db_escape_string(table_name))
 		return cursor.fetchall()[0][0]
 
+	def _db_validate_create_statement(self, statement):
+		missing_keys = None
+		if statement == self._db_table["packages"]["create"]:
+			return True, missing_keys
+
+		m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' %
+			(self._db_table["packages"]["table_name"],
+			self._db_table["packages"]["package_id"]),
+			statement)
+		if m is None:
+			return False, missing_keys
+
+		unique_constraints = set([self._db_table["packages"]["package_key"]])
+		missing_keys = set(self._allowed_keys)
+		unique_re = re.compile(r'^UNIQUE\s*\(\s*(\w*)\s*\)\s*$')
+		column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$')
+		for x in m.group(1).split(","):
+			m = unique_re.match(x)
+			if m is not None:
+				unique_constraints.discard(m.group(1))
+				continue
+			m = column_re.match(x)
+			if m is not None:
+				missing_keys.discard(m.group(1))
+				continue
+
+		if unique_constraints:
+			return False, missing_keys
+
+		return True, missing_keys
+
 	def _db_init_cache_size(self, cache_bytes):
 		cursor = self._db_cursor
 		cursor.execute("PRAGMA page_size")

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     3feff3032fefc43b35bc541a0e49472d8114de93
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 10 08:24:59 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun 10 08:28:19 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3feff303

cache/sqlite.py: dynamically add columns to table

---
 pym/portage/cache/sqlite.py |   42 ++++++++++++++++++++++++++++++++++++++++--
 1 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index fcc62ff..54f54c6 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -1,6 +1,7 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import re
 import sys
 from portage.cache import fs_template
 from portage.cache import cache_errors
@@ -117,7 +118,13 @@ class database(fs_template.FsBased):
 		for k, v in self._db_table.items():
 			if self._db_table_exists(v["table_name"]):
 				create_statement = self._db_table_get_create(v["table_name"])
-				if create_statement != v["create"]:
+				table_ok, missing_keys = self._db_validate_create_statement(create_statement)
+				if table_ok:
+					if missing_keys:
+						for k in sorted(missing_keys):
+							cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" %
+								(self._db_table["packages"]["table_name"], k))
+				else:
 					writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"])
 					cursor.execute("DROP TABLE %s" % v["table_name"])
 					cursor.execute(v["create"])
@@ -138,6 +145,37 @@ class database(fs_template.FsBased):
 			self._db_escape_string(table_name))
 		return cursor.fetchall()[0][0]
 
+	def _db_validate_create_statement(self, statement):
+		missing_keys = None
+		if statement == self._db_table["packages"]["create"]:
+			return True, missing_keys
+
+		m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' %
+			(self._db_table["packages"]["table_name"],
+			self._db_table["packages"]["package_id"]),
+			statement)
+		if m is None:
+			return False, missing_keys
+
+		unique_constraints = set([self._db_table["packages"]["package_key"]])
+		missing_keys = set(self._allowed_keys)
+		unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$')
+		column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$')
+		for x in m.group(1).split(","):
+			m = unique_re.match(x)
+			if m is not None:
+				unique_constraints.discard(m.group(1))
+				continue
+			m = column_re.match(x)
+			if m is not None:
+				missing_keys.discard(m.group(1))
+				continue
+
+		if unique_constraints:
+			return False, missing_keys
+
+		return True, missing_keys
+
 	def _db_init_cache_size(self, cache_bytes):
 		cursor = self._db_cursor
 		cursor.execute("PRAGMA page_size")

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     d6584f5ef8eb65bf0c818a0ab2f9b008663d758e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 10 08:24:59 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun 10 08:34:36 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d6584f5e

cache/sqlite.py: dynamically add columns to table

---
 pym/portage/cache/sqlite.py |   42 ++++++++++++++++++++++++++++++++++++++++--
 1 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index fcc62ff..a6a3e06 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -1,6 +1,7 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import re
 import sys
 from portage.cache import fs_template
 from portage.cache import cache_errors
@@ -117,7 +118,13 @@ class database(fs_template.FsBased):
 		for k, v in self._db_table.items():
 			if self._db_table_exists(v["table_name"]):
 				create_statement = self._db_table_get_create(v["table_name"])
-				if create_statement != v["create"]:
+				table_ok, missing_keys = self._db_validate_create_statement(create_statement)
+				if table_ok:
+					if missing_keys:
+						for k in sorted(missing_keys):
+							cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" %
+								(self._db_table["packages"]["table_name"], k))
+				else:
 					writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"])
 					cursor.execute("DROP TABLE %s" % v["table_name"])
 					cursor.execute(v["create"])
@@ -138,6 +145,37 @@ class database(fs_template.FsBased):
 			self._db_escape_string(table_name))
 		return cursor.fetchall()[0][0]
 
+	def _db_validate_create_statement(self, statement):
+		missing_keys = None
+		if statement == self._db_table["packages"]["create"]:
+			return True, missing_keys
+
+		m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' %
+			(self._db_table["packages"]["table_name"],
+			self._db_table["packages"]["package_id"]),
+			statement)
+		if m is None:
+			return False, missing_keys
+
+		unique_constraints = set([self._db_table["packages"]["package_key"]])
+		missing_keys = set(self._allowed_keys)
+		unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$')
+		column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$')
+		for x in m.group(1).split(","):
+			m = column_re.match(x)
+			if m is not None:
+				missing_keys.discard(m.group(1))
+				continue
+			m = unique_re.match(x)
+			if m is not None:
+				unique_constraints.discard(m.group(1))
+				continue
+
+		if unique_constraints:
+			return False, missing_keys
+
+		return True, missing_keys
+
 	def _db_init_cache_size(self, cache_bytes):
 		cursor = self._db_cursor
 		cursor.execute("PRAGMA page_size")

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     8279b1e6626e634b4e8f5175180cf17cf09d5b8a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 18 19:01:59 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 18 19:01:59 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8279b1e6

cache/sqlite.py: fix getitem order assumption

The order assumption was incorrect when new columns were added, since
they could be added in any order.

---
 pym/portage/cache/sqlite.py |   16 +++++-----------
 1 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index a6a3e06..1038bb2 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -29,6 +29,7 @@ class database(fs_template.FsBased):
 		self._allowed_keys = ["_mtime_", "_eclasses_"]
 		self._allowed_keys.extend(self._known_keys)
 		self._allowed_keys.sort()
+		self._allowed_keys_set = frozenset(self._allowed_keys)
 		self.location = os.path.join(self.location, 
 			self.label.lstrip(os.path.sep).rstrip(os.path.sep))
 
@@ -93,9 +94,6 @@ class database(fs_template.FsBased):
 		self._db_table["packages"]["table_name"] = mytable
 		self._db_table["packages"]["package_id"] = "internal_db_package_id"
 		self._db_table["packages"]["package_key"] = "portage_package_key"
-		self._db_table["packages"]["internal_columns"] = \
-			[self._db_table["packages"]["package_id"],
-			self._db_table["packages"]["package_key"]]
 		create_statement = []
 		create_statement.append("CREATE TABLE")
 		create_statement.append(mytable)
@@ -110,9 +108,6 @@ class database(fs_template.FsBased):
 		create_statement.append(")")
 		
 		self._db_table["packages"]["create"] = " ".join(create_statement)
-		self._db_table["packages"]["columns"] = \
-			self._db_table["packages"]["internal_columns"] + \
-			self._allowed_keys
 
 		cursor = self._db_cursor
 		for k, v in self._db_table.items():
@@ -212,11 +207,10 @@ class database(fs_template.FsBased):
 		else:
 			raise cache_errors.CacheCorruption(cpv, "key is not unique")
 		d = {}
-		internal_columns = self._db_table["packages"]["internal_columns"]
-		column_index = -1
-		for k in self._db_table["packages"]["columns"]:
-			column_index +=1
-			if k not in internal_columns:
+		allowed_keys_set = self._allowed_keys_set
+		for column_index, column_info in enumerate(cursor.description):
+			k = column_info[0]
+			if k in allowed_keys_set:
 				d[k] = result[0][column_index]
 
 		return d

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     7207ceb064617998193b4db5eb18563d2deebbef
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 25 01:41:50 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 25 01:41:50 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7207ceb0

cache/metadata.py: add HDEPEND

---
 pym/portage/cache/flat_list.py |    3 ++-
 pym/portage/cache/metadata.py  |    3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/flat_list.py b/pym/portage/cache/flat_list.py
index 7288307..c88a004 100644
--- a/pym/portage/cache/flat_list.py
+++ b/pym/portage/cache/flat_list.py
@@ -28,7 +28,8 @@ class database(fs_template.FsBased):
 	auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI',
 		'RESTRICT',  'HOMEPAGE',  'LICENSE', 'DESCRIPTION',
 		'KEYWORDS',  'IUSE', 'REQUIRED_USE',
-		'PDEPEND',   'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES')
+		'PDEPEND',   'PROVIDE', 'EAPI', 'PROPERTIES',
+		'DEFINED_PHASES', 'HDEPEND')
 
 	def __init__(self, *args, **config):
 		super(database,self).__init__(*args, **config)

diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py
index 9d2c3a5..6612e73 100644
--- a/pym/portage/cache/metadata.py
+++ b/pym/portage/cache/metadata.py
@@ -28,7 +28,8 @@ class database(flat_hash.database):
 	auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI',
 		'RESTRICT',  'HOMEPAGE',  'LICENSE', 'DESCRIPTION',
 		'KEYWORDS',  'INHERITED', 'IUSE', 'REQUIRED_USE',
-		'PDEPEND',   'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES')
+		'PDEPEND',   'PROVIDE', 'EAPI', 'PROPERTIES',
+		'DEFINED_PHASES', 'HDEPEND')
 
 	autocommits = True
 	serialize_eclasses = False

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     f6aeb08c6aedeceee44b2dc65be60a5fd0474981
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 25 01:54:32 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 25 01:54:32 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f6aeb08c

cache/sqlite.py: translate None to empty string

---
 pym/portage/cache/sqlite.py |    9 +++++++--
 1 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 1038bb2..9c1b3eb 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -21,7 +21,7 @@ class database(fs_template.FsBased):
 	# to calculate the number of pages requested, according to the following
 	# equation: cache_bytes = page_bytes * page_count
 	cache_bytes = 1024 * 1024 * 10
-	_db_table = None
+	_EMPTY_STRING = _unicode_decode("")
 
 	def __init__(self, *args, **config):
 		super(database, self).__init__(*args, **config)
@@ -206,12 +206,17 @@ class database(fs_template.FsBased):
 			raise KeyError(cpv)
 		else:
 			raise cache_errors.CacheCorruption(cpv, "key is not unique")
+		result = result[0]
 		d = {}
 		allowed_keys_set = self._allowed_keys_set
 		for column_index, column_info in enumerate(cursor.description):
 			k = column_info[0]
 			if k in allowed_keys_set:
-				d[k] = result[0][column_index]
+				v = result[column_index]
+				if v is None:
+					# This happens after a new empty column has been added.
+					v = self._EMPTY_STRING
+				d[k] = v
 
 		return d
 

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     558ada1185e9d44627298aeff1b2a699c509897d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 25 03:44:28 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 25 03:44:28 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=558ada11

Remove obsolete cache/flat_list.py

This module has been obsolete and useless for many years, which is
especially obvious since it was missing INHERITED from its
auxdbkey_order and it did not write any eclass metadata.

---
 pym/portage/cache/flat_list.py |  135 ----------------------------------------
 1 files changed, 0 insertions(+), 135 deletions(-)

diff --git a/pym/portage/cache/flat_list.py b/pym/portage/cache/flat_list.py
deleted file mode 100644
index c88a004..0000000
--- a/pym/portage/cache/flat_list.py
+++ /dev/null
@@ -1,135 +0,0 @@
-# Copyright 2005-2011 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-from portage.cache import fs_template
-from portage.cache import cache_errors
-from portage import os
-from portage import _encodings
-from portage import _unicode_decode
-from portage import _unicode_encode
-import errno
-import io
-import stat
-import sys
-
-if sys.hexversion >= 0x3000000:
-	long = int
-
-# Coerce to unicode, in order to prevent TypeError when writing
-# raw bytes to TextIOWrapper with python2.
-_setitem_fmt = _unicode_decode("%s\n")
-
-# store the current key order *here*.
-class database(fs_template.FsBased):
-
-	autocommits = True
-
-	# do not screw with this ordering. _eclasses_ needs to be last
-	auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI',
-		'RESTRICT',  'HOMEPAGE',  'LICENSE', 'DESCRIPTION',
-		'KEYWORDS',  'IUSE', 'REQUIRED_USE',
-		'PDEPEND',   'PROVIDE', 'EAPI', 'PROPERTIES',
-		'DEFINED_PHASES', 'HDEPEND')
-
-	def __init__(self, *args, **config):
-		super(database,self).__init__(*args, **config)
-		self.location = os.path.join(self.location, 
-			self.label.lstrip(os.path.sep).rstrip(os.path.sep))
-
-		if len(self._known_keys) > len(self.auxdbkey_order) + 2:
-			raise Exception("less ordered keys then auxdbkeys")
-		if not os.path.exists(self.location):
-			self._ensure_dirs()
-
-
-	def _getitem(self, cpv):
-		d = {}
-		try:
-			myf = io.open(_unicode_encode(os.path.join(self.location, cpv),
-				encoding=_encodings['fs'], errors='strict'),
-				mode='r', encoding=_encodings['repo.content'],
-				errors='replace')
-			for k,v in zip(self.auxdbkey_order, myf):
-				d[k] = v.rstrip("\n")
-		except (OSError, IOError) as e:
-			if errno.ENOENT == e.errno:
-				raise KeyError(cpv)
-			raise cache_errors.CacheCorruption(cpv, e)
-
-		try:
-			d["_mtime_"] = os.fstat(myf.fileno())[stat.ST_MTIME]
-		except OSError as e:	
-			myf.close()
-			raise cache_errors.CacheCorruption(cpv, e)
-		myf.close()
-		return d
-
-
-	def _setitem(self, cpv, values):
-		s = cpv.rfind("/")
-		fp=os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:]))
-		try:
-			myf = io.open(_unicode_encode(fp,
-				encoding=_encodings['fs'], errors='strict'),
-				mode='w', encoding=_encodings['repo.content'],
-				errors='backslashreplace')
-		except (OSError, IOError) as e:
-			if errno.ENOENT == e.errno:
-				try:
-					self._ensure_dirs(cpv)
-					myf = io.open(_unicode_encode(fp,
-						encoding=_encodings['fs'], errors='strict'),
-						mode='w', encoding=_encodings['repo.content'],
-						errors='backslashreplace')
-				except (OSError, IOError) as e:
-					raise cache_errors.CacheCorruption(cpv, e)
-			else:
-				raise cache_errors.CacheCorruption(cpv, e)
-		
-
-		for x in self.auxdbkey_order:
-			myf.write(_setitem_fmt % (values.get(x, ""),))
-
-		myf.close()
-		self._ensure_access(fp, mtime=values["_mtime_"])
-		#update written.  now we move it.
-		new_fp = os.path.join(self.location,cpv)
-		try:
-			os.rename(fp, new_fp)
-		except (OSError, IOError) as e:
-			os.remove(fp)
-			raise cache_errors.CacheCorruption(cpv, e)
-
-
-	def _delitem(self, cpv):
-		try:
-			os.remove(os.path.join(self.location,cpv))
-		except OSError as e:
-			if errno.ENOENT == e.errno:
-				raise KeyError(cpv)
-			else:
-				raise cache_errors.CacheCorruption(cpv, e)
-
-
-	def __contains__(self, cpv):
-		return os.path.exists(os.path.join(self.location, cpv))
-
-
-	def __iter__(self):
-		"""generator for walking the dir struct"""
-		dirs = [self.location]
-		len_base = len(self.location)
-		while len(dirs):
-			for l in os.listdir(dirs[0]):
-				if l.endswith(".cpickle"):
-					continue
-				p = os.path.join(dirs[0],l)
-				st = os.lstat(p)
-				if stat.S_ISDIR(st.st_mode):
-					dirs.append(p)
-					continue
-				yield p[len_base+1:]
-			dirs.pop(0)
-
-
-	def commit(self):	pass

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     6571d8b160e2e5d10ef76fcc5fba288d3e494d3e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  2 20:22:57 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct  2 20:22:57 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6571d8b1

OrderedDict: fix setitem bug #436974

This fixes some strange behavior triggered during fetch, which is only
observable with Python 2.6 since it doesn't have
collections.OrderedDict.

---
 pym/portage/cache/mappings.py |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py
index bc8ce9a..cd39a6e 100644
--- a/pym/portage/cache/mappings.py
+++ b/pym/portage/cache/mappings.py
@@ -199,10 +199,10 @@ class OrderedDict(UserDict):
 		return iter(self._order)
 
 	def __setitem__(self, key, item):
-		if key in self:
-			self._order.remove(key)
+		new_key = key not in self
 		UserDict.__setitem__(self, key, item)
-		self._order.append(key)
+		if new_key:
+			self._order.append(key)
 
 	def __delitem__(self, key):
 		UserDict.__delitem__(self, key)

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     2694610b1fcd2818e203936bf663c9c44f4e6ca7
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 21 04:38:32 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Nov 21 04:38:32 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2694610b

flat_hash.__iter__: validate keys

---
 pym/portage/cache/flat_hash.py |   10 +++++++---
 1 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index 2eae9f6..b71e118 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -13,6 +13,8 @@ from portage import os
 from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
+from portage.exception import InvalidData
+from portage.versions import _pkg_str
 
 if sys.hexversion >= 0x3000000:
 	long = int
@@ -135,8 +137,6 @@ class database(fs_template.FsBased):
 				del e
 				continue
 			for l in dir_list:
-				if l.endswith(".cpickle"):
-					continue
 				p = os.path.join(dir_path, l)
 				try:
 					st = os.lstat(p)
@@ -151,7 +151,11 @@ class database(fs_template.FsBased):
 					if depth < 1:
 						dirs.append((depth+1, p))
 					continue
-				yield p[len_base+1:]
+
+				try:
+					yield _pkg_str(p[len_base+1:])
+				except InvalidData:
+					continue
 
 
 class md5_database(database):

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     b91eff234f050ada2a7027fd780f138628f3c254
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 18 15:31:47 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jan 18 15:31:47 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b91eff23

ebuild_xattr.py: lowercase utf-8 comment

---
 pym/portage/cache/ebuild_xattr.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py
index 0086e40..db6e177 100644
--- a/pym/portage/cache/ebuild_xattr.py
+++ b/pym/portage/cache/ebuild_xattr.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF8 -*-
+# -*- coding: utf-8 -*-
 # Copyright: 2009-2011 Gentoo Foundation
 # Author(s): Petteri Räty (betelgeuse@gentoo.org)
 # License: GPL2

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     729a0d126771fc5aaae38b1221dad1aebe15345e
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 18 16:37:18 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jan 18 16:37:18 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=729a0d12

cache/sqlite.py: unicode_literals

---
 pym/portage/cache/sqlite.py |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 9c1b3eb..8a9f747 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -1,6 +1,8 @@
-# Copyright 1999-2012 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+from __future__ import unicode_literals
+
 import re
 import sys
 from portage.cache import fs_template
@@ -21,7 +23,6 @@ class database(fs_template.FsBased):
 	# to calculate the number of pages requested, according to the following
 	# equation: cache_bytes = page_bytes * page_count
 	cache_bytes = 1024 * 1024 * 10
-	_EMPTY_STRING = _unicode_decode("")
 
 	def __init__(self, *args, **config):
 		super(database, self).__init__(*args, **config)
@@ -215,7 +216,7 @@ class database(fs_template.FsBased):
 				v = result[column_index]
 				if v is None:
 					# This happens after a new empty column has been added.
-					v = self._EMPTY_STRING
+					v = ""
 				d[k] = v
 
 		return d

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     417692f361be32de353bc4d698b43d7a8121ec6d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 18 17:12:25 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jan 18 17:12:25 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=417692f3

cache/flat_hash.py: unicode_literals

---
 pym/portage/cache/flat_hash.py |   14 +++++++-------
 1 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index b1c9431..08dcbe8 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -1,7 +1,9 @@
-# Copyright: 2005-2012 Gentoo Foundation
+# Copyright: 2005-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 # Author(s): Brian Harring (ferringb@gentoo.org)
 
+from __future__ import unicode_literals
+
 from portage.cache import fs_template
 from portage.cache import cache_errors
 import errno
@@ -11,7 +13,6 @@ import sys
 import os as _os
 from portage import os
 from portage import _encodings
-from portage import _unicode_decode
 from portage import _unicode_encode
 from portage.exception import InvalidData
 from portage.versions import _pkg_str
@@ -19,10 +20,6 @@ from portage.versions import _pkg_str
 if sys.hexversion >= 0x3000000:
 	long = int
 
-# Coerce to unicode, in order to prevent TypeError when writing
-# raw bytes to TextIOWrapper with python2.
-_setitem_fmt = _unicode_decode("%s=%s\n")
-
 class database(fs_template.FsBased):
 
 	autocommits = True
@@ -93,7 +90,10 @@ class database(fs_template.FsBased):
 				v = values.get(k)
 				if not v:
 					continue
-				myf.write(_setitem_fmt % (k, v))
+				# NOTE: This format string requires unicode_literals, so that
+				# k and v are coerced to unicode, in order to prevent TypeError
+				# when writing raw bytes to TextIOWrapper with Python 2.
+				myf.write("%s=%s\n" % (k, v))
 		finally:
 			myf.close()
 		self._ensure_access(fp)

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     b18a990e60ca08010cfde900c1a2807549b5f134
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 26 07:56:55 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul 26 07:56:55 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b18a990e

Remove pysqlite2 support.

---
 pym/portage/cache/sqlite.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 8a9f747..ad8648c 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -51,10 +51,7 @@ class database(fs_template.FsBased):
 		try:
 			import sqlite3 as db_module
 		except ImportError:
-			try:
-				from pysqlite2 import dbapi2 as db_module
-			except ImportError as e:
-				raise cache_errors.InitializationError(self.__class__, e)
+			raise cache_errors.InitializationError(self.__class__, e)
 
 		self._db_module = db_module
 		self._db_error = db_module.Error

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Arfrever Frehtes Taifersar Arahesis]

commit:     7eaba631529148594b3136b9dcb55ae00c67e1be
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Fri Jul 26 08:22:47 2013 +0000
Commit:     Arfrever Frehtes Taifersar Arahesis <arfrever.fta <AT> gmail <DOT> com>
CommitDate: Fri Jul 26 08:22:47 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7eaba631

Fix NameError.

---
 pym/portage/cache/sqlite.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index ad8648c..40db070 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -40,8 +40,8 @@ class database(fs_template.FsBased):
 		config.setdefault("autocommit", self.autocommits)
 		config.setdefault("cache_bytes", self.cache_bytes)
 		config.setdefault("synchronous", self.synchronous)
-		# Timeout for throwing a "database is locked" exception (pysqlite
-		# default is 5.0 seconds).
+		# Set longer timeout for throwing a "database is locked" exception.
+		# Default timeout in sqlite3 module is 5.0 seconds.
 		config.setdefault("timeout", 15)
 		self._db_init_connection(config)
 		self._db_init_structures()
@@ -50,7 +50,7 @@ class database(fs_template.FsBased):
 		# sqlite3 is optional with >=python-2.5
 		try:
 			import sqlite3 as db_module
-		except ImportError:
+		except ImportError as e:
 			raise cache_errors.InitializationError(self.__class__, e)
 
 		self._db_module = db_module
@@ -62,7 +62,6 @@ class database(fs_template.FsBased):
 			# Avoid potential UnicodeEncodeError in python-2.x by
 			# only calling str() when it's absolutely necessary.
 			s = str(s)
-		# This is equivalent to the _quote function from pysqlite 1.1.
 		return "'%s'" % s.replace("'", "''")
 
 	def _db_init_connection(self, config):

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     9f6967dbc38ea55c0de5fbf5ad663ca676c54743
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 13 19:35:48 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Nov 14 17:30:31 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9f6967db

fs_template._ensure_dirs: handle EEXIST (529120)

There was a race inside fs_template._ensure_dirs which could cause it to
raise EEXIST if a concurrent process created the directory after
os.path.exists returned False. Fix it by using the util.ensure_dirs
function, which already handles EEXIST.

X-Gentoo-Bug: 529120
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=529120
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

---
 pym/portage/cache/fs_template.py | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/pym/portage/cache/fs_template.py b/pym/portage/cache/fs_template.py
index de4fe4b..fa44abc 100644
--- a/pym/portage/cache/fs_template.py
+++ b/pym/portage/cache/fs_template.py
@@ -10,7 +10,7 @@ from portage import os
 from portage.proxy.lazyimport import lazyimport
 lazyimport(globals(),
 	'portage.exception:PortageException',
-	'portage.util:apply_permissions',
+	'portage.util:apply_permissions,ensure_dirs',
 )
 del lazyimport
 
@@ -61,20 +61,15 @@ class FsBased(template.database):
 
 		for dir in path.lstrip(os.path.sep).rstrip(os.path.sep).split(os.path.sep):
 			base = os.path.join(base,dir)
-			if not os.path.exists(base):
-				if self._perms != -1:
-					um = os.umask(0)
-				try:
-					perms = self._perms
-					if perms == -1:
-						perms = 0
-					perms |= 0o755
-					os.mkdir(base, perms)
-					if self._gid != -1:
-						os.chown(base, -1, self._gid)
-				finally:
-					if self._perms != -1:
-						os.umask(um)
+			if ensure_dirs(base):
+				# We only call apply_permissions if ensure_dirs created
+				# a new directory, so as not to interfere with
+				# permissions of existing directories.
+				mode = self._perms
+				if mode == -1:
+					mode = 0
+				mode |= 0o755
+				apply_permissions(base, mode=mode, gid=self._gid)
 
 	def _prune_empty_dirs(self):
 		all_dirs = []

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     7e663905063ca92ad82127a797356e35bfc5bdfa
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 25 05:56:18 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Dec 29 16:40:04 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7e663905

anydbm: enable md5 validation (bug 568934)

Add forward-compatibility for cache entries containing md5 digests
instead of mtimes for validation.

X-Gentoo-Bug: 568934
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=568934
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

 pym/portage/cache/anydbm.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pym/portage/cache/anydbm.py b/pym/portage/cache/anydbm.py
index 1d56b14..80d24e5 100644
--- a/pym/portage/cache/anydbm.py
+++ b/pym/portage/cache/anydbm.py
@@ -36,6 +36,9 @@ from portage.cache import cache_errors
 
 class database(fs_template.FsBased):
 
+	validation_chf = 'mtime'
+	chf_types = ('mtime', 'md5')
+
 	autocommits = True
 	cleanse_keys = True
 	serialize_eclasses = False

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     6c09ab7d6c6b2ffcf7e2641874167b0bff12ff91
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 24 11:08:54 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Dec 29 16:39:57 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c09ab7d

template.database.__getitem__: allow missing mtime (bug 568934)

Fix __getitem__ to allow missing mtime when a suitable alternative
(such as md5) is available.

Fixes: 669d11bd8af5 ("flat_hash: enable md5 validation for /var/cache/edb/dep (bug 568934)")
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

 pym/portage/cache/template.py | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index a942b36..a7c6de0 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -46,12 +46,13 @@ class database(object):
 			self.commit()
 			self.updates = 0
 		d=self._getitem(cpv)
-		if self.serialize_eclasses and "_eclasses_" in d:
-			try:
-				chf_types = self.chf_types
-			except AttributeError:
-				chf_types = (self.validation_chf,)
 
+		try:
+			chf_types = self.chf_types
+		except AttributeError:
+			chf_types = (self.validation_chf,)
+
+		if self.serialize_eclasses and "_eclasses_" in d:
 			for chf_type in chf_types:
 				try:
 					d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"],
@@ -69,16 +70,23 @@ class database(object):
 		# to omit it in comparisons between cache entries like
 		# those that egencache uses to avoid redundant writes.
 		d.pop("INHERITED", None)
+
+		mtime_required = not any(d.get('_%s_' % x)
+			for x in chf_types if x != 'mtime')
+
 		mtime = d.get('_mtime_')
-		if mtime is None:
-			raise cache_errors.CacheCorruption(cpv,
-				'_mtime_ field is missing')
-		try:
-			mtime = long(mtime)
-		except ValueError:
-			raise cache_errors.CacheCorruption(cpv,
-				'_mtime_ conversion to long failed: %s' % (mtime,))
-		d['_mtime_'] = mtime
+		if not mtime:
+			if mtime_required:
+				raise cache_errors.CacheCorruption(cpv,
+					'_mtime_ field is missing')
+			d.pop('_mtime_', None)
+		else:
+			try:
+				mtime = long(mtime)
+			except ValueError:
+				raise cache_errors.CacheCorruption(cpv,
+					'_mtime_ conversion to long failed: %s' % (mtime,))
+			d['_mtime_'] = mtime
 		return d
 
 	def _getitem(self, cpv):

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     7bef8d93aa79ad7a4001e30c74f7aa267ac95771
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 24 11:19:32 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Dec 29 16:40:02 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7bef8d93

sqlite: enable md5 validation (bug 568934)

Add forward-compatibility for cache entries containing md5 digests
instead of mtimes for validation.

X-Gentoo-Bug: 568934
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=568934
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

 pym/portage/cache/sqlite.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 310ac94..32e4076 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -18,6 +18,9 @@ if sys.hexversion >= 0x3000000:
 
 class database(fs_template.FsBased):
 
+	validation_chf = 'mtime'
+	chf_types = ('mtime', 'md5')
+
 	autocommits = False
 	synchronous = False
 	# cache_bytes is used together with page_size (set at sqlite build time)
@@ -28,10 +31,12 @@ class database(fs_template.FsBased):
 	def __init__(self, *args, **config):
 		super(database, self).__init__(*args, **config)
 		self._import_sqlite()
-		self._allowed_keys = ["_mtime_", "_eclasses_"]
+		self._allowed_keys = ["_eclasses_"]
 		self._allowed_keys.extend(self._known_keys)
-		self._allowed_keys.sort()
+		self._allowed_keys.extend('_%s_' % k for k in self.chf_types)
 		self._allowed_keys_set = frozenset(self._allowed_keys)
+		self._allowed_keys = sorted(self._allowed_keys_set)
+
 		self.location = os.path.join(self.location, 
 			self.label.lstrip(os.path.sep).rstrip(os.path.sep))
 

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     9abbda7d054761ae6c333d3e6d420632b9658b6d
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 10 06:11:41 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jul 13 11:29:34 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9abbda7d

portage.cache: write md5 instead of mtime (bug 568934)

Change cache modules to write md5 in cache entries, instead of mtime.
Since portage-2.2.27, the relevant cache modules have had the ability
to read cache entries containing either md5 or mtime, therefore this
change is backward-compatible with portage-2.2.27 and later.

Also fix the reconstruct_eclasses function to raise CacheCorruption
when the specified chf_type is md5 and the cache entry contains mtime
data, and optimize __getitem__ to skip reconstruct_eclasses calls when
the entry appears to have a different chf_type.

X-Gentoo-Bug: 568934
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=568934
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

 pym/portage/cache/anydbm.py    |  4 ++--
 pym/portage/cache/flat_hash.py |  4 ++--
 pym/portage/cache/sqlite.py    |  4 ++--
 pym/portage/cache/template.py  | 36 ++++++++++++++++++++++++++++++++----
 4 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/pym/portage/cache/anydbm.py b/pym/portage/cache/anydbm.py
index 80d24e5..88d85b0 100644
--- a/pym/portage/cache/anydbm.py
+++ b/pym/portage/cache/anydbm.py
@@ -36,8 +36,8 @@ from portage.cache import cache_errors
 
 class database(fs_template.FsBased):
 
-	validation_chf = 'mtime'
-	chf_types = ('mtime', 'md5')
+	validation_chf = 'md5'
+	chf_types = ('md5', 'mtime')
 
 	autocommits = True
 	cleanse_keys = True

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index cca0f10..3a899c0 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -163,5 +163,5 @@ class md5_database(database):
 
 
 class mtime_md5_database(database):
-	validation_chf = 'mtime'
-	chf_types = ('mtime', 'md5')
+	validation_chf = 'md5'
+	chf_types = ('md5', 'mtime')

diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py
index 32e4076..69150f6 100644
--- a/pym/portage/cache/sqlite.py
+++ b/pym/portage/cache/sqlite.py
@@ -18,8 +18,8 @@ if sys.hexversion >= 0x3000000:
 
 class database(fs_template.FsBased):
 
-	validation_chf = 'mtime'
-	chf_types = ('mtime', 'md5')
+	validation_chf = 'md5'
+	chf_types = ('md5', 'mtime')
 
 	autocommits = False
 	synchronous = False

diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index a7c6de0..8662d85 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -54,6 +54,15 @@ class database(object):
 
 		if self.serialize_eclasses and "_eclasses_" in d:
 			for chf_type in chf_types:
+				if '_%s_' % chf_type not in d:
+					# Skip the reconstruct_eclasses call, since it's
+					# a waste of time if it contains a different chf_type
+					# than the current one. In the past, it was possible
+					# for reconstruct_eclasses called with chf_type='md5'
+					# to "successfully" return invalid data here, because
+					# it was unable to distinguish between md5 data and
+					# mtime data.
+					continue
 				try:
 					d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"],
 						chf_type, paths=self.store_eclass_paths)
@@ -62,6 +71,9 @@ class database(object):
 						raise
 				else:
 					break
+			else:
+				raise cache_errors.CacheCorruption(cpv,
+					'entry does not contain a recognized chf_type')
 
 		elif "_eclasses_" not in d:
 			d["_eclasses_"] = {}
@@ -310,6 +322,23 @@ def serialize_eclasses(eclass_dict, chf_type='mtime', paths=True):
 		for k, v in sorted(eclass_dict.items(), key=_keysorter))
 
 
+def _md5_deserializer(md5):
+	"""
+	Without this validation, it's possible for reconstruct_eclasses to
+	mistakenly interpret mtime data as md5 data, and return an invalid
+	data structure containing strings where ints are expected.
+	"""
+	if len(md5) != 32:
+		raise ValueError('expected 32 hex digits')
+	return md5
+
+
+_chf_deserializers = {
+	'md5': _md5_deserializer,
+	'mtime': long,
+}
+
+
 def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True):
 	"""returns a dict when handed a string generated by serialize_eclasses"""
 	eclasses = eclass_string.rstrip().lstrip().split("\t")
@@ -317,9 +346,7 @@ def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True):
 		# occasionally this occurs in the fs backends.  they suck.
 		return {}
 
-	converter = _unicode
-	if chf_type == 'mtime':
-		converter = long
+	converter = _chf_deserializers.get(chf_type, lambda x: x)
 
 	if paths:
 		if len(eclasses) % 3 != 0:
@@ -340,6 +367,7 @@ def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True):
 		raise cache_errors.CacheCorruption(cpv,
 			"_eclasses_ was of invalid len %i" % len(eclasses))
 	except ValueError:
-		raise cache_errors.CacheCorruption(cpv, "_eclasses_ mtime conversion to long failed")
+		raise cache_errors.CacheCorruption(cpv,
+			"_eclasses_ not valid for chf_type {}".format(chf_type))
 	del eclasses
 	return d

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     5652bc88514bdb36b36b544f7fc7e623cf25caae
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 24 22:44:31 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul 24 23:10:01 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5652bc88

flat_hash: use mkstemp in _setitem

Fix the _setitem method to use mkstemp in order to avoid a
race condition when multiple pid namespaces share the same
cache directory.

Reported-by: Mike Frysinger <vapier <AT> chromium.org>
X-Chromium-Bug: 477727
X-Chromium-Bug-url: https://bugs.chromium.org/p/chromium/issues/detail?id=477727

 pym/portage/cache/flat_hash.py | 49 +++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index 3a899c0..7978324 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2014 Gentoo Foundation
+# Copyright 2005-2016 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 # Author(s): Brian Harring (ferringb@gentoo.org)
 
@@ -10,6 +10,7 @@ import errno
 import io
 import stat
 import sys
+import tempfile
 import os as _os
 from portage import os
 from portage import _encodings
@@ -66,27 +67,14 @@ class database(fs_template.FsBased):
 			raise cache_errors.CacheCorruption(cpv, e)
 
 	def _setitem(self, cpv, values):
-		s = cpv.rfind("/")
-		fp = os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:]))
 		try:
-			myf = io.open(_unicode_encode(fp,
-				encoding=_encodings['fs'], errors='strict'),
-				mode='w', encoding=_encodings['repo.content'],
-				errors='backslashreplace')
-		except (IOError, OSError) as e:
-			if errno.ENOENT == e.errno:
-				try:
-					self._ensure_dirs(cpv)
-					myf = io.open(_unicode_encode(fp,
-						encoding=_encodings['fs'], errors='strict'),
-						mode='w', encoding=_encodings['repo.content'],
-						errors='backslashreplace')
-				except (OSError, IOError) as e:
-					raise cache_errors.CacheCorruption(cpv, e)
-			else:
-				raise cache_errors.CacheCorruption(cpv, e)
+			fd, fp = tempfile.mkstemp(dir=self.location)
+		except EnvironmentError as e:
+			raise cache_errors.CacheCorruption(cpv, e)
 
-		try:
+		with io.open(fd, mode='w',
+			encoding=_encodings['repo.content'],
+			errors='backslashreplace') as myf:
 			for k in self._write_keys:
 				v = values.get(k)
 				if not v:
@@ -95,8 +83,7 @@ class database(fs_template.FsBased):
 				# k and v are coerced to unicode, in order to prevent TypeError
 				# when writing raw bytes to TextIOWrapper with Python 2.
 				myf.write("%s=%s\n" % (k, v))
-		finally:
-			myf.close()
+
 		self._ensure_access(fp)
 
 		#update written.  now we move it.
@@ -104,9 +91,21 @@ class database(fs_template.FsBased):
 		new_fp = os.path.join(self.location,cpv)
 		try:
 			os.rename(fp, new_fp)
-		except (OSError, IOError) as e:
-			os.remove(fp)
-			raise cache_errors.CacheCorruption(cpv, e)
+		except EnvironmentError as e:
+			success = False
+			try:
+				if errno.ENOENT == e.errno:
+					try:
+						self._ensure_dirs(cpv)
+						os.rename(fp, new_fp)
+						success = True
+					except EnvironmentError as e:
+						raise cache_errors.CacheCorruption(cpv, e)
+				else:
+					raise cache_errors.CacheCorruption(cpv, e)
+			finally:
+				if not success:
+					os.remove(fp)
 
 	def _delitem(self, cpv):
 #		import pdb;pdb.set_trace()

[gentoo-commits] proj/portage:master commit in: pym/portage/cache/

[Zac Medico]

commit:     d10bafb2c84be84ee47a2204938df4b3b9f238c0
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Sep 19 16:39:38 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Sep 19 16:41:56 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d10bafb2

cache.fs_template: set 0o644 defaut perms

Fixes 5652bc88514b ("flat_hash: use mkstemp in _setitem)
X-Gentoo-Bug: 594358
X-Gentoo-Bug-URL: https://bugs.gentoo.org/594358

 pym/portage/cache/fs_template.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/cache/fs_template.py b/pym/portage/cache/fs_template.py
index fa44abc..e3c3c12 100644
--- a/pym/portage/cache/fs_template.py
+++ b/pym/portage/cache/fs_template.py
@@ -24,7 +24,7 @@ class FsBased(template.database):
 
 	def __init__(self, *args, **config):
 
-		for x, y in (("gid", -1), ("perms", -1)):
+		for x, y in (("gid", -1), ("perms", 0o644)):
 			if x in config:
 				# Since Python 3.4, chown requires int type (no proxies).
 				setattr(self, "_" + x, int(config[x]))