[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/, lib/portage/cache/

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/, lib/portage/cache/

[Zac Medico]

commit:     687dbcb0ab2f7d254bdc53b1332b3d480b2de581
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May  2 20:51:15 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May  2 21:04:37 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=687dbcb0

anydbm: avoid interference between cleanse_keys and _eclasses_

Fix this AuxdbTestCase failure for anydbm:

test_anydbm (portage.tests.dbapi.test_auxdb.AuxdbTestCase) ... Exception in callback None()
handle: <Handle cancelled>
Traceback (most recent call last):
  File "/usr/lib64/python3.6/asyncio/events.py", line 145, in _run
    self._callback(*self._args)
  File "lib/_emerge/EbuildMetadataPhase.py", line 143, in _output_handler
    self._async_waitpid()
  File "lib/_emerge/SubProcess.py", line 60, in _async_waitpid
    add_child_handler(self.pid, self._async_waitpid_cb)
  File "/usr/lib64/python3.6/asyncio/unix_events.py", line 873, in add_child_handler
    self._do_waitpid(pid)
  File "/usr/lib64/python3.6/asyncio/unix_events.py", line 919, in _do_waitpid
    callback(pid, returncode, *args)
  File "lib/_emerge/EbuildMetadataPhase.py", line 207, in _async_waitpid_cb
    self.repo_path, metadata, self.ebuild_hash)
  File "lib/portage/dbapi/porttree.py", line 545, in _write_cache
    cache[cpv] = metadata
  File "lib/portage/cache/template.py", line 146, in __setitem__
    d["_eclasses_"] = self._internal_eclasses(d["_eclasses_"],
  File "lib/portage/cache/mappings.py", line 213, in __getitem__
    raise KeyError(key)
KeyError: '_eclasses_'

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/cache/template.py         | 2 +-
 lib/portage/tests/dbapi/test_auxdb.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/portage/cache/template.py b/lib/portage/cache/template.py
index 8662d859f..6b4878347 100644
--- a/lib/portage/cache/template.py
+++ b/lib/portage/cache/template.py
@@ -133,7 +133,7 @@ class database(object):
 		d = None
 		if self.cleanse_keys:
 			d=ProtectedDict(values)
-			for k, v in list(d.items()):
+			for k, v in list(item for item in d.items() if item[0] != "_eclasses_"):
 				if not v:
 					del d[k]
 		if "_eclasses_" in values:

diff --git a/lib/portage/tests/dbapi/test_auxdb.py b/lib/portage/tests/dbapi/test_auxdb.py
index 73fc2b2c3..cfcabc8bb 100644
--- a/lib/portage/tests/dbapi/test_auxdb.py
+++ b/lib/portage/tests/dbapi/test_auxdb.py
@@ -11,6 +11,13 @@ from portage.util.futures.compat_coroutine import coroutine
 
 class AuxdbTestCase(TestCase):
 
+	def test_anydbm(self):
+		try:
+			from portage.cache.anydbm import database
+		except ImportError:
+			self.skipTest('dbm import failed')
+		self._test_mod('portage.cache.anydbm.database')
+
 	def test_flat_hash_md5(self):
 		self._test_mod('portage.cache.flat_hash.md5_database')
 

[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/, lib/portage/cache/

[Zac Medico]

commit:     d4f212def0dcba58e8a146daf8da7b481491de39
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Aug  8 01:00:50 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Aug  9 02:41:43 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=d4f212de

sqlite: fork safety (bug 736334)

Use a separate connection instance for each pid, since
it is not safe to use a connection created in a parent
process.

See: https://www.sqlite.org/howtocorrupt.html
Bug: https://bugs.gentoo.org/736334
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/cache/sqlite.py           |  9 +++++----
 lib/portage/tests/dbapi/test_auxdb.py | 38 ++++++++++++++++++++++++++++++-----
 2 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/lib/portage/cache/sqlite.py b/lib/portage/cache/sqlite.py
index 0395dd516..36a4f049e 100644
--- a/lib/portage/cache/sqlite.py
+++ b/lib/portage/cache/sqlite.py
@@ -4,6 +4,7 @@
 import collections
 import re
 
+import portage
 from portage.cache import fs_template
 from portage.cache import cache_errors
 from portage import os
@@ -25,7 +26,7 @@ class database(fs_template.FsBased):
 	cache_bytes = 1024 * 1024 * 10
 
 	_connection_info_entry = collections.namedtuple('_connection_info_entry',
-		('connection', 'cursor'))
+		('connection', 'cursor', 'pid'))
 
 	def __init__(self, *args, **config):
 		super(database, self).__init__(*args, **config)
@@ -71,13 +72,13 @@ class database(fs_template.FsBased):
 
 	@property
 	def _db_cursor(self):
-		if self._db_connection_info is None:
+		if self._db_connection_info is None or self._db_connection_info.pid != portage.getpid():
 			self._db_init_connection()
 		return self._db_connection_info.cursor
 
 	@property
 	def _db_connection(self):
-		if self._db_connection_info is None:
+		if self._db_connection_info is None or self._db_connection_info.pid != portage.getpid():
 			self._db_init_connection()
 		return self._db_connection_info.connection
 
@@ -94,7 +95,7 @@ class database(fs_template.FsBased):
 			connection = self._db_module.connect(
 				database=_unicode_decode(self._dbpath), **connection_kwargs)
 			cursor = connection.cursor()
-			self._db_connection_info = self._connection_info_entry(connection, cursor)
+			self._db_connection_info = self._connection_info_entry(connection, cursor, portage.getpid())
 			self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8"))
 			if not self.readonly and not self._ensure_access(self._dbpath):
 				raise cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % self._dbpath)

diff --git a/lib/portage/tests/dbapi/test_auxdb.py b/lib/portage/tests/dbapi/test_auxdb.py
index 5c79357d7..907c289fb 100644
--- a/lib/portage/tests/dbapi/test_auxdb.py
+++ b/lib/portage/tests/dbapi/test_auxdb.py
@@ -4,7 +4,8 @@
 from portage.tests import TestCase
 from portage.tests.resolver.ResolverPlayground import ResolverPlayground
 from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
+from portage.util.futures.executor.fork import ForkExecutor
 
 
 class AuxdbTestCase(TestCase):
@@ -14,13 +15,13 @@ class AuxdbTestCase(TestCase):
 			from portage.cache.anydbm import database
 		except ImportError:
 			self.skipTest('dbm import failed')
-		self._test_mod('portage.cache.anydbm.database')
+		self._test_mod('portage.cache.anydbm.database', multiproc=False)
 
 	def test_flat_hash_md5(self):
 		self._test_mod('portage.cache.flat_hash.md5_database')
 
 	def test_volatile(self):
-		self._test_mod('portage.cache.volatile.database')
+		self._test_mod('portage.cache.volatile.database', multiproc=False)
 
 	def test_sqite(self):
 		try:
@@ -29,7 +30,7 @@ class AuxdbTestCase(TestCase):
 			self.skipTest('sqlite3 import failed')
 		self._test_mod('portage.cache.sqlite.database')
 
-	def _test_mod(self, auxdbmodule):
+	def _test_mod(self, auxdbmodule, multiproc=True):
 		ebuilds = {
 			"cat/A-1": {
 				"EAPI": "7",
@@ -61,8 +62,33 @@ class AuxdbTestCase(TestCase):
 
 		portdb = playground.trees[playground.eroot]["porttree"].dbapi
 
+		def test_func():
+			return asyncio._wrap_loop().run_until_complete(self._test_mod_async(
+				ebuilds, ebuild_inherited, eclass_defined_phases, eclass_depend, portdb))
+
+		self.assertTrue(test_func())
+
 		loop = asyncio._wrap_loop()
-		loop.run_until_complete(self._test_mod_async(ebuilds, ebuild_inherited, eclass_defined_phases, eclass_depend, portdb))
+		self.assertTrue(loop.run_until_complete(loop.run_in_executor(ForkExecutor(), test_func)))
+
+		auxdb = portdb.auxdb[portdb.getRepositoryPath('test_repo')]
+		cpv = next(iter(ebuilds))
+
+		def modify_auxdb():
+			metadata = auxdb[cpv]
+			metadata['RESTRICT'] = 'test'
+			try:
+				del metadata['_eclasses_']
+			except KeyError:
+				pass
+			auxdb[cpv] = metadata
+
+		if multiproc:
+			loop.run_until_complete(loop.run_in_executor(ForkExecutor(), modify_auxdb))
+		else:
+			modify_auxdb()
+
+		self.assertEqual(auxdb[cpv]['RESTRICT'], 'test')
 
 	@coroutine
 	def _test_mod_async(self, ebuilds, ebuild_inherited, eclass_defined_phases, eclass_depend, portdb):
@@ -73,3 +99,5 @@ class AuxdbTestCase(TestCase):
 			self.assertEqual(depend, eclass_depend)
 			self.assertEqual(eapi, metadata['EAPI'])
 			self.assertEqual(frozenset(inherited.split()), ebuild_inherited)
+
+		coroutine_return(True)