* [gentoo-commits] proj/portage:master commit in: lib/portage/, lib/portage/dbapi/, lib/portage/tests/update/
@ 2023-12-30 21:47 Zac Medico
0 siblings, 0 replies; only message in thread
From: Zac Medico @ 2023-12-30 21:47 UTC (permalink / raw
To: gentoo-commits
commit: fb1d0a22f65747b750143080536a4129e8654f97
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 29 00:29:40 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Dec 30 21:46:04 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fb1d0a22
dbapi: KeyError tolerance during package moves
Raise a new CorruptionKeyError exception type instead
of a plain KeyError when os.stat fails. Treat this
type of exception as a warning during package moves.
Also fix this error that the test case triggered in
the binarytree.remove method:
NameError: name 'binpkg_path' is not defined
Bug: https://bugs.gentoo.org/920828
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/__init__.py | 14 ++++--
lib/portage/dbapi/bintree.py | 28 +++++++++---
lib/portage/dbapi/vartree.py | 9 ++--
lib/portage/exception.py | 6 ++-
lib/portage/tests/update/test_update_dbentry.py | 60 +++++++++++++++++++++++++
5 files changed, 103 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 09163e94d8..6f95b93a21 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -25,7 +25,11 @@ from portage.const import MERGING_IDENTIFIER
from portage import os
from portage import auxdbkeys
from portage.eapi import _get_eapi_attrs
-from portage.exception import InvalidBinaryPackageFormat, InvalidData
+from portage.exception import (
+ CorruptionKeyError,
+ InvalidBinaryPackageFormat,
+ InvalidData,
+)
from portage.localization import _
from _emerge.Package import Package
@@ -424,7 +428,7 @@ class dbapi:
if metadata_updates:
try:
aux_update(cpv, metadata_updates)
- except InvalidBinaryPackageFormat as e:
+ except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
warnings.warn(e)
if onUpdate:
onUpdate(maxval, i + 1)
@@ -470,5 +474,9 @@ class dbapi:
):
newslot = f"{newslot}/{mycpv.sub_slot}"
mydata = {"SLOT": newslot + "\n"}
- self.aux_update(mycpv, mydata)
+ try:
+ self.aux_update(mycpv, mydata)
+ except CorruptionKeyError as e:
+ warnings.warn(e)
+ continue
return moves
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index a20c8dfe26..a139e37659 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -37,6 +37,7 @@ from portage.dbapi.virtual import fakedbapi
from portage.dep import Atom, use_reduce, paren_enclose
from portage.exception import (
AlarmSignal,
+ CorruptionKeyError,
InvalidPackageName,
InvalidBinaryPackageFormat,
ParseError,
@@ -210,9 +211,9 @@ class bindbapi(fakedbapi):
raise KeyError(mycpv)
binpkg_path = os.path.join(self.bintree.pkgdir, binpkg_path)
try:
- st = os.lstat(binpkg_path)
- except OSError:
- raise KeyError(mycpv)
+ st = os.stat(binpkg_path)
+ except OSError as oe:
+ raise CorruptionKeyError(mycpv) from oe
binpkg_format = get_binpkg_format(binpkg_path)
if binpkg_format == "xpak":
@@ -283,8 +284,10 @@ class bindbapi(fakedbapi):
cpv_str += f"-{build_id}"
binpkg_path = self.bintree.getname(cpv)
- if not os.path.exists(binpkg_path):
- raise KeyError(cpv)
+ try:
+ os.stat(binpkg_path)
+ except OSError as oe:
+ raise CorruptionKeyError(cpv) from oe
binpkg_format = get_binpkg_format(binpkg_path)
if binpkg_format == "xpak":
@@ -694,7 +697,18 @@ class binarytree:
continue
binpkg_path = self.getname(mycpv)
- if os.path.exists(binpkg_path) and not os.access(binpkg_path, os.W_OK):
+ try:
+ os.stat(binpkg_path)
+ except FileNotFoundError:
+ writemsg(_("!!! File not found: %s\n") % binpkg_path, noiselevel=-1)
+ continue
+ except OSError as oe:
+ writemsg(
+ _("!!! File os error (path %s): %s\n") % (binpkg_path, oe),
+ noiselevel=-1,
+ )
+ continue
+ if not os.access(binpkg_path, os.W_OK):
writemsg(
_("!!! Cannot update readonly binary: %s\n") % mycpv, noiselevel=-1
)
@@ -1829,7 +1843,7 @@ class binarytree:
writemsg(
colorize(
"WARN",
- f"Failed to remove package: {binpkg_path} {str(err)}",
+ f"Failed to remove package: {pkg_path} {str(err)}",
)
)
finally:
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 11767a9f91..5d39ca1965 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["vardbapi", "vartree", "dblink"] + ["write_contents", "tar_contents"]
@@ -60,6 +60,7 @@ from portage.const import (
from portage.dbapi import dbapi
from portage.exception import (
CommandNotFound,
+ CorruptionKeyError,
InvalidData,
InvalidLocation,
InvalidPackageName,
@@ -990,8 +991,10 @@ class vardbapi(dbapi):
def aux_update(self, cpv, values):
mylink = self._dblink(cpv)
- if not mylink.exists():
- raise KeyError(cpv)
+ try:
+ os.stat(mylink.dbdir)
+ except OSError as oe:
+ raise CorruptionKeyError(cpv) from oe
self._bump_mtime(cpv)
self._clear_pkg_cache(mylink)
for k, v in values.items():
diff --git a/lib/portage/exception.py b/lib/portage/exception.py
index 153a9f9a55..7b48aa919e 100644
--- a/lib/portage/exception.py
+++ b/lib/portage/exception.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import signal
@@ -30,6 +30,10 @@ class CorruptionError(PortageException):
"""Corruption indication"""
+class CorruptionKeyError(CorruptionError, PortageKeyError):
+ """KeyError raised when corruption is detected (cause should be accesssible as __cause__)"""
+
+
class InvalidDependString(PortageException):
"""An invalid depend string has been encountered"""
diff --git a/lib/portage/tests/update/test_update_dbentry.py b/lib/portage/tests/update/test_update_dbentry.py
index a3c9a37e8c..3b3f0caae6 100644
--- a/lib/portage/tests/update/test_update_dbentry.py
+++ b/lib/portage/tests/update/test_update_dbentry.py
@@ -1,6 +1,7 @@
# Copyright 2012-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import shutil
import sys
import re
import textwrap
@@ -9,6 +10,7 @@ import portage
from portage import os
from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.dep import Atom
+from portage.exception import CorruptionKeyError
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import ResolverPlayground
from portage.update import update_dbentry
@@ -186,6 +188,11 @@ class UpdateDbentryTestCase(TestCase):
"EAPI": "4",
"SLOT": "2",
},
+ "dev-libs/B-2::test_repo": {
+ "SLOT": "2",
+ "RDEPEND": "dev-libs/M dev-libs/N dev-libs/P",
+ "EAPI": "4",
+ },
"dev-libs/B-1::test_repo": {
"RDEPEND": "dev-libs/M dev-libs/N dev-libs/P",
"EAPI": "4",
@@ -215,6 +222,11 @@ class UpdateDbentryTestCase(TestCase):
"RDEPEND": "dev-libs/M dev-libs/N dev-libs/P",
"EAPI": "4",
},
+ "dev-libs/B-2::test_repo": {
+ "SLOT": "2",
+ "RDEPEND": "dev-libs/M dev-libs/N dev-libs/P",
+ "EAPI": "4",
+ },
}
world = ["dev-libs/M", "dev-libs/N"]
@@ -269,6 +281,34 @@ class UpdateDbentryTestCase(TestCase):
)
)
+ # Delete some things in order to trigger CorruptionKeyError during package moves.
+ corruption_atom = Atom("dev-libs/B:2")
+ # Demonstrate initial state.
+ self.assertEqual(bindb.match(corruption_atom), ["dev-libs/B-2"])
+ for cpv in bindb.match(corruption_atom):
+ os.unlink(bindb.bintree.getname(cpv))
+ self.assertRaises(
+ CorruptionKeyError,
+ bindb.aux_update,
+ cpv,
+ {"RDEPEND": "dev-libs/M-moved"},
+ )
+ # Demonstrate corrupt state.
+ self.assertEqual(bindb.match(corruption_atom), ["dev-libs/B-2"])
+
+ # Demonstrate initial state.
+ self.assertEqual(vardb.match(corruption_atom), ["dev-libs/B-2"])
+ for cpv in vardb.match(corruption_atom):
+ shutil.rmtree(vardb.getpath(cpv))
+ self.assertRaises(
+ CorruptionKeyError,
+ vardb.aux_update,
+ cpv,
+ {"RDEPEND": "dev-libs/M-moved"},
+ )
+ # Demonstrate correct state because vardbapi checks the disk.
+ self.assertEqual(vardb.match(corruption_atom), [])
+
global_noiselimit = portage.util.noiselimit
portage.util.noiselimit = -2
try:
@@ -304,6 +344,26 @@ class UpdateDbentryTestCase(TestCase):
self.assertTrue("dev-libs/M" in rdepend)
self.assertTrue("dev-libs/M-moved" not in rdepend)
+ # Demonstrate that match still returns stale results
+ # due to intentional corruption.
+ self.assertEqual(bindb.match(corruption_atom), ["dev-libs/B-2"])
+
+ # Update bintree state so aux_get will properly raise KeyError.
+ for cpv in bindb.match(corruption_atom):
+ # Demonstrate that aux_get returns stale results.
+ self.assertEqual(
+ ["dev-libs/M dev-libs/N dev-libs/P"],
+ bindb.aux_get(cpv, ["RDEPEND"]),
+ )
+ bindb.bintree.remove(cpv)
+ self.assertEqual(bindb.match(corruption_atom), [])
+ self.assertRaises(
+ KeyError, bindb.aux_get, "dev-libs/B-2", ["RDEPEND"]
+ )
+ self.assertRaises(
+ KeyError, vardb.aux_get, "dev-libs/B-2", ["RDEPEND"]
+ )
+
selected_set.load()
self.assertTrue("dev-libs/M" not in selected_set)
self.assertTrue("dev-libs/M-moved" in selected_set)
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2023-12-30 21:47 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2023-12-30 21:47 [gentoo-commits] proj/portage:master commit in: lib/portage/, lib/portage/dbapi/, lib/portage/tests/update/ Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox