* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2018-09-24 0:46 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2018-09-24 0:46 UTC (permalink / raw
To: gentoo-commits
commit: d9151dfb8144563119f016b439de2b0f53d8858e
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Sep 24 00:20:47 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Sep 24 00:45:40 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=d9151dfb
portdbapi.xmatch: remove deprecated *match-list
The bestmatch-list and match-list level arguments have been deprecated
since v2.2.0, commit 0d375f1105ad67c8b7e3b16b57cdbb367f99cd69.
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index aa8b50a57..56955ec34 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1165,18 +1165,6 @@ class portdbapi(dbapi):
else:
myval = ""
- elif level == "bestmatch-list":
- #dep match -- find best match but restrict search to sublist
- warnings.warn("The 'bestmatch-list' mode of "
- "portage.dbapi.porttree.portdbapi.xmatch is deprecated",
- DeprecationWarning, stacklevel=2)
- myval = best(list(self._iter_match(mydep, mylist)))
- elif level == "match-list":
- #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible())
- warnings.warn("The 'match-list' mode of "
- "portage.dbapi.porttree.portdbapi.xmatch is deprecated",
- DeprecationWarning, stacklevel=2)
- myval = list(self._iter_match(mydep, mylist))
else:
raise AssertionError(
"Invalid level argument: '%s'" % level)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2018-09-24 7:30 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2018-09-24 7:30 UTC (permalink / raw
To: gentoo-commits
commit: ebd2c386f5ae2aced4c3ea05dacffdb99bd0bf5b
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Sep 24 03:04:13 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Sep 24 06:45:32 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=ebd2c386
portdbapi: add async_xmatch method (bug 666940)
Add an async_xmatch method, and use it to implement the synchronous
xmatch method. Deprecate unused xmatch method parameters. Use the
compat_coroutine decorator for backward compatibility with python2.7
(via PEP 342 enhanced generators).
Bug: https://bugs.gentoo.org/666940
Suggested-by: Daniel Robbins <drobbins <AT> funtoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 60 +++++++++++++++++++++++++++++++++++--------
1 file changed, 49 insertions(+), 11 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 56955ec34..76b7967f7 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -37,6 +37,7 @@ from portage import _unicode_encode
from portage import OrderedDict
from portage.util._eventloop.EventLoop import EventLoop
from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
from portage.util.futures.iter_completed import iter_gather
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
@@ -1055,8 +1056,20 @@ class portdbapi(dbapi):
self._better_cache = None
self.frozen = 0
- def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):
- "caching match function; very trick stuff"
+ def xmatch(self, level, origdep, mydep=DeprecationWarning,
+ mykey=DeprecationWarning, mylist=DeprecationWarning):
+ """
+ Caching match function.
+
+ @param level: xmatch level (bestmatch-visible, match-all-cpv-only
+ match-allmatch-visible, minimum-all, minimum-all-ignore-profile,
+ or minimum-visible)
+ @type level: str
+ @param origdep: dependency to match (may omit category)
+ @type origdep: portage.dep.Atom or str
+ @return: match result(s)
+ @rtype: _pkg_str or list of _pkg_str (depends on level)
+ """
if level == "list-visible":
level = "match-visible"
warnings.warn("The 'list-visible' mode of "
@@ -1064,21 +1077,46 @@ class portdbapi(dbapi):
"has been renamed to match-visible",
DeprecationWarning, stacklevel=2)
- if mydep is None:
- #this stuff only runs on first call of xmatch()
- #create mydep, mykey from origdep
- mydep = dep_expand(origdep, mydb=self, settings=self.settings)
- mykey = mydep.cp
+ if mydep is not DeprecationWarning:
+ warnings.warn("The 'mydep' parameter of "
+ "portage.dbapi.porttree.portdbapi.xmatch"
+ " is deprecated and ignored",
+ DeprecationWarning, stacklevel=2)
+
+ loop = self._event_loop
+ return loop.run_until_complete(
+ self.async_xmatch(level, origdep, loop=loop))
+
+ @coroutine
+ def async_xmatch(self, level, origdep, loop=None):
+ """
+ Asynchronous form of xmatch.
+
+ @param level: xmatch level (bestmatch-visible, match-all-cpv-only
+ match-allmatch-visible, minimum-all, minimum-all-ignore-profile,
+ or minimum-visible)
+ @type level: str
+ @param origdep: dependency to match (may omit category)
+ @type origdep: portage.dep.Atom or str
+ @param loop: event loop (defaults to global event loop)
+ @type loop: EventLoop
+ @return: match result(s)
+ @rtype: asyncio.Future (or compatible), which results in a _pkg_str
+ or list of _pkg_str (depends on level)
+ """
+ mydep = dep_expand(origdep, mydb=self, settings=self.settings)
+ mykey = mydep.cp
#if no updates are being made to the tree, we can consult our xcache...
cache_key = None
if self.frozen:
cache_key = (mydep, mydep.unevaluated_atom)
try:
- return self.xcache[level][cache_key][:]
+ coroutine_return(self.xcache[level][cache_key][:])
except KeyError:
pass
+ loop = asyncio._wrap_loop(loop)
myval = None
mytree = None
if mydep.repo is not None:
@@ -1131,8 +1169,8 @@ class portdbapi(dbapi):
for cpv in iterfunc(mylist):
try:
- metadata = dict(zip(aux_keys,
- self.aux_get(cpv, aux_keys, myrepo=cpv.repo)))
+ metadata = dict(zip(aux_keys, (yield self.async_aux_get(cpv,
+ aux_keys, myrepo=cpv.repo, loop=loop))))
except KeyError:
# ebuild not in this repo, or masked by corruption
continue
@@ -1176,7 +1214,7 @@ class portdbapi(dbapi):
if not isinstance(myval, _pkg_str):
myval = myval[:]
- return myval
+ coroutine_return(myval)
def match(self, mydep, use_cache=1):
return self.xmatch("match-visible", mydep)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2019-01-11 10:14 Fabian Groffen
0 siblings, 0 replies; 71+ messages in thread
From: Fabian Groffen @ 2019-01-11 10:14 UTC (permalink / raw
To: gentoo-commits
commit: cfa915d0d575379df4b9f17fd2db3594155861ca
Author: Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 7 14:19:15 2019 +0000
Commit: Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Fri Jan 11 10:14:22 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfa915d0
collision_protect: use dynamic report interval
The reporting of files remaining can look somewhat odd since the report
interval is hardcoded to be per 1000 objects. Adjust this interval to
be time based. This means that modern (fast) machines likely will never
see the countdown messages at all. On slow setups the message will be
informative that there is progress, albeit rather slowly. While at it,
report percentage done.
Output before this patch:
* checking 6158 files for package collisions
5158 files remaining ...
4158 files remaining ...
3158 files remaining ...
2158 files remaining ...
1158 files remaining ...
158 files remaining ...
Possible output after this patch on a slower machine:
* checking 6158 files for package collisions
48% done, 3145 files remaining ...
96% done, 192 files remaining ...
100% done
Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 9febf0c71..63389f9a3 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -35,6 +35,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.install_mask:install_mask_dir,InstallMask',
'portage.util.listdir:dircache,listdir',
'portage.util.movefile:movefile',
+ 'portage.util.monotonic:monotonic',
'portage.util.path:first_existing,iter_parents',
'portage.util.writeable_check:get_ro_checker',
'portage.util._xattr:xattr',
@@ -3453,13 +3454,21 @@ class dblink(object):
symlink_collisions = []
destroot = self.settings['ROOT']
totfiles = len(file_list) + len(symlink_list)
+ previous = monotonic()
+ progress_shown = False
+ report_interval = 1.7 # seconds
+ falign = len("%d" % totfiles)
showMessage(_(" %s checking %d files for package collisions\n") % \
(colorize("GOOD", "*"), totfiles))
for i, (f, f_type) in enumerate(chain(
((f, "reg") for f in file_list),
((f, "sym") for f in symlink_list))):
- if i % 1000 == 0 and i != 0:
- showMessage(_("%d files remaining ...\n") % (totfiles - i))
+ current = monotonic()
+ if current - previous > report_interval:
+ showMessage(_("%3d%% done, %*d files remaining ...\n") %
+ (i * 100 / totfiles, falign, totfiles - i))
+ previous = current
+ progress_shown = True
dest_path = normalize_path(
os.path.join(destroot, f.lstrip(os.path.sep)))
@@ -3548,6 +3557,8 @@ class dblink(object):
break
if stopmerge:
collisions.append(f)
+ if progress_shown:
+ showMessage(_("100% done\n"))
return collisions, dirs_ro, symlink_collisions, plib_collisions
def _lstat_inode_map(self, path_iter):
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2019-01-20 6:55 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2019-01-20 6:55 UTC (permalink / raw
To: gentoo-commits
commit: bbe00f515b80be3d6e279e4033f1cbc776a7fe27
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 20 06:39:39 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jan 20 06:53:34 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=bbe00f51
portagetree: getname and portroot DeprecationWarning
Bug: https://bugs.gentoo.org/668164
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 76b7967f7..2ff3e1b34 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1339,10 +1339,18 @@ class portagetree(object):
" constructor is unused",
DeprecationWarning, stacklevel=2)
- self.portroot = settings["PORTDIR"]
self.__virtual = virtual
self.dbapi = portdbapi(mysettings=settings)
+ @property
+ def portroot(self):
+ """Deprecated. Use the portdbapi getRepositoryPath method instead."""
+ warnings.warn("The portroot attribute of "
+ "portage.dbapi.porttree.portagetree is deprecated. Use the "
+ "portdbapi getRepositoryPath method instead.",
+ DeprecationWarning, stacklevel=3)
+ return self.settings['PORTDIR']
+
@property
def root(self):
warnings.warn("The root attribute of " + \
@@ -1383,7 +1391,11 @@ class portagetree(object):
return self.dbapi.cp_all()
def getname(self, pkgname):
- "returns file location for this particular package (DEPRECATED)"
+ """Deprecated. Use the portdbapi findname method instead."""
+ warnings.warn("The getname method of "
+ "portage.dbapi.porttree.portagetree is deprecated. "
+ "Use the portdbapi findname method instead.",
+ DeprecationWarning, stacklevel=2)
if not pkgname:
return ""
mysplit = pkgname.split("/")
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2019-05-11 21:16 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2019-05-11 21:16 UTC (permalink / raw
To: gentoo-commits
commit: fbebef9677d63db70f1c68b197e58b041ec6ac61
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu May 9 20:10:46 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 11 20:55:49 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fbebef96
repoman: check IUSE in _match_use for USE defaults (bug 685482)
This corrects a problem triggered with USE defaults where repoman
returns an incorrect negative match for an atom with USE defaults.
For example, it triggered this dependency.bad error:
RepoMan scours the neighborhood...
dependency.bad [fatal] 2
dev-libs/libxml2/libxml2-2.9.9-r1.ebuild: DEPEND: ~riscv(default/linux/riscv/17.0/rv64gc)
[ '>=dev-lang/python-exec-2:=[python_targets_python2_7(-)?,python_targets_python3_5(-)?,python_targets_python3_6(-)?,python_targets_python3_7(-)?,-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]']
dev-libs/libxml2/libxml2-2.9.9-r1.ebuild: RDEPEND: ~riscv(default/linux/riscv/17.0/rv64gc)
[ '>=dev-lang/python-exec-2:=[python_targets_python2_7(-)?,python_targets_python3_5(-)?,python_targets_python3_6(-)?,python_targets_python3_7(-)?,-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]']
State in the _match_use method looked like this:
(Pdb) atom
'>=dev-lang/python-exec-2:=[python_targets_python3_7(-),-python_single_target_python2_7(-),-python_single_target_python3_5(-),-python_single_target_python3_6(-),-python_single_target_python3_7(-)]'
(Pdb) useforce
frozenset({'python_targets_pypy', 'python_targets_pypy3', 'python_targets_python3_5', 'big-endian', 'userland_GNU', 'python_targets_python3_7', 'riscv', 'python_single_target_python3_7', 'python_targets_python2_7', 'elibc_glibc', 'python_targets_python3_6', 'abi_riscv_lp64d', 'kernel_linux', 'python_targets_jython2_7'})
(Pdb) atom.use.disabled
frozenset({'python_single_target_python3_6', 'python_single_target_python2_7', 'python_single_target_python3_7', 'python_single_target_python3_5'})
(Pdb) atom.use.missing_disabled
frozenset({'python_single_target_python2_7', 'python_targets_python3_5', 'python_targets_python3_7', 'python_single_target_python3_5', 'python_single_target_python3_7', 'python_targets_python2_7', 'python_targets_python3_6', 'python_single_target_python3_6'})
Bug: https://bugs.gentoo.org/685482
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/__init__.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 6fca6090c..80f8a689f 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -314,12 +314,12 @@ class dbapi(object):
# Check masked and forced flags for repoman.
usemask = self.settings._getUseMask(pkg,
stable=self.settings._parent_stable)
- if any(x in usemask for x in atom.use.enabled):
+ if any(x in usemask and iuse.get_real_flag(x) is not None for x in atom.use.enabled):
return False
useforce = self.settings._getUseForce(pkg,
stable=self.settings._parent_stable)
- if any(x in useforce and x not in usemask
+ if any(x in useforce and x not in usemask and iuse.get_real_flag(x) is not None
for x in atom.use.disabled):
return False
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2019-06-20 19:43 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2019-06-20 19:43 UTC (permalink / raw
To: gentoo-commits
commit: 611a53a6943bf25ac8c1d909e34e4b37c8873e83
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 20 01:37:53 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jun 20 02:21:36 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=611a53a6
portdbapi.getfetchsizes: use .__download__ suffix
If the file does not exist locally or the size is not correct,
use the file with the .__download__ suffix to compute the
remaining_size to fetch.
Fixes: ebbde237d33e ("fetch: atomic downloads (bug 175612)")
Bug: https://bugs.gentoo.org/688124
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index edff0c2f2..e4b688b9a 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -14,6 +14,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.dbapi.dep_expand:dep_expand',
'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot',
'portage.package.ebuild.doebuild:doebuild',
+ 'portage.package.ebuild.fetch:_download_suffix',
'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
'portage.util.listdir:listdir',
'portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str',
@@ -844,6 +845,17 @@ class portdbapi(dbapi):
mystat = os.stat(file_path)
except OSError:
pass
+ else:
+ if mystat.st_size != fetch_size:
+ # Use file with _download_suffix instead.
+ mystat = None
+
+ if mystat is None:
+ try:
+ mystat = os.stat(file_path + _download_suffix)
+ except OSError:
+ pass
+
if mystat is None:
existing_size = 0
ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS")
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2019-08-24 3:15 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2019-08-24 3:15 UTC (permalink / raw
To: gentoo-commits
commit: f90400eedc6a8788878c050880db564800c825d2
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Aug 23 20:00:15 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Aug 24 02:41:35 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=f90400ee
preserve-libs: ignore dropped non-soname symlink (bug 692698)
Fix the dblink _find_libs_to_preserve method to ignore a dropped
non-soname symlink. For example, pam-1.3.1-r1 drops the non-soname
symlink named libpam_misc.so, and we don't want this to trigger
unnecessary preservation of the corresponding library, since the
corresponding libpam_misc.so.0 soname symlink and the hardlink
that it references are still provided by pam-1.3.1-r1.
Bug: https://bugs.gentoo.org/692698
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 4f069474b..fa1e1523c 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3133,10 +3133,6 @@ class dblink(object):
os = portage.os
f = f_abs[root_len:]
- if not unmerge and self.isowner(f):
- # We have an indentically named replacement file,
- # so we don't try to preserve the old copy.
- continue
try:
consumers = linkmap.findConsumers(f,
exclude_providers=(installed_instance.isowner,))
@@ -3184,16 +3180,27 @@ class dblink(object):
hardlinks = set()
soname_symlinks = set()
soname = linkmap.getSoname(next(iter(preserve_node.alt_paths)))
+ have_replacement_soname_link = False
+ have_replacement_hardlink = False
for f in preserve_node.alt_paths:
f_abs = os.path.join(root, f.lstrip(os.sep))
try:
if stat.S_ISREG(os.lstat(f_abs).st_mode):
hardlinks.add(f)
+ if not unmerge and self.isowner(f):
+ have_replacement_hardlink = True
+ if os.path.basename(f) == soname:
+ have_replacement_soname_link = True
elif os.path.basename(f) == soname:
soname_symlinks.add(f)
+ if not unmerge and self.isowner(f):
+ have_replacement_soname_link = True
except OSError:
pass
+ if have_replacement_hardlink and have_replacement_soname_link:
+ continue
+
if hardlinks:
preserve_paths.update(hardlinks)
preserve_paths.update(soname_symlinks)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-02-03 3:04 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-02-03 3:04 UTC (permalink / raw
To: gentoo-commits
commit: 6d8104e665b55174980cd88695f0f956935507b7
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 3 03:01:06 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb 3 03:02:02 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=6d8104e6
sphinx-build: remove broken lazyimport for _spawn_phase
Fixes: d318bcce356b ("vartree: add missing _merge_unicode_error import")
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 29843656f..050366528 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -23,7 +23,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.locks:lockdir,unlockdir,lockfile,unlockfile',
'portage.output:bold,colorize',
'portage.package.ebuild.doebuild:doebuild_environment,' + \
- '_merge_unicode_error', '_spawn_phase',
+ '_merge_unicode_error',
'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
'portage.package.ebuild._ipc.QueryCommand:QueryCommand',
'portage.process:find_binary',
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-02-20 9:55 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-02-20 9:55 UTC (permalink / raw
To: gentoo-commits
commit: 3de00842d69669a15adaf9e81b0007a7052da5d0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 20 09:51:34 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 20 09:52:33 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=3de00842
async_fetch_map: handle _parse_uri_map exception (bug 710130)
If _parse_uri_map raises an exception then raise it to the
caller via the returned future.
Bug: https://bugs.gentoo.org/710130
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 4bb396a6c..08af17bcd 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2019 Gentoo Authors
+# Copyright 1998-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import unicode_literals
@@ -804,8 +804,11 @@ class portdbapi(dbapi):
(mypkg, eapi)))
return
- result.set_result(_parse_uri_map(mypkg,
- {'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
+ try:
+ result.set_result(_parse_uri_map(mypkg,
+ {'EAPI':eapi,'SRC_URI':myuris}, use=useflags))
+ except Exception as e:
+ result.set_exception(e)
aux_get_future = self.async_aux_get(
mypkg, ["EAPI", "SRC_URI"], mytree=mytree, loop=loop)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-06-07 3:26 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-06-07 3:26 UTC (permalink / raw
To: gentoo-commits
commit: 976c1133100f6da81cd8d6e13f8a723a9fc7cd85
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jun 7 02:25:11 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jun 7 02:53:27 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=976c1133
_better_cache._scan_cat: avoid stat calls (bug 725934)
When processing category listdir results, do not use os.path.isdir to
identify packages, in order to avoid unecessary stat calls. Instead,
use the Atom class to validate package names. This may cause creation
of some cache entries for non-packages, but it will not do any harm
since these entries will never be accessed via the __getitem__ method.
Bug: https://bugs.gentoo.org/725934
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 08af17bcd..ed992d1e2 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -164,8 +164,13 @@ class _better_cache(object):
raise
continue
for p in pkg_list:
- if os.path.isdir(cat_dir + "/" + p):
- self._items[cat + "/" + p].append(repo)
+ try:
+ atom = Atom("%s/%s" % (cat, p))
+ except InvalidAtom:
+ continue
+ if atom != atom.cp:
+ continue
+ self._items[atom.cp].append(repo)
self._scanned_cats.add(cat)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-07-22 17:46 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-07-22 17:46 UTC (permalink / raw
To: gentoo-commits
commit: 3561071e07ad47db91bf0f2c2c2b02e2061b217c
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 19 06:31:37 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jul 22 16:55:33 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=3561071e
MergeProcess: replace os.fork with multiprocessing.Process (bug 730192)
Fix the MergeProcess _spawn method to call the superclass _spawn
method, in order to replace os.fork with multiprocessing.Process,
promoting a healthy state for the forked interpreter.
Bug: https://bugs.gentoo.org/730192
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 106 +++++++++++--------------------------
1 file changed, 30 insertions(+), 76 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index 274ef586f..6924c8b0e 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -3,9 +3,6 @@
import io
import platform
-import signal
-import sys
-import traceback
import fcntl
import portage
@@ -24,7 +21,7 @@ class MergeProcess(ForkProcess):
'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild',
'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge',
'_elog_reader_fd',
- '_buf', '_elog_keys', '_locked_vdb')
+ '_buf', '_counter', '_dblink', '_elog_keys', '_locked_vdb')
def _start(self):
# Portage should always call setcpv prior to this
@@ -103,8 +100,8 @@ class MergeProcess(ForkProcess):
def _spawn(self, args, fd_pipes, **kwargs):
"""
- Fork a subprocess, apply local settings, and call
- dblink.merge(). TODO: Share code with ForkProcess.
+ Extend the superclass _spawn method to perform some pre-fork and
+ post-fork actions.
"""
elog_reader_fd, elog_writer_fd = os.pipe()
@@ -132,57 +129,31 @@ class MergeProcess(ForkProcess):
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
- counter = None
if not self.unmerge:
- counter = self.vartree.dbapi.counter_tick()
-
- parent_pid = os.getpid()
- pid = None
- try:
- pid = os.fork()
-
- if pid != 0:
- if not isinstance(pid, int):
- raise AssertionError(
- "fork returned non-integer: %s" % (repr(pid),))
-
- os.close(elog_writer_fd)
- self._elog_reader_fd = elog_reader_fd
- self._buf = ""
- self._elog_keys = set()
- # Discard messages which will be collected by the subprocess,
- # in order to avoid duplicates (bug #446136).
- portage.elog.messages.collect_messages(key=mylink.mycpv)
-
- # invalidate relevant vardbapi caches
- if self.vartree.dbapi._categories is not None:
- self.vartree.dbapi._categories = None
- self.vartree.dbapi._pkgs_changed = True
- self.vartree.dbapi._clear_pkg_cache(mylink)
-
- return [pid]
-
- os.close(elog_reader_fd)
-
- # Use default signal handlers in order to avoid problems
- # killing subprocesses as reported in bug #353239.
- signal.signal(signal.SIGINT, signal.SIG_DFL)
- signal.signal(signal.SIGTERM, signal.SIG_DFL)
-
- # Unregister SIGCHLD handler and wakeup_fd for the parent
- # process's event loop (bug 655656).
- signal.signal(signal.SIGCHLD, signal.SIG_DFL)
- try:
- wakeup_fd = signal.set_wakeup_fd(-1)
- if wakeup_fd > 0:
- os.close(wakeup_fd)
- except (ValueError, OSError):
- pass
-
- portage.locks._close_fds()
- # We don't exec, so use close_fds=False
- # (see _setup_pipes docstring).
- portage.process._setup_pipes(fd_pipes, close_fds=False)
+ self._counter = self.vartree.dbapi.counter_tick()
+
+ self._dblink = mylink
+ self._elog_reader_fd = elog_reader_fd
+ pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs)
+ os.close(elog_writer_fd)
+ self._buf = ""
+ self._elog_keys = set()
+ # Discard messages which will be collected by the subprocess,
+ # in order to avoid duplicates (bug #446136).
+ portage.elog.messages.collect_messages(key=mylink.mycpv)
+
+ # invalidate relevant vardbapi caches
+ if self.vartree.dbapi._categories is not None:
+ self.vartree.dbapi._categories = None
+ self.vartree.dbapi._pkgs_changed = True
+ self.vartree.dbapi._clear_pkg_cache(mylink)
+
+ return pids
+
+ def _run(self):
+ os.close(self._elog_reader_fd)
+ counter = self._counter
+ mylink = self._dblink
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
@@ -207,8 +178,7 @@ class MergeProcess(ForkProcess):
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
- try:
- if self.unmerge:
+ if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
@@ -219,27 +189,11 @@ class MergeProcess(ForkProcess):
finally:
mylink.unlockdb()
rval = os.EX_OK
- else:
+ else:
rval = mylink.merge(self.pkgloc, self.infloc,
myebuild=self.myebuild, mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes, counter=counter)
- except SystemExit:
- raise
- except:
- traceback.print_exc()
- # os._exit() skips stderr flush!
- sys.stderr.flush()
- finally:
- os._exit(rval)
-
- finally:
- if pid == 0 or (pid is None and os.getpid() != parent_pid):
- # Call os._exit() from a finally block in order
- # to suppress any finally blocks from earlier
- # in the call stack (see bug #345289). This
- # finally block has to be setup before the fork
- # in order to avoid a race condition.
- os._exit(1)
+ return rval
def _async_waitpid_cb(self, *args, **kwargs):
"""
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-07-22 19:52 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-07-22 19:52 UTC (permalink / raw
To: gentoo-commits
commit: f587ebf3d4920b6c3cae3654918ba9cda6625a3e
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 22 19:32:38 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jul 22 19:37:33 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=f587ebf3
MergeProcess: handle RETURNCODE_POSTINST_FAILURE in _proc_join_done
Since ForkProcess now receives process exit status in the
_proc_join_done method instead of the _async_waitpid_cb method,
MergeProcess needs to handle RETURNCODE_POSTINST_FAILURE there
instead.
Fixes: 3561071e07ad ("MergeProcess: replace os.fork with multiprocessing.Process (bug 730192)")
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index 6924c8b0e..85deedc18 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -195,15 +195,14 @@ class MergeProcess(ForkProcess):
prev_mtimes=self.prev_mtimes, counter=counter)
return rval
- def _async_waitpid_cb(self, *args, **kwargs):
+ def _proc_join_done(self, proc, future):
"""
- Override _async_waitpid_cb to perform cleanup that is
- not necessarily idempotent.
+ Extend _proc_join_done to react to RETURNCODE_POSTINST_FAILURE.
"""
- ForkProcess._async_waitpid_cb(self, *args, **kwargs)
- if self.returncode == portage.const.RETURNCODE_POSTINST_FAILURE:
+ if proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE:
self.postinst_failure = True
self.returncode = os.EX_OK
+ super(MergeProcess, self)._proc_join_done(proc, future)
def _unregister(self):
"""
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-07-22 20:14 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-07-22 20:14 UTC (permalink / raw
To: gentoo-commits
commit: 309f2cc59e19eab05dfc2e86d1df4fc7ba1be93d
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 22 20:10:30 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jul 22 20:12:16 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=309f2cc5
MergeProcess: handle cancelled future in _proc_join_done
If the future was cancelled, then avoid a possible ValueError
when accessing proc.exitcode.
Fixes: f587ebf3d492 ("MergeProcess: handle RETURNCODE_POSTINST_FAILURE in _proc_join_done")
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index 85deedc18..e89b53555 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -199,7 +199,7 @@ class MergeProcess(ForkProcess):
"""
Extend _proc_join_done to react to RETURNCODE_POSTINST_FAILURE.
"""
- if proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE:
+ if not future.cancelled() and proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE:
self.postinst_failure = True
self.returncode = os.EX_OK
super(MergeProcess, self)._proc_join_done(proc, future)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-08-03 19:30 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-08-03 19:30 UTC (permalink / raw
To: gentoo-commits
commit: c14bbdf7a32eca5d559c012682b6101d91d71bf2
Author: Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 3 19:05:59 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug 3 19:22:24 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c14bbdf7
lib/portage/dbapi/bintree.py: drop unused-import
Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index e86fa5caa..97be51121 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -27,7 +27,7 @@ from portage.const import CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
from portage.dbapi.virtual import fakedbapi
from portage.dep import Atom, use_reduce, paren_enclose
from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
- ParseError, PermissionDenied, PortageException
+ ParseError, PortageException
from portage.localization import _
from portage.package.ebuild.profile_iuse import iter_iuse_vars
from portage.util.futures import asyncio
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-08-03 21:42 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-08-03 21:42 UTC (permalink / raw
To: gentoo-commits
commit: 2af196d23ef2472432f0d8f29de9e4149978f8ef
Author: Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 3 20:20:22 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug 3 21:15:55 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2af196d2
lib/portage/dbapi/cpv_expand.py: drop unused-import
* Drop unused import
* Update copyright
Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/cpv_expand.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/lib/portage/dbapi/cpv_expand.py b/lib/portage/dbapi/cpv_expand.py
index a1a91f554..eeec446d4 100644
--- a/lib/portage/dbapi/cpv_expand.py
+++ b/lib/portage/dbapi/cpv_expand.py
@@ -1,12 +1,10 @@
-# Copyright 2010-2013 Gentoo Foundation
+# Copyright 2010-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["cpv_expand"]
import portage
from portage.exception import AmbiguousPackageName
-from portage.localization import _
-from portage.util import writemsg
from portage.versions import _pkgsplit
def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-08-03 21:42 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-08-03 21:42 UTC (permalink / raw
To: gentoo-commits
commit: 3551cfb0be232df7f1a8d575181f8629ac7b4438
Author: Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 3 20:20:25 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug 3 21:25:52 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=3551cfb0
lib/portage/dbapi/IndexedPortdb.py: drop unused-import
* Drop unused-import
* Update copyright
Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/IndexedPortdb.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/IndexedPortdb.py b/lib/portage/dbapi/IndexedPortdb.py
index 5f1cb5bd1..8d9ba89ff 100644
--- a/lib/portage/dbapi/IndexedPortdb.py
+++ b/lib/portage/dbapi/IndexedPortdb.py
@@ -1,4 +1,4 @@
-# Copyright 2014 Gentoo Foundation
+# Copyright 2014-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -15,7 +15,6 @@ from portage.cache.index.IndexStreamIterator import IndexStreamIterator
from portage.cache.index.pkg_desc_index import \
pkg_desc_index_line_read, pkg_desc_index_node
from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy
-from portage.versions import _pkg_str
class IndexedPortdb:
"""
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-08-03 23:28 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-08-03 23:28 UTC (permalink / raw
To: gentoo-commits
commit: 8af9f6d49363a1faeaa29a4fe00426dd2a1fabb4
Author: Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 3 22:43:14 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug 3 23:28:02 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8af9f6d4
lib/portage/dbapi/bintree.py: fix whitespace
Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 97be51121..59c265688 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1686,7 +1686,7 @@ class binarytree:
resume = True
writemsg(_("Resuming download of this tbz2, but it is possible that it is corrupt.\n"),
noiselevel=-1)
-
+
mydest = os.path.dirname(self.getname(pkgname))
self._ensure_dir(mydest)
# urljoin doesn't work correctly with unrecognized protocols like sftp
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-08-03 23:28 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-08-03 23:28 UTC (permalink / raw
To: gentoo-commits
commit: ca6e92e3a08d0aad1dc551cfb1e380b843fc33b4
Author: Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 3 22:43:25 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug 3 23:28:04 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=ca6e92e3
lib/portage/dbapi/virtual.py: fix whitespace
Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/virtual.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/virtual.py b/lib/portage/dbapi/virtual.py
index f62fc2a30..4ed0dea1b 100644
--- a/lib/portage/dbapi/virtual.py
+++ b/lib/portage/dbapi/virtual.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2013 Gentoo Foundation
+# Copyright 1998-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.dbapi import dbapi
@@ -214,7 +214,7 @@ class fakedbapi(dbapi):
class testdbapi:
"""A dbapi instance with completely fake functions to get by hitting disk
- TODO(antarus):
+ TODO(antarus):
This class really needs to be rewritten to have better stubs; but these work for now.
The dbapi classes themselves need unit tests...and that will be a lot of work.
"""
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-08-04 3:16 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-08-04 3:16 UTC (permalink / raw
To: gentoo-commits
commit: f74cae3d377006af710f4b70cfbbf3391a126bb5
Author: Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Tue Aug 4 02:44:56 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Aug 4 03:13:01 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=f74cae3d
lib/portage/dbapi/vartree.py: fix reimported
Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index fcf164e82..fbf455363 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1619,7 +1619,6 @@ class dblink:
At present this is implemented as a text backend in /var/db/pkg.
"""
- import re
_normalize_needed = re.compile(r'//|^[^/]|./$|(^|/)\.\.?(/|$)')
_contents_re = re.compile(r'^(' + \
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-08-09 0:15 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-08-09 0:15 UTC (permalink / raw
To: gentoo-commits
commit: 656f8a7fcd2014c833e42282744c70a21e6c7e31
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 8 23:51:52 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Aug 8 23:55:19 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=656f8a7f
treewalk: reset config for unmerge (bug 711174)
When cloning config instances for unmerge, call the reset
method in order to ensure that there is no unintended leakage
of variables which should not be shared. This fixes leakage
of the PORTAGE_LOG_FILE variable, which triggered log
corruption for FEATURES=compress-build-logs.
Bug: https://bugs.gentoo.org/711174
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 5799d94f2..3eee025ad 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -4013,7 +4013,9 @@ class dblink:
# since we need it to have private ${T} etc... for things
# like elog.
settings_clone = portage.config(clone=self.settings)
- settings_clone.pop("PORTAGE_BUILDDIR_LOCKED", None)
+ # This reset ensures that there is no unintended leakage
+ # of variables which should not be shared.
+ settings_clone.reset()
settings_clone.setcpv(cur_cpv, mydb=self.vartree.dbapi)
if self._preserve_libs and "preserve-libs" in \
settings_clone["PORTAGE_RESTRICT"].split():
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2020-09-08 2:52 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2020-09-08 2:52 UTC (permalink / raw
To: gentoo-commits
commit: 6a3bdcc427c58112075c36cc0481c53215f12db4
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 8 02:43:23 2020 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 8 02:44:46 2020 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=6a3bdcc4
binarytree.move_ent: fix path comparison prior to _movefile
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index e4393e06d..7e24589e5 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -531,7 +531,7 @@ class binarytree:
new_path = self.getname(mynewcpv)
self._pkg_paths[
self.dbapi._instance_key(mynewcpv)] = new_path[len(self.pkgdir)+1:]
- if new_path != mytbz2:
+ if new_path != tbz2path:
self._ensure_dir(os.path.dirname(new_path))
_movefile(tbz2path, new_path, mysettings=self.settings)
self.inject(mynewcpv)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-01-17 8:49 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-01-17 8:49 UTC (permalink / raw
To: gentoo-commits
commit: bdc75b3aa217649f6d835b04eb3cac879459c0a0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 17 08:47:13 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jan 17 08:47:26 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=bdc75b3a
bindbapi.move_ent: remove redundant self._pkg_str calls
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7e24589e5..528a68979 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["bindbapi", "binarytree"]
@@ -27,7 +27,7 @@ from portage.cache.mappings import slot_dict_class
from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
from portage.dbapi.virtual import fakedbapi
from portage.dep import Atom, use_reduce, paren_enclose
-from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
+from portage.exception import AlarmSignal, InvalidPackageName, \
ParseError, PortageException
from portage.localization import _
from portage.package.ebuild.profile_iuse import iter_iuse_vars
@@ -466,11 +466,7 @@ class binarytree:
if not origmatches:
return moves
for mycpv in origmatches:
- try:
- mycpv = self.dbapi._pkg_str(mycpv, None)
- except (KeyError, InvalidData):
- continue
- mycpv_cp = portage.cpv_getkey(mycpv)
+ mycpv_cp = mycpv.cp
if mycpv_cp != origcp:
# Ignore PROVIDE virtual match.
continue
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-01-17 8:49 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-01-17 8:49 UTC (permalink / raw
To: gentoo-commits
commit: 375eb5c02df19833ddeb2c63438aac9faf3d5eae
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 17 08:35:41 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jan 17 08:46:23 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=375eb5c0
vardbapi.move_ent: remove redundant self._pkg_str calls
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index f3d74cf82..2c75be4a1 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = [
@@ -372,11 +372,7 @@ class vardbapi(dbapi):
if not origmatches:
return moves
for mycpv in origmatches:
- try:
- mycpv = self._pkg_str(mycpv, None)
- except (KeyError, InvalidData):
- continue
- mycpv_cp = cpv_getkey(mycpv)
+ mycpv_cp = mycpv.cp
if mycpv_cp != origcp:
# Ignore PROVIDE virtual match.
continue
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-01-17 13:31 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-01-17 13:31 UTC (permalink / raw
To: gentoo-commits
commit: 3d39e6891a3ee002fb1aa039c5660e6015c555d2
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 17 13:24:21 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jan 17 13:27:25 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=3d39e689
bindbapi.move_ent: use cpv_exists instead of getname
The getname method is complicated by binpkg-multi-instance
and soon BINPKG_FORMAT, so it's much simpler to use
cpv_exists.
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 528a68979..e5ba5893e 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -483,7 +483,7 @@ class binarytree:
myoldpkg = catsplit(mycpv)[1]
mynewpkg = catsplit(mynewcpv)[1]
- if (mynewpkg != myoldpkg) and os.path.exists(self.getname(mynewcpv)):
+ if (mynewpkg != myoldpkg) and self.dbapi.cpv_exists(mynewcpv):
writemsg(_("!!! Cannot update binary: Destination exists.\n"),
noiselevel=-1)
writemsg("!!! "+mycpv+" -> "+mynewcpv+"\n", noiselevel=-1)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-01-18 9:20 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-01-18 9:20 UTC (permalink / raw
To: gentoo-commits
commit: 7007dbcd1013829466498d7e0708c8b84fdd68bf
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 18 08:45:34 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jan 18 08:46:44 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=7007dbcd
bindbapi.aux_update: fix spurious package file renames
Omit the filename argument for the binarytree.inject method
in aux_update, since an in-place update is desired, and the
filename argument causes the file to be renamed when
binpkg-multi-instance is enabled.
Fixes: 328dd4712f88 ("binpkg-multi-instance 3 of 7")
Bug: https://bugs.gentoo.org/765847
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index e5ba5893e..180e48c3b 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -214,7 +214,7 @@ class bindbapi(fakedbapi):
del mydata[k]
mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
# inject will clear stale caches via cpv_inject.
- self.bintree.inject(cpv, filename=tbz2path)
+ self.bintree.inject(cpv)
@coroutine
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-02-23 21:31 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-02-23 21:31 UTC (permalink / raw
To: gentoo-commits
commit: a2c59fa326ecacf9834df574a81f4a5e5b7d6b93
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 23 21:29:57 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 23 21:30:19 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=a2c59fa3
binarytree.inject: create PKGDIR
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index ab09b42bc..429c8e64a 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1255,6 +1255,7 @@ class binarytree:
# process) and then updated it, all while holding a lock.
pkgindex_lock = None
try:
+ os.makedirs(self.pkgdir, exist_ok=True)
pkgindex_lock = lockfile(self._pkgindex_file,
wantnewlockfile=1)
if filename is not None:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-03-07 11:42 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-03-07 11:42 UTC (permalink / raw
To: gentoo-commits
commit: 6eafeb89fb4d17326d6ffa1d6ca83e5407890d42
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 7 09:03:33 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Mar 7 09:30:27 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=6eafeb89
dblink: add _format_contents_line method
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 42 +++++++++++++++++++++++++++++++++++++-----
1 file changed, 37 insertions(+), 5 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 2c75be4a1..826083eae 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5045,7 +5045,14 @@ class dblink:
% (relative_path, myabsto)])
showMessage("%s %s -> %s\n" % (zing, mydest, myto))
- outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n")
+ outfile.write(
+ self._format_contents_line(
+ node_type="sym",
+ abs_path=myrealdest,
+ symlink_target=myto,
+ mtime_ns=mymtime,
+ )
+ )
else:
showMessage(_("!!! Failed to move file.\n"),
level=logging.ERROR, noiselevel=-1)
@@ -5146,7 +5153,9 @@ class dblink:
except OSError:
pass
- outfile.write("dir "+myrealdest+"\n")
+ outfile.write(
+ self._format_contents_line(node_type="dir", abs_path=myrealdest)
+ )
# recurse and merge this directory
mergelist.extend(join(relative_path, child) for child in
os.listdir(join(srcroot, relative_path)))
@@ -5194,7 +5203,14 @@ class dblink:
pass
if mymtime != None:
- outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n")
+ outfile.write(
+ self._format_contents_line(
+ node_type="obj",
+ abs_path=myrealdest,
+ md5_digest=mymd5,
+ mtime_ns=mymtime,
+ )
+ )
showMessage("%s %s\n" % (zing,mydest))
else:
# we are merging a fifo or device node
@@ -5214,9 +5230,13 @@ class dblink:
else:
return 1
if stat.S_ISFIFO(mymode):
- outfile.write("fif %s\n" % myrealdest)
+ outfile.write(
+ self._format_contents_line(node_type="fif", abs_path=myrealdest)
+ )
else:
- outfile.write("dev %s\n" % myrealdest)
+ outfile.write(
+ self._format_contents_line(node_type="dev", abs_path=myrealdest)
+ )
showMessage(zing + " " + mydest + "\n")
def _protect(self, cfgfiledict, protect_if_modified, src_md5,
@@ -5278,6 +5298,18 @@ class dblink:
return dest, protected, move_me
+ def _format_contents_line(
+ self, node_type, abs_path, md5_digest=None, symlink_target=None, mtime_ns=None
+ ):
+ fields = [node_type, abs_path]
+ if md5_digest is not None:
+ fields.append(md5_digest)
+ elif symlink_target is not None:
+ fields.append("-> {}".format(symlink_target))
+ if mtime_ns is not None:
+ fields.append(str(mtime_ns // 1000000000))
+ return "{}\n".format(" ".join(fields))
+
def _merged_path(self, path, lstatobj, exists=True):
previous_path = self._device_path_map.get(lstatobj.st_dev)
if previous_path is None or previous_path is False or \
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-06-05 18:08 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-06-05 18:08 UTC (permalink / raw
To: gentoo-commits
commit: 2fe8238b540240a62de3513839974e033f0bacd2
Author: Daniel M. Weeks <dan <AT> danweeks <DOT> net>
AuthorDate: Mon May 31 16:10:21 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun 5 18:03:45 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2fe8238b
Support GLEP 75 in PORTAGE_RO_DISTDIRS size check
Closes: https://github.com/gentoo/portage/pull/724
Signed-off-by: Daniel M. Weeks <dan <AT> danweeks.net>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 99c36450c..0f50f1338 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -12,7 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.dbapi.dep_expand:dep_expand',
'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot',
'portage.package.ebuild.doebuild:doebuild',
- 'portage.package.ebuild.fetch:_download_suffix',
+ 'portage.package.ebuild.fetch:get_mirror_url,_download_suffix',
'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
'portage.util.listdir:listdir',
'portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str',
@@ -859,7 +859,11 @@ class portdbapi(dbapi):
if ro_distdirs is not None:
for x in shlex_split(ro_distdirs):
try:
- mystat = os.stat(os.path.join(x, myfile))
+ mystat = os.stat(
+ portage.package.ebuild.fetch.get_mirror_url(
+ x, myfile, self.settings
+ )
+ )
except OSError:
pass
else:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-09-21 5:51 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-09-21 5:51 UTC (permalink / raw
To: gentoo-commits
commit: bcda30d0a6fa4962cc7597dbed9b648cf6400ab5
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 21 05:21:55 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 21 05:33:51 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=bcda30d0
vardbapi: convert compat coroutine to async
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 7f3b5d773..8ffb23b1c 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -78,7 +78,6 @@ from portage import _os_merge
from portage import _selinux_merge
from portage import _unicode_decode
from portage import _unicode_encode
-from portage.util.futures.compat_coroutine import coroutine
from portage.util.futures.executor.fork import ForkExecutor
from ._VdbMetadataDelta import VdbMetadataDelta
@@ -1006,8 +1005,7 @@ class vardbapi(dbapi):
pass
self._bump_mtime(cpv)
- @coroutine
- def unpack_metadata(self, pkg, dest_dir, loop=None):
+ async def unpack_metadata(self, pkg, dest_dir, loop=None):
"""
Unpack package metadata to a directory. This method is a coroutine.
@@ -1029,10 +1027,9 @@ class vardbapi(dbapi):
shutil.copy(os.path.join(parent, key), os.path.join(dest_dir, key))
break
- yield loop.run_in_executor(ForkExecutor(loop=loop), async_copy)
+ await loop.run_in_executor(ForkExecutor(loop=loop), async_copy)
- @coroutine
- def unpack_contents(
+ async def unpack_contents(
self,
pkg,
dest_dir,
@@ -1097,10 +1094,10 @@ class vardbapi(dbapi):
tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", dest_dir)
pr, pw = os.pipe()
- proc = yield asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
+ proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
os.close(pr)
with os.fdopen(pw, "wb", 0) as pw_file:
- excluded_config_files = yield loop.run_in_executor(
+ excluded_config_files = await loop.run_in_executor(
ForkExecutor(loop=loop),
functools.partial(
self._dblink(cpv).quickpkg,
@@ -1109,7 +1106,7 @@ class vardbapi(dbapi):
include_unmodified_config=opts.include_unmodified_config == "y",
),
)
- yield proc.wait()
+ await proc.wait()
if proc.returncode != os.EX_OK:
raise PortageException("command failed: {}".format(tar_cmd))
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-09-21 5:51 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-09-21 5:51 UTC (permalink / raw
To: gentoo-commits
commit: 7957a7f4c2ef4656f6f9a29d2c9bab6e44daae1f
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 21 05:08:21 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 21 05:33:51 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=7957a7f4
bindbapi: convert compat coroutine to async
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 024a9b5f6..777fc4918 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -39,7 +39,6 @@ from portage.localization import _
from portage.package.ebuild.profile_iuse import iter_iuse_vars
from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import coroutine
from portage.util.futures.executor.fork import ForkExecutor
from portage import _movefile
from portage import os
@@ -253,8 +252,7 @@ class bindbapi(fakedbapi):
# inject will clear stale caches via cpv_inject.
self.bintree.inject(cpv)
- @coroutine
- def unpack_metadata(self, pkg, dest_dir, loop=None):
+ async def unpack_metadata(self, pkg, dest_dir, loop=None):
"""
Unpack package metadata to a directory. This method is a coroutine.
@@ -271,17 +269,16 @@ class bindbapi(fakedbapi):
key = self._instance_key(cpv)
add_pkg = self.bintree._additional_pkgs.get(key)
if add_pkg is not None:
- yield add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop)
+ await add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop)
else:
tbz2_file = self.bintree.getname(cpv)
- yield loop.run_in_executor(
+ await loop.run_in_executor(
ForkExecutor(loop=loop),
portage.xpak.tbz2(tbz2_file).unpackinfo,
dest_dir,
)
- @coroutine
- def unpack_contents(self, pkg, dest_dir, loop=None):
+ async def unpack_contents(self, pkg, dest_dir, loop=None):
"""
Unpack package contents to a directory. This method is a coroutine.
@@ -313,7 +310,7 @@ class bindbapi(fakedbapi):
)
extractor.start()
- yield extractor.async_wait()
+ await extractor.async_wait()
if extractor.returncode != os.EX_OK:
raise PortageException("Error Extracting '{}'".format(pkg_path))
@@ -322,7 +319,7 @@ class bindbapi(fakedbapi):
add_pkg = self.bintree._additional_pkgs.get(instance_key)
if add_pkg is None:
raise portage.exception.PackageNotFound(cpv)
- yield add_pkg._db.unpack_contents(pkg, dest_dir, loop=loop)
+ await add_pkg._db.unpack_contents(pkg, dest_dir, loop=loop)
def cp_list(self, *pargs, **kwargs):
if not self.bintree.populated:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-09-21 5:51 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2021-09-21 5:51 UTC (permalink / raw
To: gentoo-commits
commit: 95935004862d1363a0e3d5b6ee33d4ed65566a2d
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 21 05:13:12 2021 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep 21 05:33:51 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=95935004
portdbapi: convert compat coroutine to async
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 72b875879..93f3fee2f 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -38,7 +38,6 @@ from portage import os
from portage import _encodings
from portage import _unicode_encode
from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import coroutine, coroutine_return
from portage.util.futures.iter_completed import iter_gather
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
@@ -1244,8 +1243,7 @@ class portdbapi(dbapi):
loop = self._event_loop
return loop.run_until_complete(self.async_xmatch(level, origdep, loop=loop))
- @coroutine
- def async_xmatch(self, level, origdep, loop=None):
+ async def async_xmatch(self, level, origdep, loop=None):
"""
Asynchronous form of xmatch.
@@ -1269,7 +1267,7 @@ class portdbapi(dbapi):
if self.frozen:
cache_key = (mydep, mydep.unevaluated_atom)
try:
- coroutine_return(self.xcache[level][cache_key][:])
+ return self.xcache[level][cache_key][:]
except KeyError:
pass
@@ -1336,7 +1334,7 @@ class portdbapi(dbapi):
zip(
aux_keys,
(
- yield self.async_aux_get(
+ await self.async_aux_get(
cpv, aux_keys, myrepo=cpv.repo, loop=loop
)
),
@@ -1384,7 +1382,7 @@ class portdbapi(dbapi):
if not isinstance(myval, _pkg_str):
myval = myval[:]
- coroutine_return(myval)
+ return myval
def match(self, mydep, use_cache=1):
return self.xmatch("match-visible", mydep)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2021-11-26 21:09 Mike Gilbert
0 siblings, 0 replies; 71+ messages in thread
From: Mike Gilbert @ 2021-11-26 21:09 UTC (permalink / raw
To: gentoo-commits
commit: 0009404ee5f293fe7076d59a52c46827c3b77738
Author: Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Fri Nov 26 17:43:45 2021 +0000
Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Fri Nov 26 17:43:45 2021 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=0009404e
dbapi: fix logic in bintree.binarytree._parse_build_id()
Resolves an error when improperly named xpak files exist in PKGDIR.
Bug: https://bugs.gentoo.org/818886
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 777fc4918..9dbf9ee8b 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1877,11 +1877,10 @@ class binarytree:
suffixlen = len(".xpak")
hyphen = filename.rfind("-", 0, -(suffixlen + 1))
if hyphen != -1:
- build_id = filename[hyphen + 1 : -suffixlen]
- try:
- build_id = int(build_id)
- except ValueError:
- pass
+ try:
+ build_id = int(filename[hyphen + 1 : -suffixlen])
+ except ValueError:
+ pass
return build_id
def isremote(self, pkgname):
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-04-13 15:34 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2022-04-13 15:34 UTC (permalink / raw
To: gentoo-commits
commit: 05c0b3d03393fe376f3ae4b49bce403d496e3e68
Author: Mike Frysinger <vapier <AT> chromium <DOT> org>
AuthorDate: Tue Mar 29 07:12:25 2022 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Apr 13 15:34:30 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=05c0b3d0
vartree: avoid lock contention when there are no blockers
No sense in grabbing the vdb lock if we aren't going to do any work.
This avoids contention on the global lock with parallel packages.
[sam: cherry-picked from chromiumos' third_party/portage_tool repo]
(cherry picked from commit ea5f6f8c0a5e05d7630f9070992a89fa6907cc14)
Signed-off-by: Sam James <sam <AT> gentoo.org>
Closes: https://github.com/gentoo/portage/pull/813
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index e7252790d..602913862 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5148,14 +5148,17 @@ class dblink:
self._clear_contents_cache()
contents = self.getcontents()
destroot_len = len(destroot) - 1
- self.lockdb()
- try:
- for blocker in blockers:
- self.vartree.dbapi.removeFromContents(
- blocker, iter(contents), relative_paths=False
- )
- finally:
- self.unlockdb()
+
+ # Avoid lock contention if we aren't going to do any work.
+ if blockers:
+ self.lockdb()
+ try:
+ for blocker in blockers:
+ self.vartree.dbapi.removeFromContents(
+ blocker, iter(contents), relative_paths=False
+ )
+ finally:
+ self.unlockdb()
plib_registry = self.vartree.dbapi._plib_registry
if plib_registry:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-04-13 15:34 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2022-04-13 15:34 UTC (permalink / raw
To: gentoo-commits
commit: 964c0e16172b76a8ebdd737cd8919870ae2b5f96
Author: Mike Frysinger <vapier <AT> chromium <DOT> org>
AuthorDate: Tue Mar 29 13:56:55 2022 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Apr 13 15:34:30 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=964c0e16
vartree: skip env-update if no updates were merged
This speeds up virtual/ installs by not constantly re-running env-update.
[sam: cherry-picked from chromiumos' third_party/portage_tool repo]
(cherry picked from commit 87ac3566ebb7155a57876d345849bd0fd6878c0e)
Bug: https://bugs.gentoo.org/836375
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 38 +++++++++++++++++++++-----------------
1 file changed, 21 insertions(+), 17 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 072bc1506..e7252790d 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -2608,14 +2608,16 @@ class dblink:
else:
self.settings.pop("PORTAGE_LOG_FILE", None)
- env_update(
- target_root=self.settings["ROOT"],
- prev_mtimes=ldpath_mtimes,
- contents=contents,
- env=self.settings,
- writemsg_level=self._display_merge,
- vardbapi=self.vartree.dbapi,
- )
+ # If we didn't unmerge anything, don't bother updating env.
+ if contents:
+ env_update(
+ target_root=self.settings["ROOT"],
+ prev_mtimes=ldpath_mtimes,
+ contents=contents,
+ env=self.settings,
+ writemsg_level=self._display_merge,
+ vardbapi=self.vartree.dbapi,
+ )
unmerge_with_replacement = preserve_paths is not None
if not unmerge_with_replacement:
@@ -5258,15 +5260,17 @@ class dblink:
],
)
- # update environment settings, library paths. DO NOT change symlinks.
- env_update(
- target_root=self.settings["ROOT"],
- prev_mtimes=prev_mtimes,
- contents=contents,
- env=self.settings,
- writemsg_level=self._display_merge,
- vardbapi=self.vartree.dbapi,
- )
+ # Update environment settings, library paths. DO NOT change symlinks.
+ # Only do this if we actually installed something.
+ if contents:
+ env_update(
+ target_root=self.settings["ROOT"],
+ prev_mtimes=prev_mtimes,
+ contents=contents,
+ env=self.settings,
+ writemsg_level=self._display_merge,
+ vardbapi=self.vartree.dbapi,
+ )
# For gcc upgrades, preserved libs have to be removed after the
# the library path has been updated.
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-04-20 20:24 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2022-04-20 20:24 UTC (permalink / raw
To: gentoo-commits
commit: e893f4fc12eb618318b1945ce7a05a94fb1ea1b4
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 17 19:12:40 2022 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 18 02:18:31 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=e893f4fc
MergeProcess: propagate mtimedb["ldpath"] to parent process (bug 836375)
Use an instance of multiprocessing.Pipe to propagate mtimedb["ldpath"]
from the MergeProcess child process to the parent process. This fixes
env_update calls to avoid unnecessary regeneration of ld.so.cache in
cases where mtimedb["ldpath"] has not changed since the last call to
env_update.
Bug: https://bugs.gentoo.org/836375
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 22 ++++++++++++++++++++++
lib/portage/dbapi/vartree.py | 10 ++++++++++
2 files changed, 32 insertions(+)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index db3f3b105..197c48a7e 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
import io
+import multiprocessing
import platform
import fcntl
@@ -38,6 +39,7 @@ class MergeProcess(ForkProcess):
"_dblink",
"_elog_keys",
"_locked_vdb",
+ "_mtime_reader",
)
def _start(self):
@@ -113,6 +115,19 @@ class MergeProcess(ForkProcess):
self._elog_reader_fd = None
return False
+ def _mtime_handler(self):
+ if self._mtime_reader is not None:
+ try:
+ mtimes = self._mtime_reader.recv()
+ except EOFError:
+ self.scheduler.remove_reader(self._mtime_reader.fileno())
+ self._mtime_reader.close()
+ self._mtime_reader = None
+ else:
+ if self.prev_mtimes is not None:
+ self.prev_mtimes.clear()
+ self.prev_mtimes.update(mtimes)
+
def _spawn(self, args, fd_pipes, **kwargs):
"""
Extend the superclass _spawn method to perform some pre-fork and
@@ -127,6 +142,11 @@ class MergeProcess(ForkProcess):
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
)
+ mtime_reader, mtime_writer = multiprocessing.Pipe(duplex=False)
+ fd_pipes[mtime_writer.fileno()] = mtime_writer.fileno()
+ self.scheduler.add_reader(mtime_reader.fileno(), self._mtime_handler)
+ self._mtime_reader = mtime_reader
+
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
@@ -142,6 +162,7 @@ class MergeProcess(ForkProcess):
vartree=self.vartree,
blockers=blockers,
pipe=elog_writer_fd,
+ mtime_pipe=mtime_writer,
)
fd_pipes[elog_writer_fd] = elog_writer_fd
self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler)
@@ -160,6 +181,7 @@ class MergeProcess(ForkProcess):
self._elog_reader_fd = elog_reader_fd
pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs)
os.close(elog_writer_fd)
+ mtime_writer.close()
self._buf = ""
self._elog_keys = set()
# Discard messages which will be collected by the subprocess,
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 602913862..a95d60691 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1806,6 +1806,7 @@ class dblink:
blockers=None,
scheduler=None,
pipe=None,
+ mtime_pipe=None,
):
"""
Creates a DBlink object for a given CPV.
@@ -1862,6 +1863,7 @@ class dblink:
self._device_path_map = {}
self._hardlink_merge_map = {}
self._hash_key = (self._eroot, self.mycpv)
+ self._mtime_pipe = mtime_pipe
self._protect_obj = None
self._pipe = pipe
self._postinst_failure = False
@@ -2618,6 +2620,7 @@ class dblink:
writemsg_level=self._display_merge,
vardbapi=self.vartree.dbapi,
)
+ self._send_mtimes(ldpath_mtimes)
unmerge_with_replacement = preserve_paths is not None
if not unmerge_with_replacement:
@@ -4243,6 +4246,12 @@ class dblink:
def _emerge_log(self, msg):
emergelog(False, msg)
+ def _send_mtimes(self, mtimes):
+ if self._mtime_pipe is None:
+ return
+
+ self._mtime_pipe.send(mtimes)
+
def treewalk(
self,
srcroot,
@@ -5274,6 +5283,7 @@ class dblink:
writemsg_level=self._display_merge,
vardbapi=self.vartree.dbapi,
)
+ self._send_mtimes(prev_mtimes)
# For gcc upgrades, preserved libs have to be removed after the
# the library path has been updated.
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-08-18 19:00 Mike Gilbert
0 siblings, 0 replies; 71+ messages in thread
From: Mike Gilbert @ 2022-08-18 19:00 UTC (permalink / raw
To: gentoo-commits
commit: 57ce385e32e79b9d332fe1fdb3be50bdb07e7838
Author: Mike Gilbert <floppym <AT> gentoo <DOT> org>
AuthorDate: Thu Aug 18 17:59:03 2022 +0000
Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Thu Aug 18 18:11:25 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=57ce385e
dbapi: avoid iterating porttrees twice in _set_porttrees()
If porttrees is a generator object, the second pass will fail.
Bug: https://bugs.gentoo.org/865635
Fixes: 9e24d0143450628f334cdb62e579efafd1bfd2ba
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index cd919ba31..126e7161b 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -367,14 +367,14 @@ class portdbapi(dbapi):
repo priority
@type porttrees: list
"""
+ self._porttrees = tuple(porttrees)
self._porttrees_repos = portage.OrderedDict(
(repo.name, repo)
for repo in (
self.repositories.get_repo_for_location(location)
- for location in porttrees
+ for location in self._porttrees
)
)
- self._porttrees = tuple(porttrees)
def _get_porttrees(self):
return self._porttrees
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-09-09 10:16 Michał Górny
0 siblings, 0 replies; 71+ messages in thread
From: Michał Górny @ 2022-09-09 10:16 UTC (permalink / raw
To: gentoo-commits
commit: 52411290c67535d94c7b20fa996ae7108014adfb
Author: Sheng Yu <syu.os <AT> protonmail <DOT> com>
AuthorDate: Fri Aug 19 20:24:59 2022 +0000
Commit: Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Fri Sep 9 10:15:57 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=52411290
Detect binary package format if not in database
Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com>
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index b441fff9a..0857ff21a 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1039,6 +1039,12 @@ class binarytree:
if match:
binpkg_format = match.get("BINPKG_FORMAT", None)
+ if not binpkg_format:
+ if myfile.endswith(SUPPORTED_XPAK_EXTENSIONS):
+ binpkg_format = "xpak"
+ elif myfile.endswith(SUPPORTED_GPKG_EXTENSIONS):
+ binpkg_format = "gpkg"
+
if gpkg_only:
if binpkg_format != "gpkg":
if not gpkg_only_warned:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-09-25 19:12 Mike Gilbert
0 siblings, 0 replies; 71+ messages in thread
From: Mike Gilbert @ 2022-09-25 19:12 UTC (permalink / raw
To: gentoo-commits
commit: a83507be7ce04d3ac421f9cbe8b63816809b0f4e
Author: David Palao <david.palao <AT> gmail <DOT> com>
AuthorDate: Fri Sep 23 13:42:33 2022 +0000
Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Sun Sep 25 19:07:52 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=a83507be
chore(bintree): removed useless ``if True:``
This commit de-indents a large block.
Signed-off-by: David Palao <david.palao <AT> gmail.com>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 263 +++++++++++++++++++++----------------------
1 file changed, 131 insertions(+), 132 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7f5dc051c..28b3c481b 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -473,143 +473,142 @@ class binarytree:
stacklevel=2,
)
- if True:
- self.pkgdir = normalize_path(pkgdir)
- # NOTE: Event if binpkg-multi-instance is disabled, it's
- # still possible to access a PKGDIR which uses the
- # binpkg-multi-instance layout (or mixed layout).
- self._multi_instance = "binpkg-multi-instance" in settings.features
- if self._multi_instance:
- self._allocate_filename = self._allocate_filename_multi
- self.dbapi = bindbapi(self, settings=settings)
- self.update_ents = self.dbapi.update_ents
- self.move_slot_ent = self.dbapi.move_slot_ent
- self.populated = 0
- self.tree = {}
- self._binrepos_conf = None
- self._remote_has_index = False
- self._remotepkgs = None # remote metadata indexed by cpv
- self._additional_pkgs = {}
- self.invalids = []
- self.settings = settings
- self._pkg_paths = {}
- self._populating = False
- self._all_directory = os.path.isdir(os.path.join(self.pkgdir, "All"))
- self._pkgindex_version = 0
- self._pkgindex_hashes = ["MD5", "SHA1"]
- self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
- self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
- self._pkgindex_keys.update(["CPV", "SIZE"])
- self._pkgindex_aux_keys = [
- "BASE_URI",
- "BDEPEND",
+ self.pkgdir = normalize_path(pkgdir)
+ # NOTE: Event if binpkg-multi-instance is disabled, it's
+ # still possible to access a PKGDIR which uses the
+ # binpkg-multi-instance layout (or mixed layout).
+ self._multi_instance = "binpkg-multi-instance" in settings.features
+ if self._multi_instance:
+ self._allocate_filename = self._allocate_filename_multi
+ self.dbapi = bindbapi(self, settings=settings)
+ self.update_ents = self.dbapi.update_ents
+ self.move_slot_ent = self.dbapi.move_slot_ent
+ self.populated = 0
+ self.tree = {}
+ self._binrepos_conf = None
+ self._remote_has_index = False
+ self._remotepkgs = None # remote metadata indexed by cpv
+ self._additional_pkgs = {}
+ self.invalids = []
+ self.settings = settings
+ self._pkg_paths = {}
+ self._populating = False
+ self._all_directory = os.path.isdir(os.path.join(self.pkgdir, "All"))
+ self._pkgindex_version = 0
+ self._pkgindex_hashes = ["MD5", "SHA1"]
+ self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
+ self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
+ self._pkgindex_keys.update(["CPV", "SIZE"])
+ self._pkgindex_aux_keys = [
+ "BASE_URI",
+ "BDEPEND",
+ "BINPKG_FORMAT",
+ "BUILD_ID",
+ "BUILD_TIME",
+ "CHOST",
+ "DEFINED_PHASES",
+ "DEPEND",
+ "DESCRIPTION",
+ "EAPI",
+ "FETCHCOMMAND",
+ "IDEPEND",
+ "IUSE",
+ "KEYWORDS",
+ "LICENSE",
+ "PDEPEND",
+ "PKGINDEX_URI",
+ "PROPERTIES",
+ "PROVIDES",
+ "RDEPEND",
+ "repository",
+ "REQUIRES",
+ "RESTRICT",
+ "RESUMECOMMAND",
+ "SIZE",
+ "SLOT",
+ "USE",
+ ]
+ self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
+ self._pkgindex_use_evaluated_keys = (
+ "BDEPEND",
+ "DEPEND",
+ "IDEPEND",
+ "LICENSE",
+ "RDEPEND",
+ "PDEPEND",
+ "PROPERTIES",
+ "RESTRICT",
+ )
+ self._pkgindex_header = None
+ self._pkgindex_header_keys = set(
+ [
+ "ACCEPT_KEYWORDS",
+ "ACCEPT_LICENSE",
+ "ACCEPT_PROPERTIES",
+ "ACCEPT_RESTRICT",
"BINPKG_FORMAT",
- "BUILD_ID",
- "BUILD_TIME",
- "CHOST",
- "DEFINED_PHASES",
- "DEPEND",
- "DESCRIPTION",
- "EAPI",
- "FETCHCOMMAND",
- "IDEPEND",
- "IUSE",
- "KEYWORDS",
- "LICENSE",
- "PDEPEND",
- "PKGINDEX_URI",
- "PROPERTIES",
- "PROVIDES",
- "RDEPEND",
- "repository",
- "REQUIRES",
- "RESTRICT",
- "RESUMECOMMAND",
- "SIZE",
- "SLOT",
+ "CBUILD",
+ "CONFIG_PROTECT",
+ "CONFIG_PROTECT_MASK",
+ "FEATURES",
+ "GENTOO_MIRRORS",
+ "INSTALL_MASK",
+ "IUSE_IMPLICIT",
"USE",
+ "USE_EXPAND",
+ "USE_EXPAND_HIDDEN",
+ "USE_EXPAND_IMPLICIT",
+ "USE_EXPAND_UNPREFIXED",
]
- self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
- self._pkgindex_use_evaluated_keys = (
- "BDEPEND",
- "DEPEND",
- "IDEPEND",
- "LICENSE",
- "RDEPEND",
- "PDEPEND",
- "PROPERTIES",
- "RESTRICT",
- )
- self._pkgindex_header = None
- self._pkgindex_header_keys = set(
- [
- "ACCEPT_KEYWORDS",
- "ACCEPT_LICENSE",
- "ACCEPT_PROPERTIES",
- "ACCEPT_RESTRICT",
- "BINPKG_FORMAT",
- "CBUILD",
- "CONFIG_PROTECT",
- "CONFIG_PROTECT_MASK",
- "FEATURES",
- "GENTOO_MIRRORS",
- "INSTALL_MASK",
- "IUSE_IMPLICIT",
- "USE",
- "USE_EXPAND",
- "USE_EXPAND_HIDDEN",
- "USE_EXPAND_IMPLICIT",
- "USE_EXPAND_UNPREFIXED",
- ]
- )
- self._pkgindex_default_pkg_data = {
- "BDEPEND": "",
- "BUILD_ID": "",
- "BUILD_TIME": "",
- "DEFINED_PHASES": "",
- "DEPEND": "",
- "EAPI": "0",
- "IDEPEND": "",
- "IUSE": "",
- "KEYWORDS": "",
- "LICENSE": "",
- "PATH": "",
- "PDEPEND": "",
- "PROPERTIES": "",
- "PROVIDES": "",
- "RDEPEND": "",
- "REQUIRES": "",
- "RESTRICT": "",
- "SLOT": "0",
- "USE": "",
- }
- self._pkgindex_inherited_keys = ["BINPKG_FORMAT", "CHOST", "repository"]
-
- # Populate the header with appropriate defaults.
- self._pkgindex_default_header_data = {
- "BINPKG_FORMAT": self.settings.get(
- "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
- ),
- "CHOST": self.settings.get("CHOST", ""),
- "repository": "",
- }
-
- self._pkgindex_translated_keys = (
- ("DESCRIPTION", "DESC"),
- ("_mtime_", "MTIME"),
- ("repository", "REPO"),
- )
+ )
+ self._pkgindex_default_pkg_data = {
+ "BDEPEND": "",
+ "BUILD_ID": "",
+ "BUILD_TIME": "",
+ "DEFINED_PHASES": "",
+ "DEPEND": "",
+ "EAPI": "0",
+ "IDEPEND": "",
+ "IUSE": "",
+ "KEYWORDS": "",
+ "LICENSE": "",
+ "PATH": "",
+ "PDEPEND": "",
+ "PROPERTIES": "",
+ "PROVIDES": "",
+ "RDEPEND": "",
+ "REQUIRES": "",
+ "RESTRICT": "",
+ "SLOT": "0",
+ "USE": "",
+ }
+ self._pkgindex_inherited_keys = ["BINPKG_FORMAT", "CHOST", "repository"]
+
+ # Populate the header with appropriate defaults.
+ self._pkgindex_default_header_data = {
+ "BINPKG_FORMAT": self.settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ ),
+ "CHOST": self.settings.get("CHOST", ""),
+ "repository": "",
+ }
+
+ self._pkgindex_translated_keys = (
+ ("DESCRIPTION", "DESC"),
+ ("_mtime_", "MTIME"),
+ ("repository", "REPO"),
+ )
- self._pkgindex_allowed_pkg_keys = set(
- chain(
- self._pkgindex_keys,
- self._pkgindex_aux_keys,
- self._pkgindex_hashes,
- self._pkgindex_default_pkg_data,
- self._pkgindex_inherited_keys,
- chain(*self._pkgindex_translated_keys),
- )
+ self._pkgindex_allowed_pkg_keys = set(
+ chain(
+ self._pkgindex_keys,
+ self._pkgindex_aux_keys,
+ self._pkgindex_hashes,
+ self._pkgindex_default_pkg_data,
+ self._pkgindex_inherited_keys,
+ chain(*self._pkgindex_translated_keys),
)
+ )
@property
def root(self):
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-11-08 23:07 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2022-11-08 23:07 UTC (permalink / raw
To: gentoo-commits
commit: a7dd39c1ae4a5ea4e3252ae8129fbd671c95d5f7
Author: Sheng Yu <syu.os <AT> protonmail <DOT> com>
AuthorDate: Tue Nov 8 22:52:46 2022 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Nov 8 23:07:46 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7dd39c1
Do not drop default PATH in the package index (avoid Packages regeneration)
The new Packages index (introduced in 445f10f4214c673f8fe0a9cc518c12767be4f159)
needs PATH but it got dropped later on when processing, so we would try
to rebuild the index on every emerge call.
This fixes regenerating Packages loop.
(Note that this didn't affect a released version.)
Fixes: 445f10f4214c673f8fe0a9cc518c12767be4f159
Bug: https://bugs.gentoo.org/877357
Bug: https://bugs.gentoo.org/877419
Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/934
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 21 +++------------------
1 file changed, 3 insertions(+), 18 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 771abedd5..d7c18e2e9 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1000,20 +1000,9 @@ class binarytree:
pkg_paths[instance_key] = mypath
# update the path if the package has been moved
oldpath = d.get("PATH")
- if oldpath and oldpath != mypath:
- update_pkgindex = True
- # Omit PATH if it is the default path for
- # the current Packages format version.
- if (mypath != mycpv + ".tbz2") and (
- mypath != mycpv + ".gpkg.tar"
- ):
+ if oldpath != mypath:
d["PATH"] = mypath
- if not oldpath:
- update_pkgindex = True
- else:
- d.pop("PATH", None)
- if oldpath:
- update_pkgindex = True
+ update_pkgindex = True
self.dbapi.cpv_inject(mycpv)
continue
if not os.access(full_path, os.R_OK):
@@ -1218,11 +1207,7 @@ class binarytree:
self.dbapi.cpv_remove(mycpv)
del pkg_paths[_instance_key(mycpv)]
- # record location if it's non-default
- if (mypath != mycpv + ".tbz2") and (mypath != mycpv + ".gpkg.tar"):
- d["PATH"] = mypath
- else:
- d.pop("PATH", None)
+ d["PATH"] = mypath
metadata[_instance_key(mycpv)] = d
if reindex:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-12-21 1:28 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2022-12-21 1:28 UTC (permalink / raw
To: gentoo-commits
commit: f05140beb2dd9a3577ff2042941cb3e4fbb1df31
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 17 05:15:27 2022 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Dec 21 01:28:02 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=f05140be
porttree: skip metadata verification if repository is immutable (not volatile)
With Python 3.11, I get:
- ~232695 total syscalls before (38268 read calls)
- ~203656 total syscalls after (27381 read calls)
It's a cheap improvement for simply promising to not touch
ebuilds in a repository.
Bug: https://bugs.gentoo.org/528394
See: a72a01746638debe472496bd8fc661992a6ba08b
See: 5c7bf4eb09f644813a6f017ffd91665664142560
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 15e1fd6ff..448a7f300 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -614,7 +614,8 @@ class portdbapi(dbapi):
if ro_auxdb is not None:
auxdbs.append(ro_auxdb)
auxdbs.append(self.auxdb[repo_path])
- eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db
+ repo = self.repositories.get_repo_for_location(repo_path)
+ eclass_db = repo.eclass_db
for auxdb in auxdbs:
try:
@@ -637,7 +638,9 @@ class portdbapi(dbapi):
# EAPI from _parse_eapi_ebuild_head, we disregard cache entries
# for unsupported EAPIs.
continue
- if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
+ if not repo.volatile or auxdb.validate_entry(
+ metadata, ebuild_hash, eclass_db
+ ):
break
else:
metadata = None
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2022-12-21 1:30 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2022-12-21 1:30 UTC (permalink / raw
To: gentoo-commits
commit: 2f020ae6aca04a0db68b7ceb5413e2b815d2169d
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Wed Dec 21 01:30:22 2022 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Wed Dec 21 01:30:55 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2f020ae6
Revert "porttree: skip metadata verification if repository is immutable (not volatile)"
This reverts commit f05140beb2dd9a3577ff2042941cb3e4fbb1df31.
I didn't mean to push this one yet, which we're going to handle
separately.
Bug: https://bugs.gentoo.org/528394
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/porttree.py | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 448a7f300..15e1fd6ff 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -614,8 +614,7 @@ class portdbapi(dbapi):
if ro_auxdb is not None:
auxdbs.append(ro_auxdb)
auxdbs.append(self.auxdb[repo_path])
- repo = self.repositories.get_repo_for_location(repo_path)
- eclass_db = repo.eclass_db
+ eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db
for auxdb in auxdbs:
try:
@@ -638,9 +637,7 @@ class portdbapi(dbapi):
# EAPI from _parse_eapi_ebuild_head, we disregard cache entries
# for unsupported EAPIs.
continue
- if not repo.volatile or auxdb.validate_entry(
- metadata, ebuild_hash, eclass_db
- ):
+ if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
break
else:
metadata = None
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-05-23 0:26 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-05-23 0:26 UTC (permalink / raw
To: gentoo-commits
commit: 35b5e4d71ebb5c7408dba7dc27cff0c22cad1562
Author: gcarq <egger.m <AT> protonmail <DOT> com>
AuthorDate: Mon Mar 27 13:31:44 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue May 23 00:22:09 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=35b5e4d7
mergeme: Rely on mydmode instead of calling os.path.exists again
Signed-off-by: gcarq <egger.m <AT> protonmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 317cf327a..676d4aa05 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5544,6 +5544,8 @@ class dblink:
destmd5,
mydest_link,
)
+ if protected and moveme:
+ mydmode = None
zing = "!!!"
if not moveme:
@@ -5584,6 +5586,7 @@ class dblink:
msg.append("")
self._eerror("preinst", msg)
mydest = newdest
+ mydmode = None
# if secondhand is None it means we're operating in "force" mode and should not create a second hand.
if (secondhand is not None) and (not os.path.exists(myrealto)):
@@ -5797,6 +5800,7 @@ class dblink:
msg.append("")
self._eerror("preinst", msg)
mydest = newdest
+ mydmode = None
# whether config protection or not, we merge the new file the
# same way. Unless moveme=0 (blocking directory)
@@ -6260,10 +6264,7 @@ class dblink:
Takes file mode and extended attributes into account.
Should only be used for regular files.
"""
- if not os.path.exists(mydest):
- return True
-
- if mymode != mydmode:
+ if mydmode is None or not stat.S_ISREG(mydmode) or mymode != mydmode:
return True
excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-05-23 0:26 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-05-23 0:26 UTC (permalink / raw
To: gentoo-commits
commit: a87be47f7d3245050da43d7c3ab4760d47e9fac5
Author: gcarq <egger.m <AT> protonmail <DOT> com>
AuthorDate: Tue Feb 21 00:04:26 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue May 23 00:22:09 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=a87be47f
mergeme: Don't overwrite files if the content matches
Uses filecmp.cmp(shallow=False) to compare file contents and
doesn't replace them if they are equal. This results in less disk
churn and helps to keep filesystem snapshots as small as possible.
Closes: https://bugs.gentoo.org/722270
Signed-off-by: gcarq <egger.m <AT> protonmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 47 +++++++++++++++++++++++++-------------------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index a9e332a74..327b72bed 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3,6 +3,8 @@
__all__ = ["vardbapi", "vartree", "dblink"] + ["write_contents", "tar_contents"]
+import filecmp
+
import portage
portage.proxy.lazyimport.lazyimport(
@@ -5800,28 +5802,33 @@ class dblink:
# whether config protection or not, we merge the new file the
# same way. Unless moveme=0 (blocking directory)
if moveme:
- # Create hardlinks only for source files that already exist
- # as hardlinks (having identical st_dev and st_ino).
- hardlink_key = (mystat.st_dev, mystat.st_ino)
+ # only replace the existing file if it differs, see #722270
+ already_merged = os.path.exists(mydest)
+ if already_merged and filecmp.cmp(mysrc, mydest, shallow=False):
+ zing = "==="
+ else:
+ # Create hardlinks only for source files that already exist
+ # as hardlinks (having identical st_dev and st_ino).
+ hardlink_key = (mystat.st_dev, mystat.st_ino)
- hardlink_candidates = self._hardlink_merge_map.get(hardlink_key)
- if hardlink_candidates is None:
- hardlink_candidates = []
- self._hardlink_merge_map[hardlink_key] = hardlink_candidates
+ hardlink_candidates = self._hardlink_merge_map.get(hardlink_key)
+ if hardlink_candidates is None:
+ hardlink_candidates = []
+ self._hardlink_merge_map[hardlink_key] = hardlink_candidates
- mymtime = movefile(
- mysrc,
- mydest,
- newmtime=thismtime,
- sstat=mystat,
- mysettings=self.settings,
- hardlink_candidates=hardlink_candidates,
- encoding=_encodings["merge"],
- )
- if mymtime is None:
- return 1
- hardlink_candidates.append(mydest)
- zing = ">>>"
+ mymtime = movefile(
+ mysrc,
+ mydest,
+ newmtime=thismtime,
+ sstat=mystat,
+ mysettings=self.settings,
+ hardlink_candidates=hardlink_candidates,
+ encoding=_encodings["merge"],
+ )
+ if mymtime is None:
+ return 1
+ hardlink_candidates.append(mydest)
+ zing = ">>>"
try:
self._merged_path(mydest, os.lstat(mydest))
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-05-23 0:26 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-05-23 0:26 UTC (permalink / raw
To: gentoo-commits
commit: 6da3e0fd0ddf5577771cf39b6ac329ee51051a22
Author: gcarq <egger.m <AT> protonmail <DOT> com>
AuthorDate: Mon Mar 27 13:47:52 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue May 23 00:22:09 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=6da3e0fd
mergeme: Put xattr comparison logic behind xattr feature flag
Signed-off-by: gcarq <egger.m <AT> protonmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 676d4aa05..739b47f7f 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -6267,9 +6267,10 @@ class dblink:
if mydmode is None or not stat.S_ISREG(mydmode) or mymode != mydmode:
return True
- excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
- if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
- return True
+ if "xattr" in self.settings.features:
+ excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
+ if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
+ return True
return not filecmp.cmp(mysrc, mydest, shallow=False)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-07-29 3:57 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-07-29 3:57 UTC (permalink / raw
To: gentoo-commits
commit: 7c0086fc0ab60cf0a726e3c88b11a21e147f3aa2
Author: Andrew Udvare <audvare <AT> gmail <DOT> com>
AuthorDate: Sun Jul 16 21:42:58 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Jul 29 03:57:45 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c0086fc
dbapi: add some typing
Signed-off-by: Andrew Udvare <audvare <AT> gmail.com>
Closes: https://github.com/gentoo/portage/pull/1069
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/__init__.py | 27 +++++++++++++++------------
lib/portage/dbapi/porttree.py | 34 ++++++++++++++++++++++++----------
2 files changed, 39 insertions(+), 22 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 31453d149..428e4a48e 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -5,6 +5,7 @@ __all__ = ["dbapi"]
import re
import warnings
+from typing import Any, Dict, List, Optional, Sequence, Tuple
import portage
@@ -29,7 +30,7 @@ from _emerge.Package import Package
class dbapi:
_category_re = re.compile(r"^\w[-.+\w]*$", re.UNICODE)
- _categories = None
+ _categories: Optional[Tuple[str, ...]] = None
_use_mutable = False
_known_keys = frozenset(auxdbkeys)
_pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository")
@@ -38,7 +39,7 @@ class dbapi:
pass
@property
- def categories(self):
+ def categories(self) -> Tuple[str, ...]:
"""
Use self.cp_all() to generate a category list. Mutable instances
can delete the self._categories attribute in cases when the cached
@@ -52,11 +53,11 @@ class dbapi:
def close_caches(self):
pass
- def cp_list(self, cp, use_cache=1):
+ def cp_list(self, cp: str, use_cache: int = 1) -> Any:
raise NotImplementedError(self)
@staticmethod
- def _cmp_cpv(cpv1, cpv2):
+ def _cmp_cpv(cpv1, cpv2) -> int:
result = vercmp(cpv1.version, cpv2.version)
if result == 0 and cpv1.build_time is not None and cpv2.build_time is not None:
result = (cpv1.build_time > cpv2.build_time) - (
@@ -65,7 +66,7 @@ class dbapi:
return result
@staticmethod
- def _cpv_sort_ascending(cpv_list):
+ def _cpv_sort_ascending(cpv_list: Sequence[Any]) -> None:
"""
Use this to sort self.cp_list() results in ascending
order. It sorts in place and returns None.
@@ -76,7 +77,7 @@ class dbapi:
# dict to map strings back to their original values.
cpv_list.sort(key=cmp_sort_key(dbapi._cmp_cpv))
- def cpv_all(self):
+ def cpv_all(self) -> List[str]:
"""Return all CPVs in the db
Args:
None
@@ -93,16 +94,18 @@ class dbapi:
cpv_list.extend(self.cp_list(cp))
return cpv_list
- def cp_all(self, sort=False):
+ def cp_all(self, sort: bool = False) -> List[str]:
"""Implement this in a child class
Args
sort - return sorted results
Returns:
A list of strings 1 per CP in the datastore
"""
- return NotImplementedError
+ raise NotImplementedError
- def aux_get(self, mycpv, mylist, myrepo=None):
+ def aux_get(
+ self, mycpv: str, mylist: str, myrepo: Optional[str] = None
+ ) -> List[str]:
"""Return the metadata keys in mylist for mycpv
Args:
mycpv - "sys-apps/foo-1.0"
@@ -114,7 +117,7 @@ class dbapi:
"""
raise NotImplementedError
- def aux_update(self, cpv, metadata_updates):
+ def aux_update(self, cpv: str, metadata_updates: Dict[str, Any]) -> None:
"""
Args:
cpv - "sys-apps/foo-1.0"
@@ -124,7 +127,7 @@ class dbapi:
"""
raise NotImplementedError
- def match(self, origdep, use_cache=1):
+ def match(self, origdep: str, use_cache: int = 1):
"""Given a dependency, try to find packages that match
Args:
origdep - Depend atom
@@ -138,7 +141,7 @@ class dbapi:
self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))
)
- def _iter_match(self, atom, cpv_iter):
+ def _iter_match(self, atom: str, cpv_iter):
cpv_iter = iter(match_from_list(atom, cpv_iter))
if atom.repo:
cpv_iter = self._iter_match_repo(atom, cpv_iter)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 3c38e99d4..c47b66bda 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -49,6 +49,7 @@ import functools
import collections
from collections import OrderedDict
+from typing import List, Optional, Sequence, Type, Tuple, Union
from urllib.parse import urlparse
@@ -435,7 +436,9 @@ class portdbapi(dbapi):
return license_path
return None
- def findname(self, mycpv, mytree=None, myrepo=None):
+ def findname(
+ self, mycpv: str, mytree: Optional[str] = None, myrepo: Optional[str] = None
+ ) -> str:
return self.findname2(mycpv, mytree, myrepo)[0]
def getRepositoryPath(self, repository_id):
@@ -494,7 +497,12 @@ class portdbapi(dbapi):
"""
return self.settings.repositories.ignored_repos
- def findname2(self, mycpv, mytree=None, myrepo=None):
+ def findname2(
+ self,
+ mycpv: str,
+ mytree: Optional[str] = None,
+ myrepo: Optional[str] = None,
+ ) -> Union[Tuple[None, int], Tuple[str, str], Tuple[str, None]]:
"""
Returns the location of the CPV, and what overlay it was in.
Searches overlays first, then PORTDIR; this allows us to return the first
@@ -643,7 +651,13 @@ class portdbapi(dbapi):
return (metadata, ebuild_hash)
- def aux_get(self, mycpv, mylist, mytree=None, myrepo=None):
+ def aux_get(
+ self,
+ mycpv: str,
+ mylist: Sequence[str],
+ mytree: Optional[str] = None,
+ myrepo: Optional[str] = None,
+ ) -> List[str]:
"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise PortageKeyError if error'
@@ -1200,12 +1214,12 @@ class portdbapi(dbapi):
def xmatch(
self,
- level,
- origdep,
- mydep=DeprecationWarning,
- mykey=DeprecationWarning,
- mylist=DeprecationWarning,
- ):
+ level: str,
+ origdep: str,
+ mydep: Type[DeprecationWarning] = DeprecationWarning,
+ mykey: Type[DeprecationWarning] = DeprecationWarning,
+ mylist: Type[DeprecationWarning] = DeprecationWarning,
+ ) -> Union[Sequence[str], str]:
"""
Caching match function.
@@ -1381,7 +1395,7 @@ class portdbapi(dbapi):
return myval
- def match(self, mydep, use_cache=1):
+ def match(self, mydep: str, use_cache: int = 1) -> Union[Sequence[str], str]:
return self.xmatch("match-visible", mydep)
def gvisible(self, mylist):
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-09-15 4:28 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-09-15 4:28 UTC (permalink / raw
To: gentoo-commits
commit: b2c91af6fb2dfdd227caf9a2eff588f02d2bf9fc
Author: Oskari Pirhonen <xxc3ncoredxx <AT> gmail <DOT> com>
AuthorDate: Tue Sep 5 01:21:22 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Sep 15 04:28:23 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2c91af6
vartree: keep build dir if postinst fails
Bug: https://bugs.gentoo.org/704866
Signed-off-by: Oskari Pirhonen <xxc3ncoredxx <AT> gmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 7d1bba712e..08ea9234f6 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -6070,8 +6070,11 @@ class dblink:
ebuild_phase.wait()
self._elog_process()
- if "noclean" not in self.settings.features and (
- retval == os.EX_OK or "fail-clean" in self.settings.features
+ # Keep the build dir around if postinst fails (bug #704866)
+ if (
+ not self._postinst_failure
+ and "noclean" not in self.settings.features
+ and (retval == os.EX_OK or "fail-clean" in self.settings.features)
):
if myebuild is None:
myebuild = os.path.join(inforoot, self.pkg + ".ebuild")
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-09-20 18:02 Mike Gilbert
0 siblings, 0 replies; 71+ messages in thread
From: Mike Gilbert @ 2023-09-20 18:02 UTC (permalink / raw
To: gentoo-commits
commit: 03489ca1880d4429c18cf7da2ed27ae65a21510b
Author: Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Tue Sep 19 18:15:30 2023 +0000
Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Wed Sep 20 18:02:20 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=03489ca1
Catch error when trust helper is not found
Closes: https://github.com/gentoo/portage/pull/1097
Signed-off-by: Andreas K. Hüttel <dilfridge <AT> gentoo.org>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 9d7c45577b..42912b2eb5 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1245,7 +1245,17 @@ class binarytree:
portage_trust_helper = self.settings.get("PORTAGE_TRUST_HELPER", "")
if portage_trust_helper == "":
return
- ret = subprocess.run(portage_trust_helper)
+ try:
+ ret = subprocess.run(portage_trust_helper)
+ except FileNotFoundError:
+ writemsg(
+ _(
+ "\n!!! Portage trust helper %s for binary packages not found\n!!! Continuing, but did you install app-portage/getuto?\n"
+ )
+ % portage_trust_helper,
+ noiselevel=-1,
+ )
+ return
ret.check_returncode()
def _populate_remote(self, getbinpkg_refresh=True):
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-09-23 22:31 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-09-23 22:31 UTC (permalink / raw
To: gentoo-commits
commit: 743722ccfa234dd9d4a54f7fbfb14cc2ddf6f0f4
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 23 22:27:37 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Sep 23 22:31:31 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=743722cc
vartree: drop 'Grabbing a set' useless noise
Prompted by genr8eofl asking about culling it.
Fascinatingly, both the message *and* people agreeing it's futile goes
back many years:
* bf01d6c02087c2b363617ce6ac208d09ea81de25 ('don't display useless messages when using --quiet')
* d9fc4acc572c6647a4f27b838d35d27d805d190e (svn migration)
* 4ba2baf8456c0eeda0df1bf52a1a3d712d5ac9b7 ('Portage-2.0.41 release. LOTS of bug fixes. Check the ChangeLog.', 2002)
This isn't useful for the user at all, so drop it.
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 08ea9234f6..35a401b33c 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -2660,10 +2660,6 @@ class dblink:
ignored_unlink_errnos = self._ignored_unlink_errnos
ignored_rmdir_errnos = self._ignored_rmdir_errnos
- if not pkgfiles:
- showMessage(_("No package files given... Grabbing a set.\n"))
- pkgfiles = self.getcontents()
-
if others_in_slot is None:
others_in_slot = []
slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot
@@ -2691,6 +2687,7 @@ class dblink:
unmerge_orphans = "unmerge-orphans" in self.settings.features
calc_prelink = "prelink-checksums" in self.settings.features
+ pkgfiles = self.getcontents()
if pkgfiles:
self.updateprotect()
mykeys = list(pkgfiles)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-09-23 22:38 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-09-23 22:38 UTC (permalink / raw
To: gentoo-commits
commit: 1fc674667de01944269dfeccd70472facf83e7ba
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 23 22:38:23 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Sep 23 22:38:31 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=1fc67466
vartree: don't clobber pkgfiles
Fixes: 743722ccfa234dd9d4a54f7fbfb14cc2ddf6f0f4
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 35a401b33c..878914ef06 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -2687,7 +2687,7 @@ class dblink:
unmerge_orphans = "unmerge-orphans" in self.settings.features
calc_prelink = "prelink-checksums" in self.settings.features
- pkgfiles = self.getcontents()
+ pkgfiles = self.getcontents() if not pkgfiles
if pkgfiles:
self.updateprotect()
mykeys = list(pkgfiles)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-09-23 22:49 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-09-23 22:49 UTC (permalink / raw
To: gentoo-commits
commit: 6293f7ba670fbfae5fd4bb6f4ba9fe3f822d19c3
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 23 22:49:29 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Sep 23 22:49:49 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=6293f7ba
vartree: fix syntax
Fixes: 1fc674667de01944269dfeccd70472facf83e7ba
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 878914ef06..d4b510082c 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -2687,7 +2687,7 @@ class dblink:
unmerge_orphans = "unmerge-orphans" in self.settings.features
calc_prelink = "prelink-checksums" in self.settings.features
- pkgfiles = self.getcontents() if not pkgfiles
+ pkgfiles = pkgfiles if pkgfiles else self.getcontents()
if pkgfiles:
self.updateprotect()
mykeys = list(pkgfiles)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-09-26 21:09 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2023-09-26 21:09 UTC (permalink / raw
To: gentoo-commits
commit: cec349d7cfcda634e3f76cd3ca0c1a89bc46f414
Author: hypersyd <70613804+siddhanthrathod <AT> users <DOT> noreply <DOT> github <DOT> com>
AuthorDate: Sat Sep 9 13:51:31 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Sep 26 21:09:28 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=cec349d7
bintree: populate invalid_paths list for eclean-pkg
Enable eclean-pkg to handle incomplete merges from binhost.
It's an extension for gentoolkit's eclean-pkg clean invalids feature to deal with
binpkgs created by the new binhost without any tar header or format due to incomplete merges.
This results in gpkg.py --> class gpkg --> method _get_tar_format returning null which raises
PortagePackageException and SignatureException in binpkg.py --> Class binarytree.
Add an input for invalid paths api is being added so the eclean-pkg can clean it up.
Invalids feature: https://github.com/gentoo/gentoolkit/commit/a16d0d4fbfb4614832c4b682b41284a9050af29f, https://github.com/gentoo/portage/commit/71daef3ac877329a0479a72ba333a9c801a36bf3
Bug: https://bugs.gentoo.org/900224
Signed-off-by: Siddhanth Rathod <xsiddhanthrathod <AT> gmail.com>
Closes: https://github.com/gentoo/portage/pull/1091
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 42912b2eb5..580ce2f290 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1050,6 +1050,10 @@ class binarytree:
else:
binpkg_format = "gpkg"
+ for ext in SUPPORTED_XPAK_EXTENSIONS + SUPPORTED_GPKG_EXTENSIONS:
+ if myfile.endswith(ext):
+ mypkg = myfile[: -len(ext)]
+ break
try:
pkg_metadata = self._read_metadata(
full_path,
@@ -1062,14 +1066,11 @@ class binarytree:
f"!!! Invalid binary package: '{full_path}', {e}\n",
noiselevel=-1,
)
+ self.invalid_paths[mypkg] = [full_path]
continue
mycat = pkg_metadata.get("CATEGORY", "")
mypf = pkg_metadata.get("PF", "")
slot = pkg_metadata.get("SLOT", "")
- for ext in SUPPORTED_XPAK_EXTENSIONS + SUPPORTED_GPKG_EXTENSIONS:
- if myfile.endswith(ext):
- mypkg = myfile[: -len(ext)]
- break
if not mycat or not mypf or not slot:
# old-style or corrupt package
writemsg(
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-04 4:29 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-04 4:29 UTC (permalink / raw
To: gentoo-commits
commit: f45e2bb561f2f5d16bdf8f6cd31cc393d2794f92
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 4 04:06:49 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Oct 4 04:11:14 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=f45e2bb5
SyncfsProcess: Migrate to ForkProcess target parameter
Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_SyncfsProcess.py | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/_SyncfsProcess.py b/lib/portage/dbapi/_SyncfsProcess.py
index 6beeac8dd4..ddc2240071 100644
--- a/lib/portage/dbapi/_SyncfsProcess.py
+++ b/lib/portage/dbapi/_SyncfsProcess.py
@@ -1,6 +1,8 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import functools
+
from portage import os
from portage.util._ctypes import find_library, LoadLibrary
from portage.util._async.ForkProcess import ForkProcess
@@ -16,6 +18,10 @@ class SyncfsProcess(ForkProcess):
__slots__ = ("paths",)
+ def _start(self):
+ self.target = functools.partial(self._target, self._get_syncfs, self.paths)
+ super()._start()
+
@staticmethod
def _get_syncfs():
filename = find_library("c")
@@ -29,12 +35,13 @@ class SyncfsProcess(ForkProcess):
return None
- def _run(self):
+ @staticmethod
+ def _target(get_syncfs, paths):
syncfs_failed = False
- syncfs = self._get_syncfs()
+ syncfs = get_syncfs()
if syncfs is not None:
- for path in self.paths:
+ for path in paths:
try:
fd = os.open(path, os.O_RDONLY)
except OSError:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-05 5:45 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-05 5:45 UTC (permalink / raw
To: gentoo-commits
commit: 2ccc08e3b629d3f56f028cd767ebd5ff1a8edaf1
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 5 04:43:50 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Oct 5 05:18:58 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2ccc08e3
MergeProcess: Migrate to ForkProcess target parameter
Bug: https://bugs.gentoo.org/915099
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 73 +++++++++++++++++++++++++++-----------
1 file changed, 52 insertions(+), 21 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index 7c018222fe..012435dce3 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -1,6 +1,7 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import functools
import io
import multiprocessing
import platform
@@ -73,6 +74,20 @@ class MergeProcess(ForkProcess):
self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
+ self.target = functools.partial(
+ self._target,
+ lambda: self._counter,
+ lambda: self._elog_reader_fd,
+ lambda: self._dblink,
+ self.infloc,
+ self.mydbapi,
+ self.myebuild,
+ self.pkgloc,
+ self.prev_mtimes,
+ self.settings,
+ self.unmerge,
+ self.vartree.dbapi,
+ )
super()._start()
def _lock_vdb(self):
@@ -197,13 +212,29 @@ class MergeProcess(ForkProcess):
return pids
- def _run(self):
- os.close(self._elog_reader_fd)
- counter = self._counter
- mylink = self._dblink
- portage.output.havecolor = not no_color(self.settings)
+ @staticmethod
+ def _target(
+ get_counter,
+ get_elog_reader_fd,
+ get_mylink,
+ infloc,
+ mydbapi,
+ myebuild,
+ pkgloc,
+ prev_mtimes,
+ settings,
+ unmerge,
+ vardb,
+ ):
+ """
+ TODO: Make all arguments picklable for the multiprocessing spawn start method.
+ """
+ os.close(get_elog_reader_fd())
+ counter = get_counter()
+ mylink = get_mylink()
+ portage.output.havecolor = not no_color(settings)
# Avoid wastful updates of the vdb cache.
- self.vartree.dbapi._flush_cache_enabled = False
+ vardb._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
@@ -211,21 +242,21 @@ class MergeProcess(ForkProcess):
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
- if not self.unmerge:
+ if not unmerge:
# unmerge phases have separate logs
- if self.settings.get("PORTAGE_BACKGROUND") == "1":
- self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
+ if settings.get("PORTAGE_BACKGROUND") == "1":
+ settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
- self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
- self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
- self.settings["PORTAGE_BACKGROUND"] = "subprocess"
- self.settings.backup_changes("PORTAGE_BACKGROUND")
+ settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
+ settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
+ settings["PORTAGE_BACKGROUND"] = "subprocess"
+ settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
- if self.unmerge:
+ if unmerge:
if not mylink.exists():
rval = os.EX_OK
- elif mylink.unmerge(ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
+ elif mylink.unmerge(ldpath_mtimes=prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
@@ -234,11 +265,11 @@ class MergeProcess(ForkProcess):
rval = os.EX_OK
else:
rval = mylink.merge(
- self.pkgloc,
- self.infloc,
- myebuild=self.myebuild,
- mydbapi=self.mydbapi,
- prev_mtimes=self.prev_mtimes,
+ pkgloc,
+ infloc,
+ myebuild=myebuild,
+ mydbapi=mydbapi,
+ prev_mtimes=prev_mtimes,
counter=counter,
)
return rval
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-08 19:48 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-08 19:48 UTC (permalink / raw
To: gentoo-commits
commit: fc70635aecbf80d3aeeb9f4e56d396040da09279
Author: Siddhanth Rathod <xsiddhanthrathod <AT> gmail <DOT> com>
AuthorDate: Sun Oct 8 09:25:15 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 8 19:47:56 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fc70635a
Fix: move msg array such that it gets assigned only when needed
Signed-off-by: Siddhanth Rathod <xsiddhanthrathod <AT> gmail.com>
Closes: https://github.com/gentoo/portage/pull/1124
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 7a4166c120..421863f3e8 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1092,10 +1092,10 @@ class binarytree:
missing_keys.append("PF")
if not slot:
missing_keys.append("SLOT")
- msg = []
- if missing_keys:
- missing_keys.sort()
- if invalid_errors:
+ if invalid_errors:
+ msg = []
+ if missing_keys:
+ missing_keys.sort()
msg.append(
_("Missing metadata key(s): %s.")
% ", ".join(missing_keys)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-15 22:02 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-15 22:02 UTC (permalink / raw
To: gentoo-commits
commit: 3e38ae92bdd5b057352a2bcb044fb587b15b25f3
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 15 19:21:44 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 15 21:38:08 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e38ae92
MergeProcess: Eliminate target arguments that reference self
This improves compatibility with the multiprocessing spawn
start method, by eliminating this error:
AttributeError: Can't pickle local object 'MergeProcess._start.<locals>.<lambda>'
Bug: https://bugs.gentoo.org/914876
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 40 +++++++++++++++++++-------------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index 012435dce3..c1085270da 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -74,20 +74,6 @@ class MergeProcess(ForkProcess):
self.fd_pipes.setdefault(0, portage._get_stdin().fileno())
self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
- self.target = functools.partial(
- self._target,
- lambda: self._counter,
- lambda: self._elog_reader_fd,
- lambda: self._dblink,
- self.infloc,
- self.mydbapi,
- self.myebuild,
- self.pkgloc,
- self.prev_mtimes,
- self.settings,
- self.unmerge,
- self.vartree.dbapi,
- )
super()._start()
def _lock_vdb(self):
@@ -195,6 +181,22 @@ class MergeProcess(ForkProcess):
self._dblink = mylink
self._elog_reader_fd = elog_reader_fd
+
+ self.target = functools.partial(
+ self._target,
+ self._counter,
+ self._elog_reader_fd,
+ self._dblink,
+ self.infloc,
+ self.mydbapi,
+ self.myebuild,
+ self.pkgloc,
+ self.prev_mtimes,
+ self.settings,
+ self.unmerge,
+ self.vartree.dbapi,
+ )
+
pids = super()._spawn(args, fd_pipes, **kwargs)
os.close(elog_writer_fd)
mtime_writer.close()
@@ -214,9 +216,9 @@ class MergeProcess(ForkProcess):
@staticmethod
def _target(
- get_counter,
- get_elog_reader_fd,
- get_mylink,
+ counter,
+ elog_reader_fd,
+ mylink,
infloc,
mydbapi,
myebuild,
@@ -229,9 +231,7 @@ class MergeProcess(ForkProcess):
"""
TODO: Make all arguments picklable for the multiprocessing spawn start method.
"""
- os.close(get_elog_reader_fd())
- counter = get_counter()
- mylink = get_mylink()
+ os.close(elog_reader_fd)
portage.output.havecolor = not no_color(settings)
# Avoid wastful updates of the vdb cache.
vardb._flush_cache_enabled = False
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-20 0:34 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-20 0:34 UTC (permalink / raw
To: gentoo-commits
commit: d92d69a52b2b127a0934656163f7075015ef7c85
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 17 19:05:07 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct 17 19:14:54 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=d92d69a5
MergeProcess: Use multiprocessing.Pipe to decouple fd_pipes
Use multiprocessing.Pipe to decouple from the fd_pipes implementation
since that currently only works for the multiprocessing "fork" start
method.
Bug: https://bugs.gentoo.org/915903
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 25 ++++++++++---------------
lib/portage/dbapi/vartree.py | 2 +-
2 files changed, 11 insertions(+), 16 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index c1085270da..cf8e6513d1 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -96,7 +96,7 @@ class MergeProcess(ForkProcess):
self._locked_vdb = False
def _elog_output_handler(self):
- output = self._read_buf(self._elog_reader_fd)
+ output = self._read_buf(self._elog_reader_fd.fileno())
if output:
lines = _unicode_decode(output).split("\n")
if len(lines) == 1:
@@ -112,8 +112,8 @@ class MergeProcess(ForkProcess):
reporter(msg, phase=phase, key=key, out=out)
elif output is not None: # EIO/POLLHUP
- self.scheduler.remove_reader(self._elog_reader_fd)
- os.close(self._elog_reader_fd)
+ self.scheduler.remove_reader(self._elog_reader_fd.fileno())
+ self._elog_reader_fd.close()
self._elog_reader_fd = None
return False
@@ -136,16 +136,15 @@ class MergeProcess(ForkProcess):
post-fork actions.
"""
- elog_reader_fd, elog_writer_fd = os.pipe()
+ elog_reader_fd, elog_writer_fd = multiprocessing.Pipe(duplex=False)
fcntl.fcntl(
- elog_reader_fd,
+ elog_reader_fd.fileno(),
fcntl.F_SETFL,
- fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
+ fcntl.fcntl(elog_reader_fd.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK,
)
mtime_reader, mtime_writer = multiprocessing.Pipe(duplex=False)
- fd_pipes[mtime_writer.fileno()] = mtime_writer.fileno()
self.scheduler.add_reader(mtime_reader.fileno(), self._mtime_handler)
self._mtime_reader = mtime_reader
@@ -166,8 +165,7 @@ class MergeProcess(ForkProcess):
pipe=elog_writer_fd,
mtime_pipe=mtime_writer,
)
- fd_pipes[elog_writer_fd] = elog_writer_fd
- self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler)
+ self.scheduler.add_reader(elog_reader_fd.fileno(), self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
@@ -185,7 +183,6 @@ class MergeProcess(ForkProcess):
self.target = functools.partial(
self._target,
self._counter,
- self._elog_reader_fd,
self._dblink,
self.infloc,
self.mydbapi,
@@ -198,7 +195,7 @@ class MergeProcess(ForkProcess):
)
pids = super()._spawn(args, fd_pipes, **kwargs)
- os.close(elog_writer_fd)
+ elog_writer_fd.close()
mtime_writer.close()
self._buf = ""
self._elog_keys = set()
@@ -217,7 +214,6 @@ class MergeProcess(ForkProcess):
@staticmethod
def _target(
counter,
- elog_reader_fd,
mylink,
infloc,
mydbapi,
@@ -231,7 +227,6 @@ class MergeProcess(ForkProcess):
"""
TODO: Make all arguments picklable for the multiprocessing spawn start method.
"""
- os.close(elog_reader_fd)
portage.output.havecolor = not no_color(settings)
# Avoid wastful updates of the vdb cache.
vardb._flush_cache_enabled = False
@@ -301,8 +296,8 @@ class MergeProcess(ForkProcess):
self._unlock_vdb()
if self._elog_reader_fd is not None:
- self.scheduler.remove_reader(self._elog_reader_fd)
- os.close(self._elog_reader_fd)
+ self.scheduler.remove_reader(self._elog_reader_fd.fileno())
+ self._elog_reader_fd.close()
self._elog_reader_fd = None
if self._elog_keys is not None:
for key in self._elog_keys:
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 835cbb8092..88fc525771 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -4198,7 +4198,7 @@ class dblink:
if str_buffer:
str_buffer = _unicode_encode("".join(str_buffer))
while str_buffer:
- str_buffer = str_buffer[os.write(self._pipe, str_buffer) :]
+ str_buffer = str_buffer[os.write(self._pipe.fileno(), str_buffer) :]
def _emerge_log(self, msg):
emergelog(False, msg)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-22 15:53 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-22 15:53 UTC (permalink / raw
To: gentoo-commits
commit: c41d109b121feb0bf6199b4082a35799fdc547b9
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 22 08:28:39 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 22 08:30:50 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c41d109b
vardbapi.unpack_metadata: Make compatible with spawn start method
This solves the following error with the multiprocessing spawn start
method:
AttributeError: Can't pickle local object 'vardbapi.unpack_metadata.<locals>.async_copy'
Bug: https://bugs.gentoo.org/916108
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 88fc525771..a80d3bc0e3 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -1003,6 +1003,13 @@ class vardbapi(dbapi):
pass
self._bump_mtime(cpv)
+ @staticmethod
+ def _async_copy(dbdir, dest_dir):
+ for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc):
+ for key in files:
+ shutil.copy(os.path.join(parent, key), os.path.join(dest_dir, key))
+ break
+
async def unpack_metadata(self, pkg, dest_dir, loop=None):
"""
Unpack package metadata to a directory. This method is a coroutine.
@@ -1018,14 +1025,9 @@ class vardbapi(dbapi):
else:
cpv = pkg.mycpv
dbdir = self.getpath(cpv)
-
- def async_copy():
- for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc):
- for key in files:
- shutil.copy(os.path.join(parent, key), os.path.join(dest_dir, key))
- break
-
- await loop.run_in_executor(ForkExecutor(loop=loop), async_copy)
+ await loop.run_in_executor(
+ ForkExecutor(loop=loop), self._async_copy, dbdir, dest_dir
+ )
async def unpack_contents(
self,
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-22 20:58 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-22 20:58 UTC (permalink / raw
To: gentoo-commits
commit: 0b9fbe389db3c3b9e625e1f5f526b3a921d2e0ce
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 22 19:57:16 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 22 19:58:20 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b9fbe38
vardbapi.unpack_contents: Use multiprocessing.Pipe for spawn compat
Bug: https://bugs.gentoo.org/916112
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 28 +++++++++++++++-------------
1 file changed, 15 insertions(+), 13 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index a80d3bc0e3..5638935606 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -101,6 +101,7 @@ import grp
import io
from itertools import chain
import logging
+import multiprocessing
import os as _os
import operator
import pickle
@@ -1094,19 +1095,18 @@ class vardbapi(dbapi):
)
if binpkg_format == "xpak":
tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", dest_dir)
- pr, pw = os.pipe()
+ pr, pw = multiprocessing.Pipe(duplex=False)
proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
- os.close(pr)
- with os.fdopen(pw, "wb", 0) as pw_file:
- excluded_config_files = await loop.run_in_executor(
- ForkExecutor(loop=loop),
- functools.partial(
- self._dblink(cpv).quickpkg,
- pw_file,
- include_config=opts.include_config == "y",
- include_unmodified_config=opts.include_unmodified_config == "y",
- ),
- )
+ pr.close()
+ excluded_config_files = await loop.run_in_executor(
+ ForkExecutor(loop=loop),
+ functools.partial(
+ self._dblink(cpv).quickpkg,
+ pw,
+ include_config=opts.include_config == "y",
+ include_unmodified_config=opts.include_unmodified_config == "y",
+ ),
+ )
await proc.wait()
if proc.returncode != os.EX_OK:
raise PortageException(f"command failed: {tar_cmd}")
@@ -2161,7 +2161,9 @@ class dblink:
# The tarfile module will write pax headers holding the
# xattrs only if PAX_FORMAT is specified here.
with tarfile.open(
- fileobj=output_file,
+ fileobj=output_file
+ if hasattr(output_file, "write")
+ else open(output_file.fileno(), mode="wb", closefd=False),
mode="w|",
format=tarfile.PAX_FORMAT if xattrs else tarfile.DEFAULT_FORMAT,
) as tar:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-22 21:30 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-22 21:30 UTC (permalink / raw
To: gentoo-commits
commit: b9a85ff987ea677e350fb70e2c3355197ee6c39b
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 22 08:08:02 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 22 21:29:49 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9a85ff9
MergeProcess: Support QueryCommand with spawn start method
Since QueryCommand._db does not propagate to the child process
in MergeProcess with the multiprocessing spawn start method,
handle it by passing QueryCommand._db as an explicit parameter.
Since self.mydbapi can be a bindbapi instance, override the
bindbapi __getstate__ method so that it is picklable (omit
the unpicklable _aux_cache_slot_dict attribute and regenerate
it after unpickling).
Bug: https://bugs.gentoo.org/916106
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 19 ++++++++++++++++---
lib/portage/dbapi/bintree.py | 20 ++++++++++++++++++--
2 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index cf8e6513d1..dd5ad71cf8 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -9,6 +9,7 @@ import platform
import fcntl
import portage
from portage import os, _unicode_decode
+from portage.package.ebuild._ipc.QueryCommand import QueryCommand
from portage.util._ctypes import find_library
import portage.elog.messages
from portage.util._async.ForkProcess import ForkProcess
@@ -180,6 +181,15 @@ class MergeProcess(ForkProcess):
self._dblink = mylink
self._elog_reader_fd = elog_reader_fd
+ # Since the entire QueryCommand._db is not required, only pass
+ # in tree types that QueryCommand specifically requires.
+ child_db = {}
+ parent_db = portage.db if QueryCommand._db is None else QueryCommand._db
+ for root in parent_db:
+ child_db[root] = {}
+ for tree_type in ("vartree", "porttree"):
+ child_db[root][tree_type] = parent_db[root][tree_type]
+
self.target = functools.partial(
self._target,
self._counter,
@@ -192,6 +202,7 @@ class MergeProcess(ForkProcess):
self.settings,
self.unmerge,
self.vartree.dbapi,
+ child_db,
)
pids = super()._spawn(args, fd_pipes, **kwargs)
@@ -223,10 +234,12 @@ class MergeProcess(ForkProcess):
settings,
unmerge,
vardb,
+ db,
):
- """
- TODO: Make all arguments picklable for the multiprocessing spawn start method.
- """
+ if QueryCommand._db is None:
+ # Initialize QueryCommand._db for AbstractEbuildProcess/EbuildIpcDaemon
+ # when not using the multiprocessing fork start method.
+ QueryCommand._db = db
portage.output.havecolor = not no_color(settings)
# Avoid wastful updates of the vdb cache.
vardb._flush_cache_enabled = False
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 421863f3e8..3f1bf9d5d2 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["bindbapi", "binarytree"]
@@ -129,8 +129,24 @@ class bindbapi(fakedbapi):
"USE",
"_mtime_",
}
- self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
self._aux_cache = {}
+ self._aux_cache_slot_dict_cache = None
+
+ @property
+ def _aux_cache_slot_dict(self):
+ if self._aux_cache_slot_dict_cache is None:
+ self._aux_cache_slot_dict_cache = slot_dict_class(self._aux_cache_keys)
+ return self._aux_cache_slot_dict_cache
+
+ def __getstate__(self):
+ # This attribute is not picklable, but it automatically
+ # regenerates after unpickling when set to None.
+ _aux_cache_slot_dict = self._aux_cache_slot_dict_cache
+ self._aux_cache_slot_dict_cache = None
+ try:
+ return super().__getstate__()
+ finally:
+ self._aux_cache_slot_dict_cache = _aux_cache_slot_dict
@property
def writable(self):
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-23 14:28 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-23 14:28 UTC (permalink / raw
To: gentoo-commits
commit: c7768ae4820201913ba28d5fad441c1acc4659c0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 23 04:14:05 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Oct 23 04:41:10 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c7768ae4
bindbapi: Fix pickling for MergeProcess
Both the _aux_cache_slot_dict_cache and _instance_key attributes are
unpicklable, so fix __getstate__ to exclude them both from the state.
Override __setstate__ to initialize _instance_key appropriately.
These changes prevent the following error:
AttributeError: Can't pickle local object 'slot_dict_class.<locals>.LocalSlotDict'
Fixes: b9a85ff987ea ("MergeProcess: Support QueryCommand with spawn start method")
Bug: https://bugs.gentoo.org/916106
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 3f1bf9d5d2..0ecfdc25d0 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -139,14 +139,19 @@ class bindbapi(fakedbapi):
return self._aux_cache_slot_dict_cache
def __getstate__(self):
- # This attribute is not picklable, but it automatically
- # regenerates after unpickling when set to None.
- _aux_cache_slot_dict = self._aux_cache_slot_dict_cache
- self._aux_cache_slot_dict_cache = None
- try:
- return super().__getstate__()
- finally:
- self._aux_cache_slot_dict_cache = _aux_cache_slot_dict
+ state = self.__dict__.copy()
+ # These attributes are not picklable, so they are automatically
+ # regenerated after unpickling.
+ state["_aux_cache_slot_dict_cache"] = None
+ state["_instance_key"] = None
+ return state
+
+ def __setstate__(self, state):
+ self.__dict__.update(state)
+ if self._multi_instance:
+ self._instance_key = self._instance_key_multi_instance
+ else:
+ self._instance_key = self._instance_key_cpv
@property
def writable(self):
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-10-24 18:37 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-10-24 18:37 UTC (permalink / raw
To: gentoo-commits
commit: 7a43de7a8ce8dc6836d998cb87a5135ba8f0bc2c
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 24 18:07:53 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct 24 18:08:23 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a43de7a
dbapi: Make _iuse_implicit_cnstr picklable for spawn compat
Bug: https://bugs.gentoo.org/916235
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/__init__.py | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 233da8d698..dd697db109 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,8 +1,9 @@
-# Copyright 1998-2020 Gentoo Authors
+# Copyright 1998-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["dbapi"]
+import functools
import re
import warnings
from typing import Any, Dict, List, Optional, Tuple
@@ -223,6 +224,10 @@ class dbapi:
yield cpv
+ @staticmethod
+ def _iuse_implicit_built(iuse_implicit_match, use, flag):
+ return iuse_implicit_match(flag) or flag in use
+
def _iuse_implicit_cnstr(self, pkg, metadata):
"""
Construct a callable that checks if a given USE flag should
@@ -257,9 +262,11 @@ class dbapi:
# This behavior is only used for EAPIs that support IUSE_EFFECTIVE,
# since built USE settings for earlier EAPIs may contain a large
# number of irrelevant flags.
- prof_iuse = iuse_implicit_match
- enabled = frozenset(metadata["USE"].split()).__contains__
- iuse_implicit_match = lambda flag: prof_iuse(flag) or enabled(flag)
+ iuse_implicit_match = functools.partial(
+ self._iuse_implicit_built,
+ iuse_implicit_match,
+ frozenset(metadata["USE"].split()),
+ )
return iuse_implicit_match
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2023-12-10 1:28 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2023-12-10 1:28 UTC (permalink / raw
To: gentoo-commits
commit: cc372a52ee2aba153c0d7c2290306b7f623e8b7c
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 9 23:58:07 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec 10 01:27:43 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=cc372a52
bindbapi: add SIZE to _pkg_str_aux_keys
The _pkg_str_aux_keys are used inside dbapi.update_ents,
and need to contain SIZE in order for the fakedbapi
_instance_key_multi_instance method to operate correctly.
Incorrect operation of _instance_key_multi_instance could
prevent binarytree.inject from removing an old instance
from its internal state. It could also trigger a KeyError
in bindbapi.aux_update as in bug 918597, since it could
cause binarytree.getname to return a non-existent path.
It could also cause binarytree.getname to return an
existing but incorrect path, which might trigger an
InvalidBinaryPackageFormat exception as in bug 906675.
Bug: https://bugs.gentoo.org/906675
Bug: https://bugs.gentoo.org/918597
Bug: https://bugs.gentoo.org/919668
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index a6e1f9773d..9c9ac66334 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -84,9 +84,12 @@ class bindbapi(fakedbapi):
_known_keys = frozenset(
list(fakedbapi._known_keys) + ["CHOST", "repository", "USE"]
)
+ # Must include keys used to create _pkg_str attributes used in
+ # the fakedbapi _instance_key_multi_instance method.
_pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + (
"BUILD_ID",
"BUILD_TIME",
+ "SIZE",
"_mtime_",
)
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-01-03 5:57 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2024-01-03 5:57 UTC (permalink / raw
To: gentoo-commits
commit: 6c915f3ded10c23127c33b771453d54e3f0d4915
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 3 04:26:36 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jan 3 05:56:59 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=6c915f3d
binarytree: Avoid unnecessary build id incrementation
When allocating a local file name for a remote package,
prefer re-using the existing build id if the file name
is unused locally, in order to avoid unnecessary build
id incrementation triggered when the _max_build_id
method counts remote build ids. Use the same file name
reservation via open with O_EXCL which has been used
during build_id incrementation since fa901a6510c.
Bug: https://bugs.gentoo.org/921208
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 39 ++++++++++++++++++++++++++++-----------
1 file changed, 28 insertions(+), 11 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index d352b6fc0e..5fc4716980 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["bindbapi", "binarytree"]
@@ -2272,16 +2272,6 @@ class binarytree:
raise InvalidBinaryPackageFormat(binpkg_format)
def _allocate_filename_multi(self, cpv, remote_binpkg_format=None):
- # First, get the max build_id found when _populate was
- # called.
- max_build_id = self._max_build_id(cpv)
-
- # A new package may have been added concurrently since the
- # last _populate call, so use increment build_id until
- # we locate an unused id.
- pf = catsplit(cpv)[1]
- build_id = max_build_id + 1
-
if remote_binpkg_format is None:
try:
binpkg_format = get_binpkg_format(cpv._metadata["PATH"])
@@ -2299,6 +2289,33 @@ class binarytree:
else:
raise InvalidBinaryPackageFormat(binpkg_format)
+ # If the preferred path is available then return
+ # that. This prevents unnecessary build_id incrementation
+ # triggered when the _max_build_id method counts remote
+ # build ids.
+ pf = catsplit(cpv)[1]
+ if getattr(cpv, "build_id", False):
+ preferred_path = f"{os.path.join(self.pkgdir, cpv.cp, pf)}-{cpv.build_id}.{binpkg_suffix}"
+ if not os.path.exists(preferred_path):
+ try:
+ # Avoid races
+ ensure_dirs(os.path.dirname(preferred_path))
+ with open(preferred_path, "x") as f:
+ pass
+ except FileExistsError:
+ pass
+ else:
+ return (preferred_path, cpv.build_id)
+
+ # First, get the max build_id found when _populate was
+ # called.
+ max_build_id = self._max_build_id(cpv)
+
+ # A new package may have been added concurrently since the
+ # last _populate call, so use increment build_id until
+ # we locate an unused id.
+ build_id = max_build_id + 1
+
while True:
filename = (
f"{os.path.join(self.pkgdir, cpv.cp, pf)}-{build_id}.{binpkg_suffix}"
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-01-16 5:16 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2024-01-16 5:16 UTC (permalink / raw
To: gentoo-commits
commit: 58b094bc79e999e44a5b108e2b7273c164aa906e
Author: Alfred Wingate <parona <AT> protonmail <DOT> com>
AuthorDate: Fri Jan 5 19:30:38 2024 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Jan 16 05:16:09 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=58b094bc
bintree: use urllib provided attributes for hostname, user and password
In addition to simplifying it also solves an issue with parsing raw
ipv6 addresses.
Bug: https://bugs.gentoo.org/921400
Signed-off-by: Alfred Wingate <parona <AT> protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1230
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 18 ++++--------------
1 file changed, 4 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 5fc4716980..2342d571b9 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1344,23 +1344,13 @@ class binarytree:
for repo in reversed(list(self._binrepos_conf.values())):
base_url = repo.sync_uri
parsed_url = urlparse(base_url)
- host = parsed_url.netloc
+ host = parsed_url.hostname or ""
port = parsed_url.port
- user = None
- passwd = None
- user_passwd = ""
+ user = parsed_url.username
+ passwd = parsed_url.password
+ user_passwd = user + "@" if user else ""
gpkg_only_warned = False
- if "@" in host:
- user, host = host.split("@", 1)
- user_passwd = user + "@"
- if ":" in user:
- user, passwd = user.split(":", 1)
-
- if port is not None:
- port_str = f":{port}"
- if host.endswith(port_str):
- host = host[: -len(port_str)]
pkgindex_file = os.path.join(
self.settings["EROOT"],
CACHE_PATH,
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-01-16 5:16 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2024-01-16 5:16 UTC (permalink / raw
To: gentoo-commits
commit: c9ed8136f9f0c56dc7b72d400eaa0acbd338778f
Author: Konstantin Tokarev <annulen <AT> yandex <DOT> ru>
AuthorDate: Fri Jan 12 17:47:45 2024 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Jan 16 05:16:08 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c9ed8136
vartree: _needs_move() should replace file if filecmp fails for whatever reason
When trying to update I've got "OSError: [Errno 5] Input/output error"
exception traceback from portage originating from _needs_move(). It turned
out that 3 files in my root filesystem were corrupted, which caused filecmp
to fail. This patch makes portage replace original file in this case.
Bug: https://bugs.gentoo.org/722270
Signed-off-by: Konstantin Tokarev <annulen <AT> yandex.ru>
Closes: https://github.com/gentoo/portage/pull/1233
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 5d39ca1965..a00c731c57 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -6283,7 +6283,19 @@ class dblink:
if not _cmpxattr(src_bytes, dest_bytes, exclude=excluded_xattrs):
return True
- return not filecmp.cmp(src_bytes, dest_bytes, shallow=False)
+ try:
+ files_equal = filecmp.cmp(src_bytes, dest_bytes, shallow=False)
+ except Exception as e:
+ writemsg(
+ _(
+ "Exception '%s' happened when comparing files %s and %s, will replace the latter\n"
+ )
+ % (e, mysrc, mydest),
+ noiselevel=-1,
+ )
+ return True
+
+ return not files_equal
def merge(
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-01-16 5:26 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2024-01-16 5:26 UTC (permalink / raw
To: gentoo-commits
commit: 68f4ea8a90d8759a1aa859d9188017e21797bdd0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 16 01:08:36 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jan 16 05:25:20 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=68f4ea8a
Handle SignatureException during package moves
Ignore package moves for packages that raise SignatureException,
just as they are ignored for packages that have a valid signature.
Bug: https://bugs.gentoo.org/922142
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 2342d571b9..ee574f435f 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -296,8 +296,12 @@ class bindbapi(fakedbapi):
encoding_key = True
elif binpkg_format == "gpkg":
mybinpkg = portage.gpkg.gpkg(self.settings, cpv_str, binpkg_path)
- mydata = mybinpkg.get_metadata()
- if mybinpkg.signature_exist:
+ try:
+ mydata = mybinpkg.get_metadata()
+ signature_exist = mybinpkg.signature_exist
+ except SignatureException:
+ signature_exist = True
+ if signature_exist:
writemsg(
colorize(
"WARN",
@@ -721,8 +725,12 @@ class binarytree:
decode_metadata_name = False
elif binpkg_format == "gpkg":
mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, binpkg_path)
- mydata = mybinpkg.get_metadata()
- if mybinpkg.signature_exist:
+ try:
+ mydata = mybinpkg.get_metadata()
+ signature_exist = mybinpkg.signature_exist
+ except SignatureException:
+ signature_exist = True
+ if signature_exist:
writemsg(
colorize(
"WARN",
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-01-16 7:52 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2024-01-16 7:52 UTC (permalink / raw
To: gentoo-commits
commit: adc0052b2c022851458f788868e6b194ed1cfe9f
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 16 07:51:47 2024 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue Jan 16 07:52:11 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=adc0052b
bintree: add missing newlines to signed binpkg update notice
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index ee574f435f..f4251b47d6 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -306,7 +306,7 @@ class bindbapi(fakedbapi):
colorize(
"WARN",
f"Binpkg update ignored for signed package: {binpkg_path}, "
- "the file will be removed.",
+ "the file will be removed.\n",
)
)
self.bintree.remove(cpv)
@@ -734,7 +734,7 @@ class binarytree:
writemsg(
colorize(
"WARN",
- f"Binpkg update ignored for signed package: {binpkg_path}",
+ f"Binpkg update ignored for signed package: {binpkg_path}\n",
)
)
continue
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-02-25 8:25 Sam James
0 siblings, 0 replies; 71+ messages in thread
From: Sam James @ 2024-02-25 8:25 UTC (permalink / raw
To: gentoo-commits
commit: 0855821572f32e81b031a250f7491f34a2fd4078
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 24 23:29:29 2024 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Feb 25 08:24:55 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=08558215
dbapi: Fix TypeError when passing Exception to warnings.warn
If an Exception is passed as a message to warnings.warn then
it triggers this TypeError:
if metadata_updates:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
> warnings.warn(e)
E TypeError: expected string or bytes-like object, got 'CorruptionKeyError'
Fixes: fb1d0a22f657 ("dbapi: KeyError tolerance during package moves")
Bug: https://bugs.gentoo.org/922935
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1282
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/dbapi/__init__.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 6f95b93a21..9105227c77 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,11 +1,12 @@
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["dbapi"]
import functools
+import logging
import re
-import warnings
+import sys
from typing import Any, Dict, List, Optional, Tuple
from collections.abc import Sequence
@@ -429,7 +430,9 @@ class dbapi:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
- warnings.warn(e)
+ logging.warning(
+ f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info()
+ )
if onUpdate:
onUpdate(maxval, i + 1)
if onProgress:
@@ -477,6 +480,6 @@ class dbapi:
try:
self.aux_update(mycpv, mydata)
except CorruptionKeyError as e:
- warnings.warn(e)
+ logging.warning(f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info())
continue
return moves
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-03-02 22:55 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2024-03-02 22:55 UTC (permalink / raw
To: gentoo-commits
commit: 62ee9bf8c680b2a18713da5bd453e3a771257409
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 2 22:48:54 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 2 22:50:54 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=62ee9bf8
binarytree._populate_remote: Fix UnboundLocalError for binpkg-request-signature
If an InvalidBinaryPackageFormat exception was raised from
get_binpkg_format for binpkg-request-signature then it
triggered an UnboundLocalError here.
Fixes: 445f10f4214c ("Use binpkg extensions and header to get format")
Bug: https://bugs.gentoo.org/926048
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index f4251b47d6..4ba1407cda 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1624,7 +1624,7 @@ class binarytree:
binpkg_format = get_binpkg_format(
d.get("PATH"), remote=True
)
- except InvalidBinaryPackageFormat:
+ except InvalidBinaryPackageFormat as e:
writemsg(
colorize(
"WARN",
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-05-27 18:13 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2024-05-27 18:13 UTC (permalink / raw
To: gentoo-commits
commit: 2db89a16ab87b85004216959ec6bc508575d96a0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 27 18:03:32 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 27 18:04:41 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2db89a16
binarytree: Rewrite remote index only on change
I noticed that the remote index was rewritten with a
new DOWNLOAD_TIMESTAMP even while frozen, and this
patch fixed it.
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/bintree.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 64dfee4faa..b32dea1eae 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1400,6 +1400,7 @@ class binarytree:
except OSError as e:
if e.errno != errno.ENOENT:
raise
+ changed = True
local_timestamp = pkgindex.header.get("TIMESTAMP", None)
try:
download_timestamp = float(pkgindex.header.get("DOWNLOAD_TIMESTAMP", 0))
@@ -1574,6 +1575,7 @@ class binarytree:
noiselevel=-1,
)
except UseCachedCopyOfRemoteIndex:
+ changed = False
desc = "frozen" if repo.frozen else "up-to-date"
writemsg_stdout("\n")
writemsg_stdout(
@@ -1611,7 +1613,7 @@ class binarytree:
os.unlink(tmp_filename)
except OSError:
pass
- if pkgindex is rmt_idx:
+ if pkgindex is rmt_idx and changed:
pkgindex.modified = False # don't update the header
pkgindex.header["DOWNLOAD_TIMESTAMP"] = "%d" % time.time()
try:
^ permalink raw reply related [flat|nested] 71+ messages in thread
* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
@ 2024-06-01 19:20 Zac Medico
0 siblings, 0 replies; 71+ messages in thread
From: Zac Medico @ 2024-06-01 19:20 UTC (permalink / raw
To: gentoo-commits
commit: 1a7fc63d20ad2e1292be3697c105c2d7e1691f91
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 1 04:56:45 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jun 1 19:18:35 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=1a7fc63d
MergeProcess: Pass bintree to subprocess
It's required for FEATURES=*-backup.
Fixes: b9a85ff987ea ("MergeProcess: Support QueryCommand with spawn start method")
Bug: https://bugs.gentoo.org/933297
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index d9ab2b47aa..34e39eb229 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2023 Gentoo Authors
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -183,11 +183,12 @@ class MergeProcess(ForkProcess):
# Since the entire QueryCommand._db is not required, only pass
# in tree types that QueryCommand specifically requires.
+ # NOTE: For FEATURES=*-backup bintree is needed (bug 933297).
child_db = {}
parent_db = portage.db if QueryCommand._db is None else QueryCommand._db
for root in parent_db:
child_db[root] = {}
- for tree_type in ("vartree", "porttree"):
+ for tree_type in ("bintree", "porttree", "vartree"):
child_db[root][tree_type] = parent_db[root][tree_type]
self.target = functools.partial(
^ permalink raw reply related [flat|nested] 71+ messages in thread
end of thread, other threads:[~2024-06-01 19:20 UTC | newest]
Thread overview: 71+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-09-25 19:12 [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/ Mike Gilbert
-- strict thread matches above, loose matches on Subject: below --
2024-06-01 19:20 Zac Medico
2024-05-27 18:13 Zac Medico
2024-03-02 22:55 Zac Medico
2024-02-25 8:25 Sam James
2024-01-16 7:52 Sam James
2024-01-16 5:26 Zac Medico
2024-01-16 5:16 Sam James
2024-01-16 5:16 Sam James
2024-01-03 5:57 Zac Medico
2023-12-10 1:28 Zac Medico
2023-10-24 18:37 Zac Medico
2023-10-23 14:28 Zac Medico
2023-10-22 21:30 Zac Medico
2023-10-22 20:58 Zac Medico
2023-10-22 15:53 Zac Medico
2023-10-20 0:34 Zac Medico
2023-10-15 22:02 Zac Medico
2023-10-08 19:48 Zac Medico
2023-10-05 5:45 Zac Medico
2023-10-04 4:29 Zac Medico
2023-09-26 21:09 Sam James
2023-09-23 22:49 Sam James
2023-09-23 22:38 Sam James
2023-09-23 22:31 Sam James
2023-09-20 18:02 Mike Gilbert
2023-09-15 4:28 Sam James
2023-07-29 3:57 Sam James
2023-05-23 0:26 Sam James
2023-05-23 0:26 Sam James
2023-05-23 0:26 Sam James
2022-12-21 1:30 Sam James
2022-12-21 1:28 Sam James
2022-11-08 23:07 Sam James
2022-09-09 10:16 Michał Górny
2022-08-18 19:00 Mike Gilbert
2022-04-20 20:24 Zac Medico
2022-04-13 15:34 Sam James
2022-04-13 15:34 Sam James
2021-11-26 21:09 Mike Gilbert
2021-09-21 5:51 Zac Medico
2021-09-21 5:51 Zac Medico
2021-09-21 5:51 Zac Medico
2021-06-05 18:08 Zac Medico
2021-03-07 11:42 Zac Medico
2021-02-23 21:31 Zac Medico
2021-01-18 9:20 Zac Medico
2021-01-17 13:31 Zac Medico
2021-01-17 8:49 Zac Medico
2021-01-17 8:49 Zac Medico
2020-09-08 2:52 Zac Medico
2020-08-09 0:15 Zac Medico
2020-08-04 3:16 Zac Medico
2020-08-03 23:28 Zac Medico
2020-08-03 23:28 Zac Medico
2020-08-03 21:42 Zac Medico
2020-08-03 21:42 Zac Medico
2020-08-03 19:30 Zac Medico
2020-07-22 20:14 Zac Medico
2020-07-22 19:52 Zac Medico
2020-07-22 17:46 Zac Medico
2020-06-07 3:26 Zac Medico
2020-02-20 9:55 Zac Medico
2020-02-03 3:04 Zac Medico
2019-08-24 3:15 Zac Medico
2019-06-20 19:43 Zac Medico
2019-05-11 21:16 Zac Medico
2019-01-20 6:55 Zac Medico
2019-01-11 10:14 Fabian Groffen
2018-09-24 7:30 Zac Medico
2018-09-24 0:46 Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox