* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-05-07 17:25 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-05-07 17:25 UTC (permalink / raw
To: gentoo-commits
commit: 48e1b835df5fefe49e2227e37c007d87fade1c55
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat May 7 17:25:20 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat May 7 17:25:20 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=48e1b835
PreserveLibsRegistry: add lock/unlock assertions
Also, add comments to store() about unobvious interaction with
locking due to atomic replacement of the inode.
---
.../util/_dyn_libs/PreservedLibsRegistry.py | 13 ++++++++++++-
1 files changed, 12 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index f3cbb33..3fb8120 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -36,11 +36,16 @@ class PreservedLibsRegistry(object):
def lock(self):
"""Grab an exclusive lock on the preserved libs registry."""
+ if self._lock is not None:
+ raise AssertionError("already locked")
self._lock = lockfile(self._filename)
def unlock(self):
"""Release our exclusive lock on the preserved libs registry."""
+ if self._lock is None:
+ raise AssertionError("not locked")
unlockfile(self._lock)
+ self._lock = None
def load(self):
""" Reload the registry data from file """
@@ -65,7 +70,13 @@ class PreservedLibsRegistry(object):
self.pruneNonExisting()
def store(self):
- """ Store the registry data to file """
+ """
+ Store the registry data to the file. The existing inode will be
+ replaced atomically, so if that inode is currently being used
+ for a lock then that lock will be rendered useless. Therefore,
+ it is important not to call this method until the current lock
+ is ready to be immediately released.
+ """
if os.environ.get("SANDBOX_ON") == "1" or \
self._data == self._data_orig:
return
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-05-08 5:02 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-05-08 5:02 UTC (permalink / raw
To: gentoo-commits
commit: 6d916753c07ccad4fcc596b155d776f297aeeb68
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 8 05:02:38 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 8 05:02:38 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6d916753
linkmap: check for access before locking vardbapi
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 7 +++++--
1 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 31aacbb..11e32b6 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -183,7 +183,9 @@ class LinkageMapELF(object):
lines.append((include_file, line))
aux_keys = [self._needed_aux_key]
- self._dbapi.lock()
+ can_lock = os.access(os.path.dirname(self._dbapi._dbroot), os.W_OK)
+ if can_lock:
+ self._dbapi.lock()
try:
for cpv in self._dbapi.cpv_all():
if exclude_pkgs is not None and cpv in exclude_pkgs:
@@ -193,7 +195,8 @@ class LinkageMapELF(object):
for line in self._dbapi.aux_get(cpv, aux_keys)[0].splitlines():
lines.append((needed_file, line))
finally:
- self._dbapi.unlock()
+ if can_lock:
+ self._dbapi.unlock()
# have to call scanelf for preserved libs here as they aren't
# registered in NEEDED.ELF.2 files
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-05-08 7:10 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-05-08 7:10 UTC (permalink / raw
To: gentoo-commits
commit: 07293a625e1b5ab086119ea1ccf7133eeb86fcee
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 8 07:09:37 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 8 07:09:37 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=07293a62
linkmap: use exclude_pkgs for the registry too
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 11 +++++++++--
1 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 11e32b6..56e6e0c 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -205,8 +205,15 @@ class LinkageMapELF(object):
plibs.update(preserve_paths)
if self._dbapi._plib_registry and \
self._dbapi._plib_registry.hasEntries():
- for items in \
- self._dbapi._plib_registry.getPreservedLibs().values():
+ for cpv, items in \
+ self._dbapi._plib_registry.getPreservedLibs().items():
+ if exclude_pkgs is not None and cpv in exclude_pkgs:
+ # These preserved libs will either be unmerged,
+ # rendering them irrelevant, or they will be
+ # preserved in the replacement package and are
+ # already represented via the preserve_paths
+ # parameter.
+ continue
plibs.update(items)
if plibs:
args = ["/usr/bin/scanelf", "-qF", "%a;%F;%S;%r;%n"]
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-05-08 21:31 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-05-08 21:31 UTC (permalink / raw
To: gentoo-commits
commit: e4320c84c71d61b41ba9c6016b06030ba1620a2f
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 8 21:31:11 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 8 21:31:11 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4320c84
PreservedLibsRegistry: normalize counter as str
---
.../util/_dyn_libs/PreservedLibsRegistry.py | 26 ++++++++++++++++---
1 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index 3fb8120..838cbaa 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -3,6 +3,7 @@
import errno
import logging
+import sys
try:
import cPickle as pickle
@@ -12,6 +13,7 @@ except ImportError:
from portage import os
from portage import _encodings
from portage import _os_merge
+from portage import _unicode_decode
from portage import _unicode_encode
from portage.exception import PermissionDenied
from portage.localization import _
@@ -20,6 +22,9 @@ from portage.util import writemsg_level
from portage.versions import cpv_getkey
from portage.locks import lockfile, unlockfile
+if sys.hexversion >= 0x3000000:
+ basestring = str
+
class PreservedLibsRegistry(object):
""" This class handles the tracking of preserved library objects """
def __init__(self, root, filename):
@@ -91,6 +96,17 @@ class PreservedLibsRegistry(object):
else:
self._data_orig = self._data.copy()
+ def _normalize_counter(self, counter):
+ """
+ For simplicity, normalize as a unicode string
+ and strip whitespace. This avoids the need for
+ int conversion and a possible ValueError resulting
+ from vardb corruption.
+ """
+ if not isinstance(counter, basestring):
+ counter = str(counter)
+ return _unicode_decode(counter).strip()
+
def register(self, cpv, slot, counter, paths):
""" Register new objects in the registry. If there is a record with the
same packagename (internally derived from cpv) and slot it is
@@ -99,19 +115,21 @@ class PreservedLibsRegistry(object):
@type cpv: CPV (as String)
@param slot: the value of SLOT of the given package instance
@type slot: String
- @param counter: vdb counter value for the package instace
- @type counter: Integer
+ @param counter: vdb counter value for the package instance
+ @type counter: String
@param paths: absolute paths of objects that got preserved during an update
@type paths: List
"""
cp = cpv_getkey(cpv)
cps = cp+":"+slot
+ counter = self._normalize_counter(counter)
if len(paths) == 0 and cps in self._data \
- and self._data[cps][0] == cpv and int(self._data[cps][1]) == int(counter):
+ and self._data[cps][0] == cpv and \
+ self._normalize_counter(self._data[cps][1]) == counter:
del self._data[cps]
elif len(paths) > 0:
self._data[cps] = (cpv, counter, paths)
-
+
def unregister(self, cpv, slot, counter):
""" Remove a previous registration of preserved objects for the given package.
@param cpv: package instance whose records should be removed
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-06-30 10:01 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-06-30 10:01 UTC (permalink / raw
To: gentoo-commits
commit: a472460f7d93b83bbc8b8c08d48f0a57ab7924d3
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 30 10:01:06 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jun 30 10:01:06 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a472460f
LinkageMapELF.isMasterLink(): literally compare
Comparing the lengths of the names alone seems like too much of an
assumption, so literally compare the beginning of the soname to the
basename of the given file.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 8 +++++---
1 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 4b23421..d4c490c 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -481,7 +481,9 @@ class LinkageMapELF(object):
def isMasterLink(self, obj):
"""
- Determine whether an object is a master link.
+ Determine whether an object is a "master" symlink, which means
+ that its basename is that same as the beginning part of the
+ soname and it lacks the soname's version component.
@param obj: absolute path to an object
@type obj: string (example: '/usr/bin/foo')
@@ -492,12 +494,12 @@ class LinkageMapELF(object):
"""
os = _os_merge
- basename = os.path.basename(obj)
obj_key = self._obj_key(obj)
if obj_key not in self._obj_properties:
raise KeyError("%s (%s) not in object list" % (obj_key, obj))
+ basename = os.path.basename(obj)
soname = self._obj_properties[obj_key][3]
- return (len(basename) < len(soname))
+ return len(basename) < len(soname) and soname.startswith(basename)
def listLibraryObjects(self):
"""
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-06-30 10:17 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-06-30 10:17 UTC (permalink / raw
To: gentoo-commits
commit: a3f0c99340a2784707d4613052074f9a73edcb49
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 30 10:17:31 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jun 30 10:17:31 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a3f0c993
Fix a typo in a docstring.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index d4c490c..4b9817e 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -482,7 +482,7 @@ class LinkageMapELF(object):
def isMasterLink(self, obj):
"""
Determine whether an object is a "master" symlink, which means
- that its basename is that same as the beginning part of the
+ that its basename is the same as the beginning part of the
soname and it lacks the soname's version component.
@param obj: absolute path to an object
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-01 1:37 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-01 1:37 UTC (permalink / raw
To: gentoo-commits
commit: 4951d8d694433995b382088cb36af7aa800719b7
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 1 01:33:22 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul 1 01:33:22 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4951d8d6
LinkageMapElf.isMasterLink(): handle libproc
The version component of the libproc-3.2.8.so soname is formed
slightly differently than most other libraries.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 11 ++++++++++-
1 files changed, 10 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 4b9817e..c56caae 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -485,6 +485,13 @@ class LinkageMapELF(object):
that its basename is the same as the beginning part of the
soname and it lacks the soname's version component.
+ Examples:
+
+ soname | master symlink name
+ --------------------------------------------
+ libarchive.so.2.8.4 | libarchive.so
+ libproc-3.2.8.so | libproc.so
+
@param obj: absolute path to an object
@type obj: string (example: '/usr/bin/foo')
@rtype: Boolean
@@ -499,7 +506,9 @@ class LinkageMapELF(object):
raise KeyError("%s (%s) not in object list" % (obj_key, obj))
basename = os.path.basename(obj)
soname = self._obj_properties[obj_key][3]
- return len(basename) < len(soname) and soname.startswith(basename)
+ return len(basename) < len(soname) and \
+ basename.endswith(".so") and \
+ soname.startswith(basename[:-3])
def listLibraryObjects(self):
"""
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-01 4:02 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-01 4:02 UTC (permalink / raw
To: gentoo-commits
commit: 8e5ff9b6d508eecc7d46eaddc674111c1ba45c42
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 1 02:19:14 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul 1 02:19:14 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8e5ff9b6
LinkageMapElf: clarify findConsumers soname code
Here it referred to an soname symlink as a "master" link, which was
inconsistent with the meaning of "master" link used in the
isMasterLink() method.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 8 ++++----
1 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index c56caae..f3a5129 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -654,21 +654,21 @@ class LinkageMapELF(object):
raise KeyError("%s (%s) not in object list" % (obj_key, obj))
# If there is another version of this lib with the
- # same soname and the master link points to that
+ # same soname and the soname symlink points to that
# other version, this lib will be shadowed and won't
# have any consumers.
if not isinstance(obj, self._ObjectKey):
soname = self._obj_properties[obj_key][3]
- master_link = os.path.join(self._root,
+ soname_link = os.path.join(self._root,
os.path.dirname(obj).lstrip(os.path.sep), soname)
try:
- master_st = os.stat(master_link)
+ soname_st = os.stat(soname_link)
obj_st = os.stat(obj)
except OSError:
pass
else:
if (obj_st.st_dev, obj_st.st_ino) != \
- (master_st.st_dev, master_st.st_ino):
+ (soname_st.st_dev, soname_st.st_ino):
return set()
# Determine the directory(ies) from the set of objects.
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-01 4:02 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-01 4:02 UTC (permalink / raw
To: gentoo-commits
commit: 038f142cffb8437dbb2e8588078e802d441c3860
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 1 02:23:23 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul 1 02:23:23 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=038f142c
LinkageMapElf.findConsumers(): fix $ROOT handling
The code which checks the soname symlink was missing a join with
$ROOT.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index f3a5129..c55d6d6 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -661,9 +661,10 @@ class LinkageMapELF(object):
soname = self._obj_properties[obj_key][3]
soname_link = os.path.join(self._root,
os.path.dirname(obj).lstrip(os.path.sep), soname)
+ obj_path = os.path.join(self._root, obj.lstrip(os.sep))
try:
soname_st = os.stat(soname_link)
- obj_st = os.stat(obj)
+ obj_st = os.stat(obj_path)
except OSError:
pass
else:
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-20 4:51 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-20 4:51 UTC (permalink / raw
To: gentoo-commits
commit: 31f7e4afee0255d93fbb7eb6c90ba719943cb883
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 20 04:44:14 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jul 20 04:44:14 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=31f7e4af
Fix unused case in LinkageMapELF.findConsumers().
Currently, we never pass in more that one package via the
exclude_providers argument, so we never trigger the flaw in the logic
that this fixes.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 20 ++++++++++++--------
1 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index c8e9acf..05043d7 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -697,14 +697,18 @@ class LinkageMapELF(object):
for provider_key in soname_node.providers:
provider_objs = self._obj_properties[provider_key][4]
for p in provider_objs:
- for excluded in exclude_providers:
- if not excluded(p):
- # This provider is not excluded. It will
- # satisfy a consumer of this soname if it
- # is in the default ld.so path or the
- # consumer's runpath.
- relevant_dir_keys.add(
- self._path_key(os.path.dirname(p)))
+ provider_excluded = False
+ for excluded_provider_isowner in exclude_providers:
+ if excluded_provider_isowner(p):
+ provider_excluded = True
+ break
+ if not provider_excluded:
+ # This provider is not excluded. It will
+ # satisfy a consumer of this soname if it
+ # is in the default ld.so path or the
+ # consumer's runpath.
+ relevant_dir_keys.add(
+ self._path_key(os.path.dirname(p)))
for consumer_key in soname_node.consumers:
_arch, _needed, path, _soname, _consumer_objs = \
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-20 8:05 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-20 8:05 UTC (permalink / raw
To: gentoo-commits
commit: 63c49337e2f56aa18e495396eb6b8d3d4a7d7e3a
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 20 08:04:00 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jul 20 08:04:00 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=63c49337
Optimize LinkageMapELF.findConsumers().
If there are no non-excluded providers then there's no need to search
for satisfied consumers.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 15 ++++++++-------
1 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 05043d7..218ce2a 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -710,13 +710,14 @@ class LinkageMapELF(object):
relevant_dir_keys.add(
self._path_key(os.path.dirname(p)))
- for consumer_key in soname_node.consumers:
- _arch, _needed, path, _soname, _consumer_objs = \
- self._obj_properties[consumer_key]
- path_keys = defpath_keys.copy()
- path_keys.update(self._path_key(x) for x in path)
- if relevant_dir_keys.intersection(path_keys):
- satisfied_consumer_keys.add(consumer_key)
+ if relevant_dir_keys:
+ for consumer_key in soname_node.consumers:
+ _arch, _needed, path, _soname, _consumer_objs = \
+ self._obj_properties[consumer_key]
+ path_keys = defpath_keys.copy()
+ path_keys.update(self._path_key(x) for x in path)
+ if relevant_dir_keys.intersection(path_keys):
+ satisfied_consumer_keys.add(consumer_key)
rValue = set()
if soname_node is not None:
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-22 8:03 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-22 8:03 UTC (permalink / raw
To: gentoo-commits
commit: 3f8a90683ddc542a0db58e00e55733ee8daaa4ee
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 22 07:58:56 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul 22 07:58:56 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3f8a9068
LinkageMapELF: optimize memory usage
This reduces memory consumption by approximately 30%, by replacing
mutable set instances with arrays, tuples, and frozensets where
appropriate. Also, identical frozenset instances are shared when
available.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 29 ++++++++++++++++++--------
1 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 218ce2a..49a7d35 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -265,6 +265,10 @@ class LinkageMapELF(object):
for x in plibs:
lines.append(("plibs", ";".join(['', x, '', '', ''])))
+ # Share identical frozenset instances when available,
+ # in order to conserve memory.
+ frozensets = {}
+
for location, l in lines:
l = l.rstrip("\n")
if not l:
@@ -278,21 +282,23 @@ class LinkageMapELF(object):
arch = fields[0]
obj = fields[1]
soname = fields[2]
- path = set([normalize_path(x) \
+ path = frozenset(normalize_path(x) \
for x in filter(None, fields[3].replace(
"${ORIGIN}", os.path.dirname(obj)).replace(
- "$ORIGIN", os.path.dirname(obj)).split(":"))])
- needed = [x for x in fields[4].split(",") if x]
+ "$ORIGIN", os.path.dirname(obj)).split(":")))
+ path = frozensets.setdefault(path, path)
+ needed = frozenset(x for x in fields[4].split(",") if x)
+ needed = frozensets.setdefault(needed, needed)
obj_key = self._obj_key(obj)
indexed = True
myprops = obj_properties.get(obj_key)
if myprops is None:
indexed = False
- myprops = (arch, needed, path, soname, set())
+ myprops = (arch, needed, path, soname, [])
obj_properties[obj_key] = myprops
# All object paths are added into the obj_properties tuple.
- myprops[4].add(obj)
+ myprops[4].append(obj)
# Don't index the same file more that once since only one
# set of data can be correct and therefore mixing data
@@ -309,16 +315,21 @@ class LinkageMapELF(object):
soname_map = arch_map.get(soname)
if soname_map is None:
soname_map = self._soname_map_class(
- providers=set(), consumers=set())
+ providers=[], consumers=[])
arch_map[soname] = soname_map
- soname_map.providers.add(obj_key)
+ soname_map.providers.append(obj_key)
for needed_soname in needed:
soname_map = arch_map.get(needed_soname)
if soname_map is None:
soname_map = self._soname_map_class(
- providers=set(), consumers=set())
+ providers=[], consumers=[])
arch_map[needed_soname] = soname_map
- soname_map.consumers.add(obj_key)
+ soname_map.consumers.append(obj_key)
+
+ for arch, sonames in libs.items():
+ for soname_node in sonames.values():
+ soname_node.providers = tuple(set(soname_node.providers))
+ soname_node.consumers = tuple(set(soname_node.consumers))
def listBrokenBinaries(self, debug=False):
"""
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-22 20:38 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-22 20:38 UTC (permalink / raw
To: gentoo-commits
commit: 53bce8a6d96f04acc97871712c107387d14f45e6
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 22 20:38:04 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jul 22 20:38:04 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=53bce8a6
LinkageMapELF: use a class for _obj_properies
This simplifies the interface, avoiding the need for hardcoded indexes
and making it easy to add new attributes.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 42 ++++++++++++++++++++-------
1 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 49a7d35..90d43e7 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -26,6 +26,25 @@ class LinkageMapELF(object):
_soname_map_class = slot_dict_class(
("consumers", "providers"), prefix="")
+ class _obj_properies_class(object):
+
+ __slots__ = ("arch", "needed", "runpaths", "soname", "alt_paths",)
+
+ def __init__(self, arch, needed, runpaths, soname, alt_paths):
+ self.arch = arch
+ self.needed = needed
+ self.runpaths = runpaths
+ self.soname = soname
+ self.alt_paths = alt_paths
+
+ def __iter__(self):
+ """Backward compatibility with 5-tuples."""
+ yield self.arch
+ yield self.needed
+ yield self.runpaths
+ yield self.soname
+ yield self.alt_paths
+
def __init__(self, vardbapi):
self._dbapi = vardbapi
self._root = self._dbapi.settings['ROOT']
@@ -295,10 +314,11 @@ class LinkageMapELF(object):
myprops = obj_properties.get(obj_key)
if myprops is None:
indexed = False
- myprops = (arch, needed, path, soname, [])
+ myprops = self._obj_properies_class(
+ arch, needed, path, soname, [])
obj_properties[obj_key] = myprops
# All object paths are added into the obj_properties tuple.
- myprops[4].append(obj)
+ myprops.alt_paths.append(obj)
# Don't index the same file more that once since only one
# set of data can be correct and therefore mixing data
@@ -430,7 +450,7 @@ class LinkageMapELF(object):
writemsg_level(
_("Found provider outside of findProviders:") + \
(" %s -> %s %s\n" % (os.path.join(directory, soname),
- self._obj_properties[cachedKey][4], libraries)),
+ self._obj_properties[cachedKey].alt_paths, libraries)),
level=logging.DEBUG,
noiselevel=-1)
# A valid library has been found, so there is no need to
@@ -516,7 +536,7 @@ class LinkageMapELF(object):
if obj_key not in self._obj_properties:
raise KeyError("%s (%s) not in object list" % (obj_key, obj))
basename = os.path.basename(obj)
- soname = self._obj_properties[obj_key][3]
+ soname = self._obj_properties[obj_key].soname
return len(basename) < len(soname) and \
basename.endswith(".so") and \
soname.startswith(basename[:-3])
@@ -537,7 +557,7 @@ class LinkageMapELF(object):
for arch_map in self._libs.values():
for soname_map in arch_map.values():
for obj_key in soname_map.providers:
- rValue.extend(self._obj_properties[obj_key][4])
+ rValue.extend(self._obj_properties[obj_key].alt_paths)
return rValue
def getSoname(self, obj):
@@ -556,10 +576,10 @@ class LinkageMapELF(object):
obj_key = obj
if obj_key not in self._obj_properties:
raise KeyError("%s not in object list" % obj_key)
- return self._obj_properties[obj_key][3]
+ return self._obj_properties[obj_key].soname
if obj not in self._obj_key_cache:
raise KeyError("%s not in object list" % obj)
- return self._obj_properties[self._obj_key_cache[obj]][3]
+ return self._obj_properties[self._obj_key_cache[obj]].soname
def findProviders(self, obj):
"""
@@ -608,7 +628,7 @@ class LinkageMapELF(object):
# For each potential provider of the soname, add it to rValue if it
# resides in the obj's runpath.
for provider_key in self._libs[arch][soname].providers:
- providers = self._obj_properties[provider_key][4]
+ providers = self._obj_properties[provider_key].alt_paths
for provider in providers:
if self._path_key(os.path.dirname(provider)) in path_keys:
rValue[soname].add(provider)
@@ -667,7 +687,7 @@ class LinkageMapELF(object):
obj_key = obj
if obj_key not in self._obj_properties:
raise KeyError("%s not in object list" % obj_key)
- objs = self._obj_properties[obj_key][4]
+ objs = self._obj_properties[obj_key].alt_paths
else:
objs = set([obj])
obj_key = self._obj_key(obj)
@@ -679,7 +699,7 @@ class LinkageMapELF(object):
# other version, this lib will be shadowed and won't
# have any consumers.
if not isinstance(obj, self._ObjectKey):
- soname = self._obj_properties[obj_key][3]
+ soname = self._obj_properties[obj_key].soname
soname_link = os.path.join(self._root,
os.path.dirname(obj).lstrip(os.path.sep), soname)
obj_path = os.path.join(self._root, obj.lstrip(os.sep))
@@ -706,7 +726,7 @@ class LinkageMapELF(object):
if exclude_providers is not None:
relevant_dir_keys = set()
for provider_key in soname_node.providers:
- provider_objs = self._obj_properties[provider_key][4]
+ provider_objs = self._obj_properties[provider_key].alt_paths
for p in provider_objs:
provider_excluded = False
for excluded_provider_isowner in exclude_providers:
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-23 6:23 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-23 6:23 UTC (permalink / raw
To: gentoo-commits
commit: f0ef754b7dafe892ce58725b32e2146904414b7e
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 23 06:22:52 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jul 23 06:22:52 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f0ef754b
LinkageMapELF: remove 5-tuple unpack code
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 39 +++++++++++++++-----------
1 files changed, 22 insertions(+), 17 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index c3c0706..b616292 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -39,14 +39,6 @@ class LinkageMapELF(object):
self.alt_paths = alt_paths
self.owner = owner
- def __iter__(self):
- """Backward compatibility with 5-tuples."""
- yield self.arch
- yield self.needed
- yield self.runpaths
- yield self.soname
- yield self.alt_paths
-
def __init__(self, vardbapi):
self._dbapi = vardbapi
self._root = self._dbapi.settings['ROOT']
@@ -405,8 +397,13 @@ class LinkageMapELF(object):
if obj_key.file_exists():
# Get the arch and soname from LinkageMap._obj_properties if
# it exists. Otherwise, None.
- arch, _needed, _path, soname, _objs = \
- self._obj_properties.get(obj_key, (None,)*5)
+ obj_props = self._obj_properties.get(obj_key)
+ if obj_props is None:
+ arch = None
+ soname = None
+ else:
+ arch = obj_props.arch
+ soname = obj_props.soname
return cache_self.cache.setdefault(obj, \
(arch, soname, obj_key, True))
else:
@@ -419,7 +416,10 @@ class LinkageMapELF(object):
# Iterate over all obj_keys and their providers.
for obj_key, sonames in providers.items():
- arch, _needed, path, _soname, objs = self._obj_properties[obj_key]
+ obj_props = self._obj_properties[obj_key]
+ arch = obj_props.arch
+ path = obj_props.runpaths
+ objs = obj_props.alt_paths
path = path.union(self._defpath)
# Iterate over each needed soname and the set of library paths that
# fulfill the soname to determine if the dependency is broken.
@@ -647,7 +647,10 @@ class LinkageMapELF(object):
if obj_key not in self._obj_properties:
raise KeyError("%s (%s) not in object list" % (obj_key, obj))
- arch, needed, path, _soname, _objs = self._obj_properties[obj_key]
+ obj_props = self._obj_properties[obj_key]
+ arch = obj_props.arch
+ needed = obj_props.needed
+ path = obj_props.runpaths
path_keys = set(self._path_key(x) for x in path.union(self._defpath))
for soname in needed:
rValue[soname] = set()
@@ -741,7 +744,9 @@ class LinkageMapELF(object):
(soname_st.st_dev, soname_st.st_ino):
return set()
- arch, _needed, _path, soname, _objs = self._obj_properties[obj_key]
+ obj_props = self._obj_properties[obj_key]
+ arch = obj_props.arch
+ soname = obj_props.soname
soname_node = None
arch_map = self._libs.get(arch)
@@ -771,8 +776,7 @@ class LinkageMapELF(object):
if relevant_dir_keys:
for consumer_key in soname_node.consumers:
- _arch, _needed, path, _soname, _consumer_objs = \
- self._obj_properties[consumer_key]
+ path = self._obj_properties[consumer_key].runpaths
path_keys = defpath_keys.copy()
path_keys.update(self._path_key(x) for x in path)
if relevant_dir_keys.intersection(path_keys):
@@ -787,8 +791,9 @@ class LinkageMapELF(object):
for consumer_key in soname_node.consumers:
if consumer_key in satisfied_consumer_keys:
continue
- _arch, _needed, path, _soname, consumer_objs = \
- self._obj_properties[consumer_key]
+ consumer_props = self._obj_properties[consumer_key]
+ path = consumer_props.runpaths
+ consumer_objs = consumer_props.alt_paths
path_keys = defpath_keys.union(self._path_key(x) for x in path)
if objs_dir_keys.intersection(path_keys):
rValue.update(consumer_objs)
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-24 2:15 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-24 2:15 UTC (permalink / raw
To: gentoo-commits
commit: 757631f0de7b3baf5295c1d0fecfe1008c1ab1f2
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 24 02:14:52 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul 24 02:14:52 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=757631f0
LinkageMapELF: fix reversed x, cpv var refs
This was an error in commit f393413c3f823ef4a60acfcc41c3920933510fc1.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index b616292..2ad97bb 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -275,7 +275,7 @@ class LinkageMapELF(object):
# preserved library has an entry in self._obj_properties. This
# is important in order to prevent findConsumers from raising
# an unwanted KeyError.
- for cpv, x in plibs.items():
+ for x, cpv in plibs.items():
lines.append((cpv, "plibs", ";".join(['', x, '', '', ''])))
# Share identical frozenset instances when available,
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-24 2:58 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-24 2:58 UTC (permalink / raw
To: gentoo-commits
commit: 2fb34f15fa81696b8b31c341e7db506f5da6ee67
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 24 02:57:52 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul 24 02:57:52 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=2fb34f15
LinkageMapELF.getOwners(): add note about plibs
For preserved libraries, the owner(s) may have been been previously
uninstalled, but these uninstalled owners can be returned by this
method since they are registered in the PreservedLibsRegistry.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 5 +++++
1 files changed, 5 insertions(+), 0 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 2ad97bb..cf8def3 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -568,6 +568,11 @@ class LinkageMapELF(object):
if the object is unknown. Returns an empty tuple if the owner(s)
are unknown.
+ NOTE: For preserved libraries, the owner(s) may have been been
+ previously uninstalled, but these uninstalled owners can be
+ returned by this method since they are registered in the
+ PreservedLibsRegistry.
+
@param obj: absolute path to an object
@type obj: string (example: '/usr/bin/bar')
@rtype: tuple
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2011-07-24 2:58 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2011-07-24 2:58 UTC (permalink / raw
To: gentoo-commits
commit: 57ab2c80af9e4990411f51f8682d340926eea969
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 24 02:58:43 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul 24 02:58:43 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=57ab2c80
Fix a typo in the previous commit.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index cf8def3..52670d9 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -568,7 +568,7 @@ class LinkageMapELF(object):
if the object is unknown. Returns an empty tuple if the owner(s)
are unknown.
- NOTE: For preserved libraries, the owner(s) may have been been
+ NOTE: For preserved libraries, the owner(s) may have been
previously uninstalled, but these uninstalled owners can be
returned by this method since they are registered in the
PreservedLibsRegistry.
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2012-02-01 2:04 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2012-02-01 2:04 UTC (permalink / raw
To: gentoo-commits
commit: 972866403e8adc992625d2ff5e608c028b08bf30
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 1 02:04:11 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 1 02:04:11 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=97286640
LinkageMapELF: handle null bytes in NEEDED.ELF.2
This avoids an error from os.stat:
TypeError: must be encoded string without NULL bytes, not str
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 7 +++++++
1 files changed, 7 insertions(+), 0 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index f751357..e71ac73 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -287,6 +287,13 @@ class LinkageMapELF(object):
l = l.rstrip("\n")
if not l:
continue
+ if '\0' in l:
+ # os.stat() will raise "TypeError: must be encoded string
+ # without NULL bytes, not str" in this case.
+ writemsg_level(_("\nLine contains null byte(s) " \
+ "in %s: %s\n\n") % (location, l),
+ level=logging.ERROR, noiselevel=-1)
+ continue
fields = l.split(";")
if len(fields) < 5:
writemsg_level(_("\nWrong number of fields " \
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2012-02-18 2:06 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2012-02-18 2:06 UTC (permalink / raw
To: gentoo-commits
commit: 42adf1d5d2be6f186e27bfa2a808cfd1b065dd32
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 18 02:01:27 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 18 02:01:27 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=42adf1d5
PreservedLibsRegistry: add JSON read/write
Support serialization as JSON instead of pickle, so that
/var/lib/portage/preserved_libs_registry is human readable/writable,
for those rare cases where it may be useful. Currently, pickle is still
used for writes. The plan is to migrate to JSON after JSON read has
been supported for some time.
---
.../util/_dyn_libs/PreservedLibsRegistry.py | 61 ++++++++++++++++---
1 files changed, 51 insertions(+), 10 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index 510c390..90d6732 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -1,7 +1,8 @@
-# Copyright 1998-2011 Gentoo Foundation
+# Copyright 1998-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import errno
+import json
import logging
import sys
@@ -27,6 +28,19 @@ if sys.hexversion >= 0x3000000:
class PreservedLibsRegistry(object):
""" This class handles the tracking of preserved library objects """
+
+ # Enable this after JSON read has been supported for some time.
+ _json_write = False
+
+ _json_write_opts = {
+ "ensure_ascii": False,
+ "indent": "\t",
+ "sort_keys": True
+ }
+ if sys.hexversion < 0x3020000:
+ # indent only supports int number of spaces
+ _json_write_opts["indent"] = 4
+
def __init__(self, root, filename):
"""
@param root: root used to check existence of paths in pruneNonExisting
@@ -56,17 +70,11 @@ class PreservedLibsRegistry(object):
""" Reload the registry data from file """
self._data = None
f = None
+ content = None
try:
f = open(_unicode_encode(self._filename,
encoding=_encodings['fs'], errors='strict'), 'rb')
- if os.fstat(f.fileno()).st_size == 0:
- # ignore empty lock file
- pass
- else:
- self._data = pickle.load(f)
- except (AttributeError, EOFError, ValueError, pickle.UnpicklingError) as e:
- writemsg_level(_("!!! Error loading '%s': %s\n") % \
- (self._filename, e), level=logging.ERROR, noiselevel=-1)
+ content = f.read()
except EnvironmentError as e:
if not hasattr(e, 'errno'):
raise
@@ -79,8 +87,33 @@ class PreservedLibsRegistry(object):
finally:
if f is not None:
f.close()
+
+ # content is empty if it's an empty lock file
+ if content:
+ try:
+ self._data = pickle.loads(content)
+ except SystemExit:
+ raise
+ except Exception as e:
+ try:
+ self._data = json.loads(_unicode_decode(content,
+ encoding=_encodings['repo.content'], errors='strict'))
+ except SystemExit:
+ raise
+ except Exception:
+ writemsg_level(_("!!! Error loading '%s': %s\n") %
+ (self._filename, e), level=logging.ERROR,
+ noiselevel=-1)
+
if self._data is None:
self._data = {}
+ else:
+ for k, v in self._data.items():
+ if isinstance(v, (list, tuple)) and len(v) == 3 and \
+ isinstance(v[2], set):
+ # convert set to list, for write with JSONEncoder
+ self._data[k] = (v[0], v[1], list(v[2]))
+
self._data_orig = self._data.copy()
self.pruneNonExisting()
@@ -97,7 +130,12 @@ class PreservedLibsRegistry(object):
return
try:
f = atomic_ofstream(self._filename, 'wb')
- pickle.dump(self._data, f, protocol=2)
+ if self._json_write:
+ f.write(_unicode_encode(
+ json.dumps(self._data, f, **self._json_write_opts),
+ encoding=_encodings['repo.content'], errors='strict'))
+ else:
+ pickle.dump(self._data, f, protocol=2)
f.close()
except EnvironmentError as e:
if e.errno != PermissionDenied.errno:
@@ -138,6 +176,9 @@ class PreservedLibsRegistry(object):
self._normalize_counter(self._data[cps][1]) == counter:
del self._data[cps]
elif len(paths) > 0:
+ if isinstance(paths, set):
+ # convert set to list, for write with JSONEncoder
+ paths = list(paths)
self._data[cps] = (cpv, counter, paths)
def unregister(self, cpv, slot, counter):
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2012-02-18 3:11 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2012-02-18 3:11 UTC (permalink / raw
To: gentoo-commits
commit: 4c8d1060304a4e793c5d813a1d925e7cf1fcccbd
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 18 02:01:27 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 18 03:11:18 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4c8d1060
PreservedLibsRegistry: add JSON read/write
Support serialization as JSON instead of pickle, so that
/var/lib/portage/preserved_libs_registry is human readable/writable,
for those rare cases where it may be useful. Currently, pickle is still
used for writes. The plan is to migrate to JSON after JSON read has
been supported for some time.
---
.../util/_dyn_libs/PreservedLibsRegistry.py | 61 ++++++++++++++++---
1 files changed, 51 insertions(+), 10 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index 510c390..405a23a 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -1,7 +1,8 @@
-# Copyright 1998-2011 Gentoo Foundation
+# Copyright 1998-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import errno
+import json
import logging
import sys
@@ -27,6 +28,19 @@ if sys.hexversion >= 0x3000000:
class PreservedLibsRegistry(object):
""" This class handles the tracking of preserved library objects """
+
+ # Enable this after JSON read has been supported for some time.
+ _json_write = False
+
+ _json_write_opts = {
+ "ensure_ascii": False,
+ "indent": "\t",
+ "sort_keys": True
+ }
+ if sys.hexversion < 0x3020000:
+ # indent only supports int number of spaces
+ _json_write_opts["indent"] = 4
+
def __init__(self, root, filename):
"""
@param root: root used to check existence of paths in pruneNonExisting
@@ -56,17 +70,11 @@ class PreservedLibsRegistry(object):
""" Reload the registry data from file """
self._data = None
f = None
+ content = None
try:
f = open(_unicode_encode(self._filename,
encoding=_encodings['fs'], errors='strict'), 'rb')
- if os.fstat(f.fileno()).st_size == 0:
- # ignore empty lock file
- pass
- else:
- self._data = pickle.load(f)
- except (AttributeError, EOFError, ValueError, pickle.UnpicklingError) as e:
- writemsg_level(_("!!! Error loading '%s': %s\n") % \
- (self._filename, e), level=logging.ERROR, noiselevel=-1)
+ content = f.read()
except EnvironmentError as e:
if not hasattr(e, 'errno'):
raise
@@ -79,8 +87,33 @@ class PreservedLibsRegistry(object):
finally:
if f is not None:
f.close()
+
+ # content is empty if it's an empty lock file
+ if content:
+ try:
+ self._data = pickle.loads(content)
+ except SystemExit:
+ raise
+ except Exception as e:
+ try:
+ self._data = json.loads(_unicode_decode(content,
+ encoding=_encodings['repo.content'], errors='strict'))
+ except SystemExit:
+ raise
+ except Exception:
+ writemsg_level(_("!!! Error loading '%s': %s\n") %
+ (self._filename, e), level=logging.ERROR,
+ noiselevel=-1)
+
if self._data is None:
self._data = {}
+ else:
+ for k, v in self._data.items():
+ if isinstance(v, (list, tuple)) and len(v) == 3 and \
+ isinstance(v[2], set):
+ # convert set to list, for write with JSONEncoder
+ self._data[k] = (v[0], v[1], list(v[2]))
+
self._data_orig = self._data.copy()
self.pruneNonExisting()
@@ -97,7 +130,12 @@ class PreservedLibsRegistry(object):
return
try:
f = atomic_ofstream(self._filename, 'wb')
- pickle.dump(self._data, f, protocol=2)
+ if self._json_write:
+ f.write(_unicode_encode(
+ json.dumps(self._data, **self._json_write_opts),
+ encoding=_encodings['repo.content'], errors='strict'))
+ else:
+ pickle.dump(self._data, f, protocol=2)
f.close()
except EnvironmentError as e:
if e.errno != PermissionDenied.errno:
@@ -138,6 +176,9 @@ class PreservedLibsRegistry(object):
self._normalize_counter(self._data[cps][1]) == counter:
del self._data[cps]
elif len(paths) > 0:
+ if isinstance(paths, set):
+ # convert set to list, for write with JSONEncoder
+ paths = list(paths)
self._data[cps] = (cpv, counter, paths)
def unregister(self, cpv, slot, counter):
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2012-03-05 7:32 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2012-03-05 7:32 UTC (permalink / raw
To: gentoo-commits
commit: 32d19be14e22ada479963ba8627452f5f2d89b94
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 5 07:30:28 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Mar 5 07:30:28 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=32d19be1
pruneNonExisting: handle eselect-opengl symlinks
Only count symlinks as preserved if they still point to a hardink
in the same directory, in order to handle cases where a tool such
as eselect-opengl has updated the symlink to point to a hardlink
in a different directory (see bug #406837). The unused hardlink is
automatically found by _find_unused_preserved_libs, since the soname
symlink no longer points to it. After the hardlink is removed by
_remove_preserved_libs, it calls pruneNonExisting which eliminates
the irrelevant symlink from the registry here.
---
.../util/_dyn_libs/PreservedLibsRegistry.py | 36 ++++++++++++++++++--
1 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index 405a23a..7d4708b 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -4,6 +4,7 @@
import errno
import json
import logging
+import stat
import sys
try:
@@ -196,9 +197,38 @@ class PreservedLibsRegistry(object):
os = _os_merge
for cps in list(self._data):
- cpv, counter, paths = self._data[cps]
- paths = [f for f in paths \
- if os.path.exists(os.path.join(self._root, f.lstrip(os.sep)))]
+ cpv, counter, _paths = self._data[cps]
+
+ paths = []
+ hardlinks = set()
+ symlinks = {}
+ for f in _paths:
+ f_abs = os.path.join(self._root, f.lstrip(os.sep))
+ try:
+ lst = os.lstat(f_abs)
+ except OSError:
+ continue
+ if stat.S_ISLNK(lst.st_mode):
+ try:
+ symlinks[f] = os.readlink(f_abs)
+ except OSError:
+ continue
+ elif stat.S_ISREG(lst.st_mode):
+ hardlinks.add(f)
+ paths.append(f)
+
+ # Only count symlinks as preserved if they still point to a hardink
+ # in the same directory, in order to handle cases where a tool such
+ # as eselect-opengl has updated the symlink to point to a hardlink
+ # in a different directory (see bug #406837). The unused hardlink
+ # is automatically found by _find_unused_preserved_libs, since the
+ # soname symlink no longer points to it. After the hardlink is
+ # removed by _remove_preserved_libs, it calls pruneNonExisting
+ # which eliminates the irrelevant symlink from the registry here.
+ for f, target in symlinks.items():
+ if os.path.join(os.path.dirname(f), target) in hardlinks:
+ paths.append(f)
+
if len(paths) > 0:
self._data[cps] = (cpv, counter, paths)
else:
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2012-03-21 21:24 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2012-03-21 21:24 UTC (permalink / raw
To: gentoo-commits
commit: 716b1864e03212b09f94ddf3573fbfa451473558
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 21 21:24:15 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar 21 21:24:15 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=716b1864
PreservedLibsRegistry: enable JSON write support
JSON read support has been available since portage-2.2.0_alpha89.
---
.../util/_dyn_libs/PreservedLibsRegistry.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index 7d4708b..d774f80 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -30,8 +30,8 @@ if sys.hexversion >= 0x3000000:
class PreservedLibsRegistry(object):
""" This class handles the tracking of preserved library objects """
- # Enable this after JSON read has been supported for some time.
- _json_write = False
+ # JSON read support has been available since portage-2.2.0_alpha89.
+ _json_write = True
_json_write_opts = {
"ensure_ascii": False,
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2012-03-22 15:24 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2012-03-22 15:24 UTC (permalink / raw
To: gentoo-commits
commit: 0b4cf4cc31684bd1800fd1588f7114d8fe45d137
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 22 15:23:51 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Mar 22 15:23:51 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0b4cf4cc
PreservedLibsRegistry: try to parse as JSON first
---
.../util/_dyn_libs/PreservedLibsRegistry.py | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index d774f80..d3d100f 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -92,13 +92,13 @@ class PreservedLibsRegistry(object):
# content is empty if it's an empty lock file
if content:
try:
- self._data = pickle.loads(content)
+ self._data = json.loads(_unicode_decode(content,
+ encoding=_encodings['repo.content'], errors='strict'))
except SystemExit:
raise
except Exception as e:
try:
- self._data = json.loads(_unicode_decode(content,
- encoding=_encodings['repo.content'], errors='strict'))
+ self._data = pickle.loads(content)
except SystemExit:
raise
except Exception:
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2013-02-22 18:52 Arfrever Frehtes Taifersar Arahesis
0 siblings, 0 replies; 32+ messages in thread
From: Arfrever Frehtes Taifersar Arahesis @ 2013-02-22 18:52 UTC (permalink / raw
To: gentoo-commits
commit: 26dd7d649bcb158210653e0896a5bd3de84511d4
Author: Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Fri Feb 22 18:51:52 2013 +0000
Commit: Arfrever Frehtes Taifersar Arahesis <arfrever.fta <AT> gmail <DOT> com>
CommitDate: Fri Feb 22 18:51:52 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=26dd7d64
Fix a typo.
---
pym/portage/util/_dyn_libs/LinkageMapELF.py | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index e71ac73..8d0c09d 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2011 Gentoo Foundation
+# Copyright 1998-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -26,7 +26,7 @@ class LinkageMapELF(object):
_soname_map_class = slot_dict_class(
("consumers", "providers"), prefix="")
- class _obj_properies_class(object):
+ class _obj_properties_class(object):
__slots__ = ("arch", "needed", "runpaths", "soname", "alt_paths",
"owner",)
@@ -316,7 +316,7 @@ class LinkageMapELF(object):
myprops = obj_properties.get(obj_key)
if myprops is None:
indexed = False
- myprops = self._obj_properies_class(
+ myprops = self._obj_properties_class(
arch, needed, path, soname, [], owner)
obj_properties[obj_key] = myprops
# All object paths are added into the obj_properties tuple.
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2013-03-18 23:35 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2013-03-18 23:35 UTC (permalink / raw
To: gentoo-commits
commit: 523618c10f92e78221cb1a5b26dfc51e945ad323
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 18 23:35:38 2013 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Mar 18 23:35:38 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=523618c1
display_preserved_libs: distinguish prsvd cnsmrs
This will fix bug #461908.
---
.../util/_dyn_libs/display_preserved_libs.py | 35 +++++++++++++++----
1 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/display_preserved_libs.py b/pym/portage/util/_dyn_libs/display_preserved_libs.py
index bcb7827..420bd2e 100644
--- a/pym/portage/util/_dyn_libs/display_preserved_libs.py
+++ b/pym/portage/util/_dyn_libs/display_preserved_libs.py
@@ -1,4 +1,4 @@
-# Copyright 2007-2012 Gentoo Foundation
+# Copyright 2007-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
@@ -50,6 +50,9 @@ def display_preserved_libs(vardb):
if owner_set:
owners[f] = owner_set
+ all_preserved = set()
+ all_preserved.update(*plibdata.values())
+
for cpv in plibdata:
print(colorize("WARN", ">>>") + " package: %s" % cpv)
samefile_map = {}
@@ -67,13 +70,29 @@ def display_preserved_libs(vardb):
print(colorize("WARN", " * ") + " - %s" % (p,))
f = alt_paths[0]
consumers = consumer_map.get(f, [])
- for c in consumers[:MAX_DISPLAY]:
- print(colorize("WARN", " * ") + " used by %s (%s)" % \
- (c, ", ".join(x.mycpv for x in owners.get(c, []))))
+ consumers_non_preserved = [c for x in consumers
+ if c not in all_preserved]
+ if consumers_non_preserved:
+ # Filter the consumers that are preserved libraries, since
+ # they don't need to be rebuilt (see bug #461908).
+ consumers = consumers_non_preserved
+
if len(consumers) == MAX_DISPLAY + 1:
+ # Display 1 extra consumer, instead of displaying
+ # "used by 1 other files".
+ max_display = MAX_DISPLAY + 1
+ else:
+ max_display = MAX_DISPLAY
+ for c in consumers[:max_display]:
+ if c in all_preserved:
+ # The owner is displayed elsewhere due to having
+ # its libs preserved, so distinguish this special
+ # case (see bug #461908).
+ owners_desc = "preserved"
+ else:
+ owners_desc = ", ".join(x.mycpv for x in owners.get(c, []))
print(colorize("WARN", " * ") + " used by %s (%s)" % \
- (consumers[MAX_DISPLAY], ", ".join(x.mycpv \
- for x in owners.get(consumers[MAX_DISPLAY], []))))
- elif len(consumers) > MAX_DISPLAY:
+ (c, owners_desc))
+ if len(consumers) > max_display:
print(colorize("WARN", " * ") + " used by %d other files" %
- (len(consumers) - MAX_DISPLAY))
+ (len(consumers) - max_display))
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2013-03-18 23:43 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2013-03-18 23:43 UTC (permalink / raw
To: gentoo-commits
commit: 8f990c85819c8f5f1c6dba89ab17fa4e122cfba0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 18 23:35:38 2013 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Mar 18 23:42:29 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8f990c85
display_preserved_libs: distinguish prsvd cnsmrs
This will fix bug #461908.
---
.../util/_dyn_libs/display_preserved_libs.py | 35 +++++++++++++++----
1 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/display_preserved_libs.py b/pym/portage/util/_dyn_libs/display_preserved_libs.py
index bcb7827..238274f 100644
--- a/pym/portage/util/_dyn_libs/display_preserved_libs.py
+++ b/pym/portage/util/_dyn_libs/display_preserved_libs.py
@@ -1,4 +1,4 @@
-# Copyright 2007-2012 Gentoo Foundation
+# Copyright 2007-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
@@ -50,6 +50,9 @@ def display_preserved_libs(vardb):
if owner_set:
owners[f] = owner_set
+ all_preserved = set()
+ all_preserved.update(*plibdata.values())
+
for cpv in plibdata:
print(colorize("WARN", ">>>") + " package: %s" % cpv)
samefile_map = {}
@@ -67,13 +70,29 @@ def display_preserved_libs(vardb):
print(colorize("WARN", " * ") + " - %s" % (p,))
f = alt_paths[0]
consumers = consumer_map.get(f, [])
- for c in consumers[:MAX_DISPLAY]:
- print(colorize("WARN", " * ") + " used by %s (%s)" % \
- (c, ", ".join(x.mycpv for x in owners.get(c, []))))
+ consumers_non_preserved = [c for c in consumers
+ if c not in all_preserved]
+ if consumers_non_preserved:
+ # Filter the consumers that are preserved libraries, since
+ # they don't need to be rebuilt (see bug #461908).
+ consumers = consumers_non_preserved
+
if len(consumers) == MAX_DISPLAY + 1:
+ # Display 1 extra consumer, instead of displaying
+ # "used by 1 other files".
+ max_display = MAX_DISPLAY + 1
+ else:
+ max_display = MAX_DISPLAY
+ for c in consumers[:max_display]:
+ if c in all_preserved:
+ # The owner is displayed elsewhere due to having
+ # its libs preserved, so distinguish this special
+ # case (see bug #461908).
+ owners_desc = "preserved"
+ else:
+ owners_desc = ", ".join(x.mycpv for x in owners.get(c, []))
print(colorize("WARN", " * ") + " used by %s (%s)" % \
- (consumers[MAX_DISPLAY], ", ".join(x.mycpv \
- for x in owners.get(consumers[MAX_DISPLAY], []))))
- elif len(consumers) > MAX_DISPLAY:
+ (c, owners_desc))
+ if len(consumers) > max_display:
print(colorize("WARN", " * ") + " used by %d other files" %
- (len(consumers) - MAX_DISPLAY))
+ (len(consumers) - max_display))
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2015-04-24 17:53 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2015-04-24 17:53 UTC (permalink / raw
To: gentoo-commits
commit: a47407e864d9a5a43d6f05ef06b46e34ff0e0c24
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 23 17:47:03 2015 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Apr 24 17:51:45 2015 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=a47407e8
LinkageMapElf.rebuild: pass error_leader to varexpand (bug 542796)
Since commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de,
LinkageMapElf.rebuild could produce mysterious "bad substitution"
messages.
Fixes: f1c1b8a77eeb ("Generate soname dependency metadata (bug 282639)")
X-Gentoo-Bug: 542796
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=542796
X-Gentoo-forum-thread: https://forums.gentoo.org/viewtopic-t-1014842.html
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>
pym/portage/util/_dyn_libs/LinkageMapELF.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index c44666a..f4d8b5d 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -339,7 +339,8 @@ class LinkageMapELF(object):
obj = entry.filename
soname = entry.soname
expand = {"ORIGIN": os.path.dirname(entry.filename)}
- path = frozenset(normalize_path(varexpand(x, expand))
+ path = frozenset(normalize_path(
+ varexpand(x, expand, error_leader=lambda: "%s: " % location))
for x in entry.runpaths)
path = frozensets.setdefault(path, path)
needed = frozenset(entry.needed)
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2015-06-02 3:48 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2015-06-02 3:48 UTC (permalink / raw
To: gentoo-commits
commit: c3ef903d1c95ce28922a2d7dcb00cc7a1d690201
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 31 18:42:49 2015 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jun 2 03:47:14 2015 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c3ef903d
_approx_multilib_categories: rename ia to ia64 (bug 550898)
In LinkageMapELF.py, there's a fallback mapping for
multilib categories, which is used by the LinkMapElf class when
NEEDED.ELF.2 does not contain multilib categories due to being
generated by older portage. This mapping should be consistent with
the mutilib categories generated by the compute_multilib_category
function which was modified by commit
b50bbcb70506254d66937dac376056bbf99b3fe9.
Fixes: b50bbcb70506 ("multilib: use ia64 as ia64 multilib name")
X-Gentoo-Bug: 550898
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=550898
Acked-by: Mike Frysinger <vapier <AT> gentoo.org>
pym/portage/util/_dyn_libs/LinkageMapELF.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index f4d8b5d..63e2213 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -30,7 +30,7 @@ _approx_multilib_categories = {
"AARCH64": "arm_64",
"ALPHA": "alpha_64",
"ARM": "arm_32",
- "IA_64": "ia_64",
+ "IA_64": "ia64_64",
"MIPS": "mips_o32",
"PARISC": "hppa_64",
"PPC": "ppc_32",
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2016-06-06 16:03 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2016-06-06 16:03 UTC (permalink / raw
To: gentoo-commits
commit: 54d3676d4dc90444895c99c0b6c29d6be6a25b77
Author: Benda Xu <heroxbd <AT> gentoo <DOT> org>
AuthorDate: Mon Jun 6 16:00:59 2016 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Jun 6 16:00:59 2016 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=54d3676d
LinkageMapELF: Account for EPREFIX in scanelf path (bug 583754)
X-Gentoo-Bug: 583754
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=583754
pym/portage/util/_dyn_libs/LinkageMapELF.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 63e2213..0b09fe5 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2013 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -11,6 +11,7 @@ from portage import _os_merge
from portage import _unicode_decode
from portage import _unicode_encode
from portage.cache.mappings import slot_dict_class
+from portage.const import EPREFIX
from portage.exception import CommandNotFound, InvalidData
from portage.localization import _
from portage.util import getlibpaths
@@ -259,7 +260,7 @@ class LinkageMapELF(object):
continue
plibs.update((x, cpv) for x in items)
if plibs:
- args = ["/usr/bin/scanelf", "-qF", "%a;%F;%S;%r;%n"]
+ args = [os.path.join(EPREFIX or "/", "usr/bin/scanelf"), "-qF", "%a;%F;%S;%r;%n"]
args.extend(os.path.join(root, x.lstrip("." + os.sep)) \
for x in plibs)
try:
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2016-12-27 21:43 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2016-12-27 21:43 UTC (permalink / raw
To: gentoo-commits
commit: cbaee3c3b28f52947c142da8df24d6b0817962f9
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 27 04:17:01 2016 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Dec 27 21:25:06 2016 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=cbaee3c3
LinkageMapELF: compute multilib category for preserved libs (bug 598080)
When support for parsing ELF headers in order to compute multilib
category was added in commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de,
the LinkageMapELF class was not updated to do the same for preserved
libraries. This has resulted in incorrect preserve-libs handling
as reported in bug 598080, for ABIs including x32 where the
_approx_multilib_categories mapping is insufficient. This patch fixes
LinkageMapELF to compute the multilib category for each preserved
library, in the same way as the _post_src_install_soname_symlinks
function, so that the LinkageMapELF.findConsumers method will operate
correctly with preserved libraries of all ABIs.
Fixes: f1c1b8a77eeb ("Generate soname dependency metadata (bug 282639)")
X-Gentoo-bug: 598080
X-Gentoo-bug-url: https://bugs.gentoo.org/598080
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>
pym/portage/util/_dyn_libs/LinkageMapELF.py | 33 +++++++++++++++++++++++------
1 file changed, 26 insertions(+), 7 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py b/pym/portage/util/_dyn_libs/LinkageMapELF.py
index 0b09fe5..a063621 100644
--- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
+++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
@@ -4,6 +4,7 @@
import errno
import logging
import subprocess
+import sys
import portage
from portage import _encodings
@@ -12,6 +13,7 @@ from portage import _unicode_decode
from portage import _unicode_encode
from portage.cache.mappings import slot_dict_class
from portage.const import EPREFIX
+from portage.dep.soname.multilib_category import compute_multilib_category
from portage.exception import CommandNotFound, InvalidData
from portage.localization import _
from portage.util import getlibpaths
@@ -20,6 +22,12 @@ from portage.util import normalize_path
from portage.util import varexpand
from portage.util import writemsg_level
from portage.util._dyn_libs.NeededEntry import NeededEntry
+from portage.util.elf.header import ELFHeader
+
+if sys.hexversion >= 0x3000000:
+ _unicode = str
+else:
+ _unicode = unicode
# Map ELF e_machine values from NEEDED.ELF.2 to approximate multilib
# categories. This approximation will produce incorrect results on x32
@@ -283,15 +291,26 @@ class LinkageMapELF(object):
l = l[3:].rstrip("\n")
if not l:
continue
- fields = l.split(";")
- if len(fields) < 5:
- writemsg_level(_("\nWrong number of fields " \
- "returned from scanelf: %s\n\n") % (l,),
+ try:
+ entry = NeededEntry.parse("scanelf", l)
+ except InvalidData as e:
+ writemsg_level("\n%s\n\n" % (e,),
level=logging.ERROR, noiselevel=-1)
continue
- fields[1] = fields[1][root_len:]
- owner = plibs.pop(fields[1], None)
- lines.append((owner, "scanelf", ";".join(fields)))
+ try:
+ with open(_unicode_encode(entry.filename,
+ encoding=_encodings['fs'],
+ errors='strict'), 'rb') as f:
+ elf_header = ELFHeader.read(f)
+ except EnvironmentError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ # File removed concurrently.
+ continue
+ entry.multilib_category = compute_multilib_category(elf_header)
+ entry.filename = entry.filename[root_len:]
+ owner = plibs.pop(entry.filename, None)
+ lines.append((owner, "scanelf", _unicode(entry)))
proc.wait()
proc.stdout.close()
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2018-01-02 23:04 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2018-01-02 23:04 UTC (permalink / raw
To: gentoo-commits
commit: b9fc8e55f96c17aeece87387226ada1b184d2f77
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 2 09:09:34 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Jan 2 23:02:39 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9fc8e55
PreservedLibsRegistry: fix pruneNonExisting for symlinks to other dirs (bug 642672)
Fix pruneNonExisting to use the abssymlink function to detect symlinks
in the registry that no longer point to a preserved library. The previous
code only worked correctly for symlinks pointing to files in the same
directory, which failed for packages like dev-ada/xmlada which have
symlinks that point into a subdirectory.
Fixes: 32d19be14e22 ("pruneNonExisting: handle eselect-opengl symlinks")
Tested-by: Tupone Alfredo <tupone <AT> gentoo.org>
Bug: https://bugs.gentoo.org/642672
pym/portage/util/_dyn_libs/PreservedLibsRegistry.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
index a422ffefd..f83b82a31 100644
--- a/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
+++ b/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py
@@ -12,6 +12,7 @@ try:
except ImportError:
import pickle
+from portage import abssymlink
from portage import os
from portage import _encodings
from portage import _os_merge
@@ -227,7 +228,7 @@ class PreservedLibsRegistry(object):
# removed by _remove_preserved_libs, it calls pruneNonExisting
# which eliminates the irrelevant symlink from the registry here.
for f, target in symlinks.items():
- if os.path.join(os.path.dirname(f), target) in hardlinks:
+ if abssymlink(f, target=target) in hardlinks:
paths.append(f)
if len(paths) > 0:
^ permalink raw reply related [flat|nested] 32+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/
@ 2018-05-25 16:12 Zac Medico
0 siblings, 0 replies; 32+ messages in thread
From: Zac Medico @ 2018-05-25 16:12 UTC (permalink / raw
To: gentoo-commits
commit: 466cc6c60fcf1d56049af7a1d37fc959f6808351
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri May 25 16:07:33 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri May 25 16:10:34 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=466cc6c6
SonameDepsProcessor: fix 'set' object has no attribute 'items' (bug 656492)
Use collections.defaultdict to ensure that requires_map
contains correct defaults.
Fixes: 1364cd44e7a6 ("SonameDepsProcessor: handle internal libs without DT_SONAME (bug 646190)")
Bug: https://bugs.gentoo.org/656492
pym/portage/util/_dyn_libs/soname_deps.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/pym/portage/util/_dyn_libs/soname_deps.py b/pym/portage/util/_dyn_libs/soname_deps.py
index c6302afc2..544cbc8f1 100644
--- a/pym/portage/util/_dyn_libs/soname_deps.py
+++ b/pym/portage/util/_dyn_libs/soname_deps.py
@@ -3,7 +3,9 @@
from __future__ import unicode_literals
+import collections
import fnmatch
+import functools
from itertools import chain
import os
import re
@@ -33,7 +35,8 @@ class SonameDepsProcessor(object):
"""
self._provides_exclude = self._exclude_pattern(provides_exclude)
self._requires_exclude = self._exclude_pattern(requires_exclude)
- self._requires_map = {}
+ self._requires_map = collections.defaultdict(
+ functools.partial(collections.defaultdict, set))
self._provides_map = {}
self._provides_unfiltered = {}
self._basename_map = {}
@@ -84,8 +87,7 @@ class SonameDepsProcessor(object):
for x in entry.needed:
if (self._requires_exclude is None or
self._requires_exclude.match(x) is None):
- self._requires_map.setdefault(
- multilib_cat, {}).setdefault(x, set()).add(runpaths)
+ self._requires_map[multilib_cat][x].add(runpaths)
if entry.soname:
self._provides_unfiltered.setdefault(
@@ -105,7 +107,6 @@ class SonameDepsProcessor(object):
provides_unfiltered = self._provides_unfiltered
for multilib_cat in set(chain(requires_map, provides_map)):
- requires_map.setdefault(multilib_cat, set())
provides_map.setdefault(multilib_cat, set())
provides_unfiltered.setdefault(multilib_cat, set())
for soname, consumers in list(requires_map[multilib_cat].items()):
^ permalink raw reply related [flat|nested] 32+ messages in thread
end of thread, other threads:[~2018-05-25 16:12 UTC | newest]
Thread overview: 32+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2011-07-20 8:05 [gentoo-commits] proj/portage:master commit in: pym/portage/util/_dyn_libs/ Zac Medico
-- strict thread matches above, loose matches on Subject: below --
2018-05-25 16:12 Zac Medico
2018-01-02 23:04 Zac Medico
2016-12-27 21:43 Zac Medico
2016-06-06 16:03 Zac Medico
2015-06-02 3:48 Zac Medico
2015-04-24 17:53 Zac Medico
2013-03-18 23:43 Zac Medico
2013-03-18 23:35 Zac Medico
2013-02-22 18:52 Arfrever Frehtes Taifersar Arahesis
2012-03-22 15:24 Zac Medico
2012-03-21 21:24 Zac Medico
2012-03-05 7:32 Zac Medico
2012-02-18 3:11 Zac Medico
2012-02-18 2:06 Zac Medico
2012-02-01 2:04 Zac Medico
2011-07-24 2:58 Zac Medico
2011-07-24 2:58 Zac Medico
2011-07-24 2:15 Zac Medico
2011-07-23 6:23 Zac Medico
2011-07-22 20:38 Zac Medico
2011-07-22 8:03 Zac Medico
2011-07-20 4:51 Zac Medico
2011-07-01 4:02 Zac Medico
2011-07-01 4:02 Zac Medico
2011-07-01 1:37 Zac Medico
2011-06-30 10:17 Zac Medico
2011-06-30 10:01 Zac Medico
2011-05-08 21:31 Zac Medico
2011-05-08 7:10 Zac Medico
2011-05-08 5:02 Zac Medico
2011-05-07 17:25 Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox