[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/dbapi/

[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/dbapi/

[Zac Medico]

commit:     6e6d8e7f522fef3c32a7c71298024167c066a3c5
Author:     Frédéric Pierret (fepitre) <frederic.pierret <AT> qubes-os <DOT> org>
AuthorDate: Thu Aug 20 09:35:23 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Sep  7 22:23:08 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=6e6d8e7f

Use portage proxy settings for fetching BINPKG host

Bug: https://bugs.gentoo.org/740898
See: https://github.com/gentoo/portage/pull/607
Signed-off-by: Frédéric Pierret (fepitre) <frederic.pierret <AT> qubes-os.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/dbapi/bintree.py |  9 ++++++++-
 lib/portage/util/_urlopen.py | 10 +++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 620865a79..ee30542a5 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -964,11 +964,18 @@ class binarytree:
 						download_timestamp + ttl > time.time():
 						raise UseCachedCopyOfRemoteIndex()
 
+				# Set proxy settings for _urlopen -> urllib_request
+				proxies = {}
+				for proto in ('http', 'https'):
+					value = self.settings.get(proto + '_proxy')
+					if value is not None:
+						proxies[proto] = value
+
 				# Don't use urlopen for https, unless
 				# PEP 476 is supported (bug #469888).
 				if parsed_url.scheme not in ('https',) or _have_pep_476():
 					try:
-						f = _urlopen(url, if_modified_since=local_timestamp)
+						f = _urlopen(url, if_modified_since=local_timestamp, proxies=proxies)
 						if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
 							remote_timestamp = f.headers.get('timestamp')
 					except IOError as err:

diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py
index b46d1554c..b67d02739 100644
--- a/lib/portage/util/_urlopen.py
+++ b/lib/portage/util/_urlopen.py
@@ -26,7 +26,7 @@ def have_pep_476():
 	return hasattr(__import__('ssl'), '_create_unverified_context')
 
 
-def urlopen(url, if_modified_since=None):
+def urlopen(url, if_modified_since=None, proxies=None):
 	parse_result = urllib_parse.urlparse(url)
 	if parse_result.scheme not in ("http", "https"):
 		return _urlopen(url)
@@ -40,8 +40,12 @@ def urlopen(url, if_modified_since=None):
 		request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since))
 	if parse_result.username is not None:
 		password_manager.add_password(None, url, parse_result.username, parse_result.password)
-	auth_handler = CompressedResponseProcessor(password_manager)
-	opener = urllib_request.build_opener(auth_handler)
+
+	handlers = [CompressedResponseProcessor(password_manager)]
+	if proxies:
+		handlers.append(urllib_request.ProxyHandler(proxies))
+	opener = urllib_request.build_opener(*handlers)
+
 	hdl = opener.open(request)
 	if hdl.headers.get('last-modified', ''):
 		try:

[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/dbapi/

[Sam James]

commit:     5572b73744121f67c4e55040966bfe91a0e5fb6c
Author:     gcarq <egger.m <AT> protonmail <DOT> com>
AuthorDate: Tue Mar 21 16:19:13 2023 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue May 23 00:22:09 2023 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5572b737

mergeme: Consider file mode and xattr for comparison

Implements dblink._needs_move(...) to take care of file equality
checks which consist of file mode, extended attributes and
file content.

Signed-off-by: gcarq <egger.m <AT> protonmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/dbapi/vartree.py | 32 +++++++++++++++++++++++++-------
 lib/portage/util/movefile.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 327b72bed..317cf327a 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3,8 +3,6 @@
 
 __all__ = ["vardbapi", "vartree", "dblink"] + ["write_contents", "tar_contents"]
 
-import filecmp
-
 import portage
 
 portage.proxy.lazyimport.lazyimport(
@@ -34,7 +32,7 @@ portage.proxy.lazyimport.lazyimport(
     "portage.util.env_update:env_update",
     "portage.util.install_mask:install_mask_dir,InstallMask,_raise_exc",
     "portage.util.listdir:dircache,listdir",
-    "portage.util.movefile:movefile",
+    "portage.util.movefile:movefile,_cmpxattr",
     "portage.util.path:first_existing,iter_parents",
     "portage.util.writeable_check:get_ro_checker",
     "portage.util._xattr:xattr",
@@ -95,6 +93,7 @@ from ._ContentsCaseSensitivityManager import ContentsCaseSensitivityManager
 
 import argparse
 import errno
+import filecmp
 import fnmatch
 import functools
 import gc
@@ -5803,10 +5802,7 @@ class dblink:
                 # same way.  Unless moveme=0 (blocking directory)
                 if moveme:
                     # only replace the existing file if it differs, see #722270
-                    already_merged = os.path.exists(mydest)
-                    if already_merged and filecmp.cmp(mysrc, mydest, shallow=False):
-                        zing = "==="
-                    else:
+                    if self._needs_move(mysrc, mydest, mymode, mydmode):
                         # Create hardlinks only for source files that already exist
                         # as hardlinks (having identical st_dev and st_ino).
                         hardlink_key = (mystat.st_dev, mystat.st_ino)
@@ -5829,6 +5825,8 @@ class dblink:
                             return 1
                         hardlink_candidates.append(mydest)
                         zing = ">>>"
+                    else:
+                        zing = "==="
 
                     try:
                         self._merged_path(mydest, os.lstat(mydest))
@@ -6254,6 +6252,26 @@ class dblink:
         finally:
             self.unlockdb()
 
+    def _needs_move(self, mysrc, mydest, mymode, mydmode):
+        """
+        Checks whether the given file at |mysrc| needs to be moved to |mydest| or if
+        they are identical.
+
+        Takes file mode and extended attributes into account.
+        Should only be used for regular files.
+        """
+        if not os.path.exists(mydest):
+            return True
+
+        if mymode != mydmode:
+            return True
+
+        excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
+        if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
+            return True
+
+        return not filecmp.cmp(mysrc, mydest, shallow=False)
+
 
 def merge(
     mycat,

diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index d46c56ade..e2f19ba92 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -105,6 +105,35 @@ def _copyxattr(src, dest, exclude=None):
             )
 
 
+def _cmpxattr(src, dest, exclude=None):
+    """
+    Compares extended attributes between |src| and |dest| and returns True
+    if they are equal or xattrs are not supported, False otherwise
+    """
+    try:
+        src_attrs = xattr.list(src)
+        dest_attrs = xattr.list(dest)
+    except OSError as e:
+        if e.errno != OperationNotSupported.errno:
+            raise
+        return True
+
+    if src_attrs:
+        if exclude is not None and isinstance(src_attrs[0], bytes):
+            exclude = exclude.encode(_encodings["fs"])
+    exclude = _get_xattr_excluder(exclude)
+
+    src_attrs = {attr for attr in src_attrs if not exclude(attr)}
+    dest_attrs = {attr for attr in dest_attrs if not exclude(attr)}
+    if src_attrs != dest_attrs:
+        return False
+
+    for attr in src_attrs:
+        if xattr.get(src, attr) != xattr.get(dest, attr):
+            return False
+    return True
+
+
 def movefile(
     src,
     dest,