* [gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
@ 2024-02-22 15:36 Zac Medico
0 siblings, 0 replies; 7+ messages in thread
From: Zac Medico @ 2024-02-22 15:36 UTC (permalink / raw
To: gentoo-commits
commit: fbaaa4a733aaadc2744b656527756ac4e2b7ab58
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 22 06:47:33 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 22 07:28:38 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fbaaa4a7
socks5: Use real asyncio.run
Use real asyncio.run to demonstrate that it is compatible with
portage internals. Since the socks5 ProxyManager uses the
process.spawn function, the internal _running_loop function
needs to return the correct loop for use in the wait method of
MultiprocessingProcess, or else it will lead to Future
"attached to a different loop" errors.
Bug: https://bugs.gentoo.org/761538
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/tests/util/test_socks5.py | 45 +++++++++++++++++++----------------
lib/portage/util/socks5.py | 30 ++++++++++++++++-------
2 files changed, 46 insertions(+), 29 deletions(-)
diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py
index 987b41af29..7b33cb3f6b 100644
--- a/lib/portage/tests/util/test_socks5.py
+++ b/lib/portage/tests/util/test_socks5.py
@@ -1,6 +1,7 @@
-# Copyright 2019-2021 Gentoo Authors
+# Copyright 2019-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import asyncio
import functools
import shutil
import socket
@@ -10,7 +11,6 @@ import time
import portage
from portage.tests import TestCase
-from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util import socks5
from portage.const import PORTAGE_BIN_PATH
@@ -88,18 +88,20 @@ class AsyncHTTPServerTestCase(TestCase):
if f is not None:
f.close()
- def test_http_server(self):
+ async def _test_http_server(self):
+ asyncio.run(self._test_http_server())
+
+ async def _test_http_server(self):
host = "127.0.0.1"
content = b"Hello World!\n"
path = "/index.html"
- loop = global_event_loop()
+
+ loop = asyncio.get_running_loop()
for i in range(2):
with AsyncHTTPServer(host, {path: content}, loop) as server:
for j in range(2):
- result = loop.run_until_complete(
- loop.run_in_executor(
- None, self._fetch_directly, host, server.server_port, path
- )
+ result = await loop.run_in_executor(
+ None, self._fetch_directly, host, server.server_port, path
)
self.assertEqual(result, content)
@@ -177,7 +179,10 @@ class Socks5ServerTestCase(TestCase):
return f.read()
def test_socks5_proxy(self):
- loop = global_event_loop()
+ asyncio.run(self._test_socks5_proxy())
+
+ async def _test_socks5_proxy(self):
+ loop = asyncio.get_running_loop()
host = "127.0.0.1"
content = b"Hello World!"
@@ -193,20 +198,18 @@ class Socks5ServerTestCase(TestCase):
}
proxy = socks5.get_socks5_proxy(settings)
- loop.run_until_complete(socks5.proxy.ready())
-
- result = loop.run_until_complete(
- loop.run_in_executor(
- None,
- self._fetch_via_proxy,
- proxy,
- host,
- server.server_port,
- path,
- )
+ await socks5.proxy.ready()
+
+ result = await loop.run_in_executor(
+ None,
+ self._fetch_via_proxy,
+ proxy,
+ host,
+ server.server_port,
+ path,
)
self.assertEqual(result, content)
finally:
- socks5.proxy.stop()
+ await socks5.proxy.stop()
shutil.rmtree(tempdir)
diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
index 6c68ff4106..74592aeefe 100644
--- a/lib/portage/util/socks5.py
+++ b/lib/portage/util/socks5.py
@@ -2,15 +2,16 @@
# Copyright 2015-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import asyncio
import errno
import os
import socket
+from typing import Union
import portage.data
from portage import _python_interpreter
from portage.data import portage_gid, portage_uid, userpriv_groups
from portage.process import atexit_register, spawn
-from portage.util.futures import asyncio
class ProxyManager:
@@ -57,23 +58,36 @@ class ProxyManager:
**spawn_kwargs,
)
- def stop(self):
+ def stop(self) -> Union[None, asyncio.Future]:
"""
Stop the SOCKSv5 server.
+
+ If there is a running asyncio event loop then asyncio.Future is
+ returned which should be used to wait for the server process
+ to exit.
"""
+ future = None
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = None
if self._proc is not None:
self._proc.terminate()
- loop = asyncio.get_event_loop()
- if self._proc_waiter is None:
- self._proc_waiter = asyncio.ensure_future(self._proc.wait(), loop)
- if loop.is_running():
- self._proc_waiter.add_done_callback(lambda future: future.result())
+ if loop is None:
+ asyncio.run(self._proc.wait())
else:
- loop.run_until_complete(self._proc_waiter)
+ if self._proc_waiter is None:
+ self._proc_waiter = asyncio.ensure_future(self._proc.wait(), loop)
+ future = asyncio.shield(self._proc_waiter)
+
+ if loop is not None and future is None:
+ future = loop.create_future()
+ future.set_result(None)
self.socket_path = None
self._proc = None
self._proc_waiter = None
+ return future
def is_running(self):
"""
^ permalink raw reply related [flat|nested] 7+ messages in thread* [gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
@ 2022-12-31 13:33 Sam James
0 siblings, 0 replies; 7+ messages in thread
From: Sam James @ 2022-12-31 13:33 UTC (permalink / raw
To: gentoo-commits
commit: 98f984875251f86d3a5c367765c7d6e249277a26
Author: Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 30 10:44:10 2022 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Dec 31 13:33:04 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=98f98487
Test C whirlpool implementation separately from Python impl
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/tests/util/test_whirlpool.py | 17 +++++++++++------
lib/portage/util/whirlpool.py | 2 ++
2 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/lib/portage/tests/util/test_whirlpool.py b/lib/portage/tests/util/test_whirlpool.py
index d93467b21..ac156abf3 100644
--- a/lib/portage/tests/util/test_whirlpool.py
+++ b/lib/portage/tests/util/test_whirlpool.py
@@ -2,26 +2,31 @@
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
-from portage.util.whirlpool import Whirlpool
+from portage.util.whirlpool import CWhirlpool, PyWhirlpool
class WhirlpoolTestCase(TestCase):
- def testBundledWhirlpool(self):
+ def testBundledWhirlpool(self, cls=PyWhirlpool):
self.assertEqual(
- Whirlpool(b"The quick brown fox jumps over the lazy dog").hexdigest(),
+ cls(b"The quick brown fox jumps over the lazy dog").hexdigest(),
"b97de512e91e3828b40d2b0fdce9ceb3c4a71f9bea8d88e75c4fa854df36725fd2b52eb6544edcacd6f8beddfea403cb55ae31f03ad62a5ef54e42ee82c3fb35",
)
self.assertEqual(
- Whirlpool(b"The quick brown fox jumps over the lazy eog").hexdigest(),
+ cls(b"The quick brown fox jumps over the lazy eog").hexdigest(),
"c27ba124205f72e6847f3e19834f925cc666d0974167af915bb462420ed40cc50900d85a1f923219d832357750492d5c143011a76988344c2635e69d06f2d38c",
)
self.assertEqual(
- Whirlpool(b"").hexdigest(),
+ cls(b"").hexdigest(),
"19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
)
- w = Whirlpool()
+ w = cls()
w.update(b"")
self.assertEqual(
w.hexdigest(),
"19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
)
+
+ def testCWhirlpool(self):
+ if not CWhirlpool.is_available:
+ self.skipTest("C Whirlpool extension is not importable")
+ self.testBundledWhirlpool(CWhirlpool)
diff --git a/lib/portage/util/whirlpool.py b/lib/portage/util/whirlpool.py
index 1431c18fb..7caa20f6e 100644
--- a/lib/portage/util/whirlpool.py
+++ b/lib/portage/util/whirlpool.py
@@ -83,6 +83,8 @@ class CWhirlpool:
may be provided; if present, this string will be automatically
hashed."""
+ is_available = WhirlpoolExt is not None
+
def __init__(self, arg=b""):
self.obj = WhirlpoolExt()
self.dig = None
^ permalink raw reply related [flat|nested] 7+ messages in thread* [gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
@ 2022-06-07 23:48 Mike Gilbert
0 siblings, 0 replies; 7+ messages in thread
From: Mike Gilbert @ 2022-06-07 23:48 UTC (permalink / raw
To: gentoo-commits
commit: 4540f2b0a6529705343f493abb478370c15f101f
Author: David Palao <david.palao <AT> gmail <DOT> com>
AuthorDate: Mon May 30 07:00:32 2022 +0000
Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Tue Jun 7 23:47:55 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=4540f2b0
improvement(mtimedb): class has better API
New attribute ``is_readonly`` and new method ``make_readonly`` added.
Signed-off-by: David Palao <david.palao <AT> gmail.com>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 28 +++++++++++++++++++++++++---
lib/portage/util/mtimedb.py | 10 ++++++++++
2 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py b/lib/portage/tests/util/test_mtimedb.py
index a65a6be91..ecc0e7135 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -9,6 +9,8 @@ from portage.util.mtimedb import MtimeDB, _MTIMEDBKEYS
from portage.exception import DigestException
+# Some random data for the fixtures:
+
_ONE_RESUME_LIST_JSON = b"""{
"info": {
"/tmp/stage1root/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info": 1711785090,
@@ -213,7 +215,7 @@ _TWO_RESUME_LISTS_JSON = b"""{
class MtimeDBTestCase(TestCase):
text = b"Unit tests for MtimeDB"
- def test_instance_created_with_only_expected_keys(self):
+ def test_instances_are_created_with_only_expected_keys(self):
all_fixtures = (
_ONE_RESUME_LIST_JSON,
_EMPTY_FILE,
@@ -227,7 +229,7 @@ class MtimeDBTestCase(TestCase):
mtimedb = MtimeDB("/path/to/mtimedb")
self.assertLessEqual(set(mtimedb.keys()), _MTIMEDBKEYS)
- def test_instance_has_default_values(self):
+ def test_instances_have_default_values(self):
with patch("portage.util.mtimedb.open",
mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
@@ -237,7 +239,7 @@ class MtimeDBTestCase(TestCase):
self.assertEqual(mtimedb["ldpath"], {})
self.assertEqual(mtimedb["updates"], {})
- def test_instance_has_a_deepcopy_of_clean_data(self):
+ def test_instances_keep_a_deepcopy_of_clean_data(self):
with patch("portage.util.mtimedb.open",
mock_open(read_data=_ONE_RESUME_LIST_JSON)):
mtimedb = MtimeDB("/some/path/mtimedb")
@@ -274,3 +276,23 @@ class MtimeDBTestCase(TestCase):
mtimedb = MtimeDB("/some/path/mtimedb")
mtimedb.commit()
pwrite2disk.assert_not_called()
+
+ def test_is_readonly_attribute(self):
+ with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ mtimedb = MtimeDB("/some/path/mtimedb")
+ self.assertFalse(mtimedb.is_readonly)
+
+ mtimedb.filename = None
+ self.assertTrue(mtimedb.is_readonly)
+
+ mtimedb.filename = "/what/ever/mtimedb"
+ self.assertFalse(mtimedb.is_readonly)
+
+ def test_make_readonly(self):
+ with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ mtimedb = MtimeDB("/some/path/mtimedb")
+ mtimedb.make_readonly()
+ self.assertTrue(mtimedb.is_readonly)
+ self.assertIs(mtimedb.filename, None)
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index a6566e3f8..95da2235f 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -51,6 +51,16 @@ class MtimeDB(dict):
self.filename = filename
self._load(filename)
+ @property
+ def is_readonly(self):
+ if self.filename is None:
+ return True
+ else:
+ return False
+
+ def make_readonly(self):
+ self.filename = None
+
def _load(self, filename):
f = None
content = None
^ permalink raw reply related [flat|nested] 7+ messages in thread* [gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
@ 2022-06-07 23:48 Mike Gilbert
0 siblings, 0 replies; 7+ messages in thread
From: Mike Gilbert @ 2022-06-07 23:48 UTC (permalink / raw
To: gentoo-commits
commit: e22cf80ec99c52c1192ecf99f6b66d1d892f3f86
Author: David Palao <david.palao <AT> gmail <DOT> com>
AuthorDate: Fri Jun 3 09:13:17 2022 +0000
Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Tue Jun 7 23:47:56 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=e22cf80e
improvement(pylint/black): some formatting issues fixed
Signed-off-by: David Palao <david.palao <AT> gmail.com>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 126 ++++++++++++++-------------------
lib/portage/util/mtimedb.py | 14 ++--
2 files changed, 59 insertions(+), 81 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py b/lib/portage/tests/util/test_mtimedb.py
index 09d72049f..d80b4f1da 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -9,7 +9,6 @@ from portage.tests import TestCase
import portage
from portage.data import portage_gid, uid
from portage.util.mtimedb import MtimeDB, _MTIMEDBKEYS
-from portage.exception import DigestException
# Some data for the fixtures:
@@ -210,15 +209,12 @@ class MtimeDBTestCase(TestCase):
_TWO_RESUME_LISTS_JSON,
)
for contents in all_fixtures:
- with patch(
- "portage.util.mtimedb.open", mock_open(read_data=contents)
- ):
+ with patch("portage.util.mtimedb.open", mock_open(read_data=contents)):
mtimedb = MtimeDB("/path/to/mtimedb")
self.assertLessEqual(set(mtimedb.keys()), _MTIMEDBKEYS)
def test_default_values(self):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_EMPTY_FILE)):
+ with patch("portage.util.mtimedb.open", mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(mtimedb["starttime"], 0)
self.assertEqual(mtimedb["version"], "")
@@ -227,88 +223,75 @@ class MtimeDBTestCase(TestCase):
self.assertEqual(mtimedb["updates"], {})
def test_instances_keep_a_deepcopy_of_clean_data(self):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ with patch(
+ "portage.util.mtimedb.open", mock_open(read_data=_ONE_RESUME_LIST_JSON)
+ ):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(dict(mtimedb), dict(mtimedb._clean_data))
self.assertIsNot(mtimedb, mtimedb._clean_data)
def test_load_data_called_at_instance_creation_time(self):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ with patch(
+ "portage.util.mtimedb.open", mock_open(read_data=_ONE_RESUME_LIST_JSON)
+ ):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(
mtimedb["info"],
{
- "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info": (
- 1711787325),
- "/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info": (
- 1735158257),
- "/usr/share/info": 1650633847
- }
+ "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info": (1711787325),
+ "/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info": (1735158257),
+ "/usr/share/info": 1650633847,
+ },
)
self.assertEqual(
mtimedb["ldpath"],
{
- "/lib": 1748456830,
- "/lib64": 1750523381,
- "/usr/lib": 1750461195,
- "/usr/lib/llvm/13/lib64": 1747003135,
- "/usr/lib/rust/lib": 1750461173,
- "/usr/lib64": 1750881821,
- "/usr/local/lib": 1711784303,
- "/usr/local/lib64": 1711784303
- }
+ "/lib": 1748456830,
+ "/lib64": 1750523381,
+ "/usr/lib": 1750461195,
+ "/usr/lib/llvm/13/lib64": 1747003135,
+ "/usr/lib/rust/lib": 1750461173,
+ "/usr/lib64": 1750881821,
+ "/usr/local/lib": 1711784303,
+ "/usr/local/lib64": 1711784303,
+ },
)
self.assertEqual(
mtimedb["resume"],
{
- "favorites": [
- "@world"
- ],
- "mergelist": [
- [
- "ebuild",
- "/",
- "some-cat/some-package-1.2.3-r4",
- "merge"
- ],
- [
- "ebuild",
- "/",
- "another-cat/another-package-4.3.2-r1",
- "merge"
- ]
- ],
- "myopts": {
- "--buildpkg": True,
- "--deep": True,
- "--getbinpkg": True,
- "--keep-going": True,
- "--newuse": True,
- "--quiet": True,
- "--regex-search-auto": "y",
- "--update": True,
- "--usepkg": True,
- "--verbose": True
- }
- }
+ "favorites": ["@world"],
+ "mergelist": [
+ ["ebuild", "/", "some-cat/some-package-1.2.3-r4", "merge"],
+ ["ebuild", "/", "another-cat/another-package-4.3.2-r1", "merge"],
+ ],
+ "myopts": {
+ "--buildpkg": True,
+ "--deep": True,
+ "--getbinpkg": True,
+ "--keep-going": True,
+ "--newuse": True,
+ "--quiet": True,
+ "--regex-search-auto": "y",
+ "--update": True,
+ "--usepkg": True,
+ "--verbose": True,
+ },
+ },
)
self.assertEqual(mtimedb["starttime"], 0)
self.assertEqual(
mtimedb["updates"],
{
- "/var/db/repos/gentoo/profiles/updates/1Q-2022": 1747854791,
- "/var/db/repos/gentoo/profiles/updates/2Q-2022": 1752846209,
- "/var/db/repos/gentoo/profiles/updates/4Q-2021": 1742787797
- }
+ "/var/db/repos/gentoo/profiles/updates/1Q-2022": 1747854791,
+ "/var/db/repos/gentoo/profiles/updates/2Q-2022": 1752846209,
+ "/var/db/repos/gentoo/profiles/updates/4Q-2021": 1742787797,
+ },
)
self.assertEqual(mtimedb["version"], "3.0.30")
@patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
def test_commit_writes_to_disk_if_needed_and_possible(self, pwrite2disk):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_EMPTY_FILE)):
+ with patch("portage.util.mtimedb.open", mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
mtimedb.commit()
pwrite2disk.assert_not_called()
@@ -320,8 +303,7 @@ class MtimeDBTestCase(TestCase):
@patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
def test_commit_does_not_write_to_disk_if_no_file(self, pwrite2disk):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_EMPTY_FILE)):
+ with patch("portage.util.mtimedb.open", mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
mtimedb["updates"]["/long/path/1Q-2021"] = 1739992409
mtimedb.filename = None
@@ -330,15 +312,15 @@ class MtimeDBTestCase(TestCase):
@patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
def test_commit_does_not_write_to_disk_if_no_changes(self, pwrite2disk):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_EMPTY_FILE)):
+ with patch("portage.util.mtimedb.open", mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
mtimedb.commit()
pwrite2disk.assert_not_called()
def test_is_readonly_attribute(self):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ with patch(
+ "portage.util.mtimedb.open", mock_open(read_data=_ONE_RESUME_LIST_JSON)
+ ):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertFalse(mtimedb.is_readonly)
@@ -349,8 +331,9 @@ class MtimeDBTestCase(TestCase):
self.assertFalse(mtimedb.is_readonly)
def test_make_readonly(self):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ with patch(
+ "portage.util.mtimedb.open", mock_open(read_data=_ONE_RESUME_LIST_JSON)
+ ):
mtimedb = MtimeDB("/some/path/mtimedb")
mtimedb.make_readonly()
self.assertTrue(mtimedb.is_readonly)
@@ -359,8 +342,9 @@ class MtimeDBTestCase(TestCase):
@patch("portage.util.mtimedb.apply_secpass_permissions")
@patch("portage.util.mtimedb.atomic_ofstream")
def test_write_to_disk(self, matomic_ofstream, mapply_perms):
- with patch("portage.util.mtimedb.open",
- mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ with patch(
+ "portage.util.mtimedb.open", mock_open(read_data=_ONE_RESUME_LIST_JSON)
+ ):
mtimedb = MtimeDB("/some/path/mtimedb")
d = {"z": "zome", "a": "AAA"}
encoding = portage._encodings["repo.content"]
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index 497c01e05..098350526 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -71,9 +71,7 @@ class MtimeDB(dict):
if getattr(e, "errno", None) in (errno.ENOENT, errno.EACCES):
pass
else:
- writemsg(
- _(f"!!! Error loading '{filename}': {e}\n"), noiselevel=-1
- )
+ writemsg(_(f"!!! Error loading '{filename}': {e}\n"), noiselevel=-1)
finally:
if f is not None:
f.close()
@@ -83,8 +81,7 @@ class MtimeDB(dict):
try:
d = json.loads(
_unicode_decode(
- content, encoding=_encodings["repo.content"],
- errors="strict"
+ content, encoding=_encodings["repo.content"], errors="strict"
)
)
except SystemExit:
@@ -101,10 +98,7 @@ class MtimeDB(dict):
except SystemExit:
raise
except Exception:
- writemsg(
- _(f"!!! Error loading '{filename}': {e}\n"),
- noiselevel=-1
- )
+ writemsg(_(f"!!! Error loading '{filename}': {e}\n"), noiselevel=-1)
if "old" in d:
d["updates"] = d["old"]
@@ -117,7 +111,7 @@ class MtimeDB(dict):
for k in ("info", "ldpath", "updates"):
d.setdefault(k, {})
- for k in (set(d.keys())-_MTIMEDBKEYS):
+ for k in set(d.keys()) - _MTIMEDBKEYS:
writemsg(_(f"Deleting invalid mtimedb key: {k}\n"))
del d[k]
self.update(d)
^ permalink raw reply related [flat|nested] 7+ messages in thread* [gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
@ 2022-06-07 23:48 Mike Gilbert
0 siblings, 0 replies; 7+ messages in thread
From: Mike Gilbert @ 2022-06-07 23:48 UTC (permalink / raw
To: gentoo-commits
commit: 8a0a42ceb530b277e17220614f8c34bc8d0e503c
Author: David Palao <david.palao <AT> gmail <DOT> com>
AuthorDate: Tue May 31 15:30:48 2022 +0000
Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Tue Jun 7 23:47:55 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a0a42ce
test(MtimeDB): added some unit tests for __write_to_disk and for _load
Signed-off-by: David Palao <david.palao <AT> gmail.com>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 122 +++++++++++++++++++++++++++------
lib/portage/util/mtimedb.py | 1 +
2 files changed, 102 insertions(+), 21 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py b/lib/portage/tests/util/test_mtimedb.py
index ecc0e7135..09d72049f 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -2,31 +2,28 @@
# Distributed under the terms of the GNU General Public License v2
from unittest.mock import patch, mock_open
+import json
from portage.tests import TestCase
+import portage
+from portage.data import portage_gid, uid
from portage.util.mtimedb import MtimeDB, _MTIMEDBKEYS
from portage.exception import DigestException
-# Some random data for the fixtures:
+# Some data for the fixtures:
_ONE_RESUME_LIST_JSON = b"""{
"info": {
- "/tmp/stage1root/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info": 1711785090,
- "/tmp/stage1root/usr/share/gcc-data/x86_64-pc-linux-gnu/9.3.0/info": 1711785090,
- "/tmp/stage1root/usr/share/info": 1611785090,
- "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info": 1711787325,
+ "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info": 1711787325,
"/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info": 1735158257,
- "/usr/share/gcc-data/x86_64-pc-linux-gnu/9.3.0/info": 1711787325,
"/usr/share/info": 1650633847
},
"ldpath": {
"/lib": 1748456830,
"/lib64": 1750523381,
"/usr/lib": 1750461195,
- "/usr/lib/llvm/11/lib64": 1723048948,
- "/usr/lib/llvm/12/lib64": 1730499781,
"/usr/lib/llvm/13/lib64": 1747003135,
"/usr/lib/rust/lib": 1750461173,
"/usr/lib64": 1750881821,
@@ -66,12 +63,8 @@ _ONE_RESUME_LIST_JSON = b"""{
},
"starttime": 0,
"updates": {
- "/var/db/repos/gentoo/profiles/updates/1Q-2021": 1739992409,
"/var/db/repos/gentoo/profiles/updates/1Q-2022": 1747854791,
- "/var/db/repos/gentoo/profiles/updates/2Q-2021": 1724404379,
"/var/db/repos/gentoo/profiles/updates/2Q-2022": 1752846209,
- "/var/db/repos/gentoo/profiles/updates/3Q-2021": 1741119203,
- "/var/db/repos/gentoo/profiles/updates/4Q-2020": 1709167362,
"/var/db/repos/gentoo/profiles/updates/4Q-2021": 1742787797
},
"version": "3.0.30"
@@ -85,8 +78,6 @@ _PARTIAL_FILE_JSON = b"""{
"/lib": 1748456830,
"/lib64": 1750523381,
"/usr/lib": 1750461195,
- "/usr/lib/llvm/11/lib64": 1723048948,
- "/usr/lib/llvm/12/lib64": 1730499781,
"/usr/lib/llvm/13/lib64": 1747003135,
"/usr/lib/rust/lib": 1750461173,
"/usr/lib64": 1750881821,
@@ -130,7 +121,7 @@ _PARTIAL_FILE_JSON = b"""{
_TWO_RESUME_LISTS_JSON = b"""{
"info": {
- "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info": 1711787325,
+ "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info": 1711787325,
"/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info": 1735158257,
"/usr/share/info": 1650633847
},
@@ -199,12 +190,8 @@ _TWO_RESUME_LISTS_JSON = b"""{
},
"starttime": 0,
"updates": {
- "/var/db/repos/gentoo/profiles/updates/1Q-2021": 1739992409,
"/var/db/repos/gentoo/profiles/updates/1Q-2022": 1747854791,
- "/var/db/repos/gentoo/profiles/updates/2Q-2021": 1724404379,
"/var/db/repos/gentoo/profiles/updates/2Q-2022": 1752846209,
- "/var/db/repos/gentoo/profiles/updates/3Q-2021": 1741119203,
- "/var/db/repos/gentoo/profiles/updates/4Q-2020": 1709167362,
"/var/db/repos/gentoo/profiles/updates/4Q-2021": 1742787797
},
"version": "3.0.30"
@@ -215,7 +202,7 @@ _TWO_RESUME_LISTS_JSON = b"""{
class MtimeDBTestCase(TestCase):
text = b"Unit tests for MtimeDB"
- def test_instances_are_created_with_only_expected_keys(self):
+ def test_instances_created_with_only_expected_keys(self):
all_fixtures = (
_ONE_RESUME_LIST_JSON,
_EMPTY_FILE,
@@ -229,7 +216,7 @@ class MtimeDBTestCase(TestCase):
mtimedb = MtimeDB("/path/to/mtimedb")
self.assertLessEqual(set(mtimedb.keys()), _MTIMEDBKEYS)
- def test_instances_have_default_values(self):
+ def test_default_values(self):
with patch("portage.util.mtimedb.open",
mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
@@ -246,6 +233,78 @@ class MtimeDBTestCase(TestCase):
self.assertEqual(dict(mtimedb), dict(mtimedb._clean_data))
self.assertIsNot(mtimedb, mtimedb._clean_data)
+ def test_load_data_called_at_instance_creation_time(self):
+ with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ mtimedb = MtimeDB("/some/path/mtimedb")
+ self.assertEqual(
+ mtimedb["info"],
+ {
+ "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.37/info": (
+ 1711787325),
+ "/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info": (
+ 1735158257),
+ "/usr/share/info": 1650633847
+ }
+ )
+ self.assertEqual(
+ mtimedb["ldpath"],
+ {
+ "/lib": 1748456830,
+ "/lib64": 1750523381,
+ "/usr/lib": 1750461195,
+ "/usr/lib/llvm/13/lib64": 1747003135,
+ "/usr/lib/rust/lib": 1750461173,
+ "/usr/lib64": 1750881821,
+ "/usr/local/lib": 1711784303,
+ "/usr/local/lib64": 1711784303
+ }
+ )
+ self.assertEqual(
+ mtimedb["resume"],
+ {
+ "favorites": [
+ "@world"
+ ],
+ "mergelist": [
+ [
+ "ebuild",
+ "/",
+ "some-cat/some-package-1.2.3-r4",
+ "merge"
+ ],
+ [
+ "ebuild",
+ "/",
+ "another-cat/another-package-4.3.2-r1",
+ "merge"
+ ]
+ ],
+ "myopts": {
+ "--buildpkg": True,
+ "--deep": True,
+ "--getbinpkg": True,
+ "--keep-going": True,
+ "--newuse": True,
+ "--quiet": True,
+ "--regex-search-auto": "y",
+ "--update": True,
+ "--usepkg": True,
+ "--verbose": True
+ }
+ }
+ )
+ self.assertEqual(mtimedb["starttime"], 0)
+ self.assertEqual(
+ mtimedb["updates"],
+ {
+ "/var/db/repos/gentoo/profiles/updates/1Q-2022": 1747854791,
+ "/var/db/repos/gentoo/profiles/updates/2Q-2022": 1752846209,
+ "/var/db/repos/gentoo/profiles/updates/4Q-2021": 1742787797
+ }
+ )
+ self.assertEqual(mtimedb["version"], "3.0.30")
+
@patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
def test_commit_writes_to_disk_if_needed_and_possible(self, pwrite2disk):
with patch("portage.util.mtimedb.open",
@@ -296,3 +355,24 @@ class MtimeDBTestCase(TestCase):
mtimedb.make_readonly()
self.assertTrue(mtimedb.is_readonly)
self.assertIs(mtimedb.filename, None)
+
+ @patch("portage.util.mtimedb.apply_secpass_permissions")
+ @patch("portage.util.mtimedb.atomic_ofstream")
+ def test_write_to_disk(self, matomic_ofstream, mapply_perms):
+ with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+ mtimedb = MtimeDB("/some/path/mtimedb")
+ d = {"z": "zome", "a": "AAA"}
+ encoding = portage._encodings["repo.content"]
+ # I'm interested here in unit testing, but the method is private
+ # and shouldn't be called directly from outside, obviously:
+ mtimedb._MtimeDB__write_to_disk(d)
+ self.assertEqual(d["version"], str(portage.VERSION))
+ matomic_ofstream.return_value.write.assert_called_once_with(
+ json.dumps(d, **mtimedb._json_write_opts).encode(encoding)
+ )
+ mapply_perms.assert_called_once_with(
+ mtimedb.filename, uid=uid, gid=portage_gid, mode=0o644
+ )
+ self.assertEqual(d, mtimedb._clean_data)
+ self.assertIsNot(d, mtimedb._clean_data)
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index 95da2235f..9884746f3 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -133,6 +133,7 @@ class MtimeDB(dict):
self.__write_to_disk(d)
def __write_to_disk(self, d):
+ """Private method used by the ``commit`` method."""
d["version"] = str(portage.VERSION)
try:
f = atomic_ofstream(self.filename, mode="wb")
^ permalink raw reply related [flat|nested] 7+ messages in thread* [gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
@ 2022-06-07 23:48 Mike Gilbert
0 siblings, 0 replies; 7+ messages in thread
From: Mike Gilbert @ 2022-06-07 23:48 UTC (permalink / raw
To: gentoo-commits
commit: c79c8898dc4ccd85829d00e7104b8e69ae8aa7f4
Author: David Palao <david.palao <AT> gmail <DOT> com>
AuthorDate: Wed May 25 15:23:20 2022 +0000
Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org>
CommitDate: Tue Jun 7 23:47:55 2022 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c79c8898
refactor(mtimedb): MtimeDB.commit splitted into two methods
- commit itself handles only the logic of when to write to disk
- __write_to_disk performs the actual writing
Signed-off-by: David Palao <david.palao <AT> gmail.com>
Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 37 +++++++++++++++++++++++++++---
lib/portage/util/mtimedb.py | 41 ++++++++++++++++++----------------
2 files changed, 56 insertions(+), 22 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py b/lib/portage/tests/util/test_mtimedb.py
index e6ddf5b80..a65a6be91 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -222,13 +222,13 @@ class MtimeDBTestCase(TestCase):
)
for contents in all_fixtures:
with patch(
- 'portage.util.mtimedb.open', mock_open(read_data=contents)
+ "portage.util.mtimedb.open", mock_open(read_data=contents)
):
mtimedb = MtimeDB("/path/to/mtimedb")
self.assertLessEqual(set(mtimedb.keys()), _MTIMEDBKEYS)
def test_instance_has_default_values(self):
- with patch('portage.util.mtimedb.open',
+ with patch("portage.util.mtimedb.open",
mock_open(read_data=_EMPTY_FILE)):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(mtimedb["starttime"], 0)
@@ -238,8 +238,39 @@ class MtimeDBTestCase(TestCase):
self.assertEqual(mtimedb["updates"], {})
def test_instance_has_a_deepcopy_of_clean_data(self):
- with patch('portage.util.mtimedb.open',
+ with patch("portage.util.mtimedb.open",
mock_open(read_data=_ONE_RESUME_LIST_JSON)):
mtimedb = MtimeDB("/some/path/mtimedb")
self.assertEqual(dict(mtimedb), dict(mtimedb._clean_data))
self.assertIsNot(mtimedb, mtimedb._clean_data)
+
+ @patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
+ def test_commit_writes_to_disk_if_needed_and_possible(self, pwrite2disk):
+ with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_EMPTY_FILE)):
+ mtimedb = MtimeDB("/some/path/mtimedb")
+ mtimedb.commit()
+ pwrite2disk.assert_not_called()
+ mtimedb["updates"]["/long/path/1Q-2021"] = 1739992409
+ d = {}
+ d.update(mtimedb)
+ mtimedb.commit()
+ pwrite2disk.assert_called_once_with(d)
+
+ @patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
+ def test_commit_does_not_write_to_disk_if_no_file(self, pwrite2disk):
+ with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_EMPTY_FILE)):
+ mtimedb = MtimeDB("/some/path/mtimedb")
+ mtimedb["updates"]["/long/path/1Q-2021"] = 1739992409
+ mtimedb.filename = None
+ mtimedb.commit()
+ pwrite2disk.assert_not_called()
+
+ @patch("portage.util.mtimedb.MtimeDB._MtimeDB__write_to_disk")
+ def test_commit_does_not_write_to_disk_if_no_changes(self, pwrite2disk):
+ with patch("portage.util.mtimedb.open",
+ mock_open(read_data=_EMPTY_FILE)):
+ mtimedb = MtimeDB("/some/path/mtimedb")
+ mtimedb.commit()
+ pwrite2disk.assert_not_called()
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index 3aab0b90b..a6566e3f8 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -120,24 +120,27 @@ class MtimeDB(dict):
d.update(self)
# Only commit if the internal state has changed.
if d != self._clean_data:
- d["version"] = str(portage.VERSION)
- try:
- f = atomic_ofstream(self.filename, mode="wb")
- except EnvironmentError:
- pass
- else:
- if self._json_write:
- f.write(
- _unicode_encode(
- json.dumps(d, **self._json_write_opts),
- encoding=_encodings["repo.content"],
- errors="strict",
- )
+ self.__write_to_disk(d)
+
+ def __write_to_disk(self, d):
+ d["version"] = str(portage.VERSION)
+ try:
+ f = atomic_ofstream(self.filename, mode="wb")
+ except EnvironmentError:
+ pass
+ else:
+ if self._json_write:
+ f.write(
+ _unicode_encode(
+ json.dumps(d, **self._json_write_opts),
+ encoding=_encodings["repo.content"],
+ errors="strict",
)
- else:
- pickle.dump(d, f, protocol=2)
- f.close()
- apply_secpass_permissions(
- self.filename, uid=uid, gid=portage_gid, mode=0o644
)
- self._clean_data = copy.deepcopy(d)
+ else:
+ pickle.dump(d, f, protocol=2)
+ f.close()
+ apply_secpass_permissions(
+ self.filename, uid=uid, gid=portage_gid, mode=0o644
+ )
+ self._clean_data = copy.deepcopy(d)
^ permalink raw reply related [flat|nested] 7+ messages in thread* [gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/
@ 2019-01-16 8:33 Zac Medico
0 siblings, 0 replies; 7+ messages in thread
From: Zac Medico @ 2019-01-16 8:33 UTC (permalink / raw
To: gentoo-commits
commit: 035582f0e31c071606635aac9cc4ba4b411612e7
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 14 08:11:57 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jan 16 07:48:59 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=035582f0
tests: add unit test for portage.util.socks5 (FEATURES=network-sandbox-proxy)
Bug: https://bugs.gentoo.org/604474
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/portage/tests/util/test_socks5.py | 211 ++++++++++++++++++++++++++++++++++
lib/portage/util/socks5.py | 48 +++++++-
2 files changed, 256 insertions(+), 3 deletions(-)
diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py
new file mode 100644
index 000000000..5db85b0a6
--- /dev/null
+++ b/lib/portage/tests/util/test_socks5.py
@@ -0,0 +1,211 @@
+# Copyright 2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import functools
+import platform
+import shutil
+import socket
+import struct
+import sys
+import tempfile
+import time
+
+import portage
+from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
+from portage.util import socks5
+from portage.const import PORTAGE_BIN_PATH
+
+try:
+ from http.server import BaseHTTPRequestHandler, HTTPServer
+except ImportError:
+ from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+
+try:
+ from urllib.request import urlopen
+except ImportError:
+ from urllib import urlopen
+
+
+class _Handler(BaseHTTPRequestHandler):
+
+ def __init__(self, content, *args, **kwargs):
+ self.content = content
+ BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
+
+ def do_GET(self):
+ doc = self.send_head()
+ if doc is not None:
+ self.wfile.write(doc)
+
+ def do_HEAD(self):
+ self.send_head()
+
+ def send_head(self):
+ doc = self.content.get(self.path)
+ if doc is None:
+ self.send_error(404, "File not found")
+ return None
+
+ self.send_response(200)
+ self.send_header("Content-type", "text/plain")
+ self.send_header("Content-Length", len(doc))
+ self.send_header("Last-Modified", self.date_time_string(time.time()))
+ self.end_headers()
+ return doc
+
+ def log_message(self, fmt, *args):
+ pass
+
+
+class AsyncHTTPServer(object):
+ def __init__(self, host, content, loop):
+ self._host = host
+ self._content = content
+ self._loop = loop
+ self.server_port = None
+ self._httpd = None
+
+ def __enter__(self):
+ httpd = self._httpd = HTTPServer((self._host, 0), functools.partial(_Handler, self._content))
+ self.server_port = httpd.server_port
+ self._loop.add_reader(httpd.socket.fileno(), self._httpd._handle_request_noblock)
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ if self._httpd is not None:
+ self._loop.remove_reader(self._httpd.socket.fileno())
+ self._httpd.socket.close()
+ self._httpd = None
+
+
+class AsyncHTTPServerTestCase(TestCase):
+
+ @staticmethod
+ def _fetch_directly(host, port, path):
+ # NOTE: python2.7 does not have context manager support here
+ try:
+ f = urlopen('http://{host}:{port}{path}'.format( # nosec
+ host=host, port=port, path=path))
+ return f.read()
+ finally:
+ if f is not None:
+ f.close()
+
+ def test_http_server(self):
+ host = '127.0.0.1'
+ content = b'Hello World!\n'
+ path = '/index.html'
+ loop = global_event_loop()
+ for i in range(2):
+ with AsyncHTTPServer(host, {path: content}, loop) as server:
+ for j in range(2):
+ result = loop.run_until_complete(loop.run_in_executor(None,
+ self._fetch_directly, host, server.server_port, path))
+ self.assertEqual(result, content)
+
+
+class _socket_file_wrapper(portage.proxy.objectproxy.ObjectProxy):
+ """
+ A file-like object that wraps a socket and closes the socket when
+ closed. Since python2.7 does not support socket.detach(), this is a
+ convenient way to have a file attached to a socket that closes
+ automatically (without resource warnings about unclosed sockets).
+ """
+
+ __slots__ = ('_file', '_socket')
+
+ def __init__(self, socket, f):
+ object.__setattr__(self, '_socket', socket)
+ object.__setattr__(self, '_file', f)
+
+ def _get_target(self):
+ return object.__getattribute__(self, '_file')
+
+ def __getattribute__(self, attr):
+ if attr == 'close':
+ return object.__getattribute__(self, 'close')
+ return super(_socket_file_wrapper, self).__getattribute__(attr)
+
+ def __enter__(self):
+ return self
+
+ def close(self):
+ object.__getattribute__(self, '_file').close()
+ object.__getattribute__(self, '_socket').close()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+
+def socks5_http_get_ipv4(proxy, host, port, path):
+ """
+ Open http GET request via socks5 proxy listening on a unix socket,
+ and return a file to read the response body from.
+ """
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ f = _socket_file_wrapper(s, s.makefile('rb', 1024))
+ try:
+ s.connect(proxy)
+ s.send(struct.pack('!BBB', 0x05, 0x01, 0x00))
+ vers, method = struct.unpack('!BB', s.recv(2))
+ s.send(struct.pack('!BBBB', 0x05, 0x01, 0x00, 0x01))
+ s.send(socket.inet_pton(socket.AF_INET, host))
+ s.send(struct.pack('!H', port))
+ reply = struct.unpack('!BBB', s.recv(3))
+ if reply != (0x05, 0x00, 0x00):
+ raise AssertionError(repr(reply))
+ struct.unpack('!B4sH', s.recv(7)) # contains proxied address info
+ s.send("GET {} HTTP/1.1\r\nHost: {}:{}\r\nAccept: */*\r\nConnection: close\r\n\r\n".format(
+ path, host, port).encode())
+ headers = []
+ while True:
+ headers.append(f.readline())
+ if headers[-1] == b'\r\n':
+ return f
+ except Exception:
+ f.close()
+ raise
+
+
+class Socks5ServerTestCase(TestCase):
+
+ @staticmethod
+ def _fetch_via_proxy(proxy, host, port, path):
+ with socks5_http_get_ipv4(proxy, host, port, path) as f:
+ return f.read()
+
+ def test_socks5_proxy(self):
+
+ loop = global_event_loop()
+
+ host = '127.0.0.1'
+ content = b'Hello World!'
+ path = '/index.html'
+ proxy = None
+ tempdir = tempfile.mkdtemp()
+
+ try:
+ with AsyncHTTPServer(host, {path: content}, loop) as server:
+
+ settings = {
+ 'PORTAGE_TMPDIR': tempdir,
+ 'PORTAGE_BIN_PATH': PORTAGE_BIN_PATH,
+ }
+
+ try:
+ proxy = socks5.get_socks5_proxy(settings)
+ except NotImplementedError:
+ # bug 658172 for python2.7
+ self.skipTest('get_socks5_proxy not implemented for {} {}.{}'.format(
+ platform.python_implementation(), *sys.version_info[:2]))
+ else:
+ loop.run_until_complete(socks5.proxy.ready())
+
+ result = loop.run_until_complete(loop.run_in_executor(None,
+ self._fetch_via_proxy, proxy, host, server.server_port, path))
+
+ self.assertEqual(result, content)
+ finally:
+ socks5.proxy.stop()
+ shutil.rmtree(tempdir)
diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
index 74b0714eb..59e6699ec 100644
--- a/lib/portage/util/socks5.py
+++ b/lib/portage/util/socks5.py
@@ -1,13 +1,18 @@
# SOCKSv5 proxy manager for network-sandbox
-# Copyright 2015 Gentoo Foundation
+# Copyright 2015-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import errno
import os
import signal
+import socket
+import portage.data
from portage import _python_interpreter
from portage.data import portage_gid, portage_uid, userpriv_groups
from portage.process import atexit_register, spawn
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures import asyncio
class ProxyManager(object):
@@ -36,9 +41,16 @@ class ProxyManager(object):
self.socket_path = os.path.join(settings['PORTAGE_TMPDIR'],
'.portage.%d.net.sock' % os.getpid())
server_bin = os.path.join(settings['PORTAGE_BIN_PATH'], 'socks5-server.py')
+ spawn_kwargs = {}
+ # The portage_uid check solves EPERM failures in Travis CI.
+ if portage.data.secpass > 1 and os.geteuid() != portage_uid:
+ spawn_kwargs.update(
+ uid=portage_uid,
+ gid=portage_gid,
+ groups=userpriv_groups,
+ umask=0o077)
self._pids = spawn([_python_interpreter, server_bin, self.socket_path],
- returnpid=True, uid=portage_uid, gid=portage_gid,
- groups=userpriv_groups, umask=0o077)
+ returnpid=True, **spawn_kwargs)
def stop(self):
"""
@@ -60,6 +72,36 @@ class ProxyManager(object):
return self.socket_path is not None
+ @coroutine
+ def ready(self):
+ """
+ Wait for the proxy socket to become ready. This method is a coroutine.
+ """
+
+ while True:
+ try:
+ wait_retval = os.waitpid(self._pids[0], os.WNOHANG)
+ except OSError as e:
+ if e.errno == errno.EINTR:
+ continue
+ raise
+
+ if wait_retval is not None and wait_retval != (0, 0):
+ raise OSError(3, 'No such process')
+
+ try:
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ s.connect(self.socket_path)
+ except EnvironmentError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ yield asyncio.sleep(0.2)
+ else:
+ break
+ finally:
+ s.close()
+
+
proxy = ProxyManager()
^ permalink raw reply related [flat|nested] 7+ messages in thread
end of thread, other threads:[~2024-02-22 15:36 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-02-22 15:36 [gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/ Zac Medico
-- strict thread matches above, loose matches on Subject: below --
2022-12-31 13:33 Sam James
2022-06-07 23:48 Mike Gilbert
2022-06-07 23:48 Mike Gilbert
2022-06-07 23:48 Mike Gilbert
2022-06-07 23:48 Mike Gilbert
2019-01-16 8:33 Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox