[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/tests/util/

["Zac Medico" <zmedico@gentoo.org>]

commit:     035582f0e31c071606635aac9cc4ba4b411612e7
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 14 08:11:57 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jan 16 07:48:59 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=035582f0

tests: add unit test for portage.util.socks5 (FEATURES=network-sandbox-proxy)

Bug: https://bugs.gentoo.org/604474
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/tests/util/test_socks5.py | 211 ++++++++++++++++++++++++++++++++++
 lib/portage/util/socks5.py            |  48 +++++++-
 2 files changed, 256 insertions(+), 3 deletions(-)

diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py
new file mode 100644
index 000000000..5db85b0a6
--- /dev/null
+++ b/lib/portage/tests/util/test_socks5.py
@@ -0,0 +1,211 @@
+# Copyright 2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import functools
+import platform
+import shutil
+import socket
+import struct
+import sys
+import tempfile
+import time
+
+import portage
+from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
+from portage.util import socks5
+from portage.const import PORTAGE_BIN_PATH
+
+try:
+	from http.server import BaseHTTPRequestHandler, HTTPServer
+except ImportError:
+	from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+
+try:
+	from urllib.request import urlopen
+except ImportError:
+	from urllib import urlopen
+
+
+class _Handler(BaseHTTPRequestHandler):
+
+	def __init__(self, content, *args, **kwargs):
+		self.content = content
+		BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
+
+	def do_GET(self):
+		doc = self.send_head()
+		if doc is not None:
+			self.wfile.write(doc)
+
+	def do_HEAD(self):
+		self.send_head()
+
+	def send_head(self):
+		doc = self.content.get(self.path)
+		if doc is None:
+			self.send_error(404, "File not found")
+			return None
+
+		self.send_response(200)
+		self.send_header("Content-type", "text/plain")
+		self.send_header("Content-Length", len(doc))
+		self.send_header("Last-Modified", self.date_time_string(time.time()))
+		self.end_headers()
+		return doc
+
+	def log_message(self, fmt, *args):
+		pass
+
+
+class AsyncHTTPServer(object):
+	def __init__(self, host, content, loop):
+		self._host = host
+		self._content = content
+		self._loop = loop
+		self.server_port = None
+		self._httpd = None
+
+	def __enter__(self):
+		httpd = self._httpd = HTTPServer((self._host, 0), functools.partial(_Handler, self._content))
+		self.server_port = httpd.server_port
+		self._loop.add_reader(httpd.socket.fileno(), self._httpd._handle_request_noblock)
+		return self
+
+	def __exit__(self, exc_type, exc_value, exc_traceback):
+		if self._httpd is not None:
+			self._loop.remove_reader(self._httpd.socket.fileno())
+			self._httpd.socket.close()
+			self._httpd = None
+
+
+class AsyncHTTPServerTestCase(TestCase):
+
+	@staticmethod
+	def _fetch_directly(host, port, path):
+		# NOTE: python2.7 does not have context manager support here
+		try:
+			f = urlopen('http://{host}:{port}{path}'.format( # nosec
+				host=host, port=port, path=path))
+			return f.read()
+		finally:
+			if f is not None:
+				f.close()
+
+	def test_http_server(self):
+		host = '127.0.0.1'
+		content = b'Hello World!\n'
+		path = '/index.html'
+		loop = global_event_loop()
+		for i in range(2):
+			with AsyncHTTPServer(host, {path: content}, loop) as server:
+				for j in range(2):
+					result = loop.run_until_complete(loop.run_in_executor(None,
+						self._fetch_directly, host, server.server_port, path))
+					self.assertEqual(result, content)
+
+
+class _socket_file_wrapper(portage.proxy.objectproxy.ObjectProxy):
+	"""
+	A file-like object that wraps a socket and closes the socket when
+	closed. Since python2.7 does not support socket.detach(), this is a
+	convenient way to have a file attached to a socket that closes
+	automatically (without resource warnings about unclosed sockets).
+	"""
+
+	__slots__ = ('_file', '_socket')
+
+	def __init__(self, socket, f):
+		object.__setattr__(self, '_socket', socket)
+		object.__setattr__(self, '_file', f)
+
+	def _get_target(self):
+		return object.__getattribute__(self, '_file')
+
+	def __getattribute__(self, attr):
+		if attr == 'close':
+			return object.__getattribute__(self, 'close')
+		return super(_socket_file_wrapper, self).__getattribute__(attr)
+
+	def __enter__(self):
+		return self
+
+	def close(self):
+		object.__getattribute__(self, '_file').close()
+		object.__getattribute__(self, '_socket').close()
+
+	def __exit__(self, exc_type, exc_value, traceback):
+		self.close()
+
+
+def socks5_http_get_ipv4(proxy, host, port, path):
+	"""
+	Open http GET request via socks5 proxy listening on a unix socket,
+	and return a file to read the response body from.
+	"""
+	s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+	f = _socket_file_wrapper(s, s.makefile('rb', 1024))
+	try:
+		s.connect(proxy)
+		s.send(struct.pack('!BBB', 0x05, 0x01, 0x00))
+		vers, method = struct.unpack('!BB', s.recv(2))
+		s.send(struct.pack('!BBBB', 0x05, 0x01, 0x00, 0x01))
+		s.send(socket.inet_pton(socket.AF_INET, host))
+		s.send(struct.pack('!H', port))
+		reply = struct.unpack('!BBB', s.recv(3))
+		if reply != (0x05, 0x00, 0x00):
+			raise AssertionError(repr(reply))
+		struct.unpack('!B4sH', s.recv(7)) # contains proxied address info
+		s.send("GET {} HTTP/1.1\r\nHost: {}:{}\r\nAccept: */*\r\nConnection: close\r\n\r\n".format(
+			path, host, port).encode())
+		headers = []
+		while True:
+			headers.append(f.readline())
+			if headers[-1] == b'\r\n':
+				return f
+	except Exception:
+		f.close()
+		raise
+
+
+class Socks5ServerTestCase(TestCase):
+
+	@staticmethod
+	def _fetch_via_proxy(proxy, host, port, path):
+		with socks5_http_get_ipv4(proxy, host, port, path) as f:
+			return f.read()
+
+	def test_socks5_proxy(self):
+
+		loop = global_event_loop()
+
+		host = '127.0.0.1'
+		content = b'Hello World!'
+		path = '/index.html'
+		proxy = None
+		tempdir = tempfile.mkdtemp()
+
+		try:
+			with AsyncHTTPServer(host, {path: content}, loop) as server:
+
+				settings = {
+					'PORTAGE_TMPDIR': tempdir,
+					'PORTAGE_BIN_PATH': PORTAGE_BIN_PATH,
+				}
+
+				try:
+					proxy = socks5.get_socks5_proxy(settings)
+				except NotImplementedError:
+					# bug 658172 for python2.7
+					self.skipTest('get_socks5_proxy not implemented for {} {}.{}'.format(
+						platform.python_implementation(), *sys.version_info[:2]))
+				else:
+					loop.run_until_complete(socks5.proxy.ready())
+
+					result = loop.run_until_complete(loop.run_in_executor(None,
+						self._fetch_via_proxy, proxy, host, server.server_port, path))
+
+					self.assertEqual(result, content)
+		finally:
+			socks5.proxy.stop()
+			shutil.rmtree(tempdir)

diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
index 74b0714eb..59e6699ec 100644
--- a/lib/portage/util/socks5.py
+++ b/lib/portage/util/socks5.py
@@ -1,13 +1,18 @@
 # SOCKSv5 proxy manager for network-sandbox
-# Copyright 2015 Gentoo Foundation
+# Copyright 2015-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+import errno
 import os
 import signal
+import socket
 
+import portage.data
 from portage import _python_interpreter
 from portage.data import portage_gid, portage_uid, userpriv_groups
 from portage.process import atexit_register, spawn
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures import asyncio
 
 
 class ProxyManager(object):
@@ -36,9 +41,16 @@ class ProxyManager(object):
 		self.socket_path = os.path.join(settings['PORTAGE_TMPDIR'],
 				'.portage.%d.net.sock' % os.getpid())
 		server_bin = os.path.join(settings['PORTAGE_BIN_PATH'], 'socks5-server.py')
+		spawn_kwargs = {}
+		# The portage_uid check solves EPERM failures in Travis CI.
+		if portage.data.secpass > 1 and os.geteuid() != portage_uid:
+			spawn_kwargs.update(
+				uid=portage_uid,
+				gid=portage_gid,
+				groups=userpriv_groups,
+				umask=0o077)
 		self._pids = spawn([_python_interpreter, server_bin, self.socket_path],
-				returnpid=True, uid=portage_uid, gid=portage_gid,
-				groups=userpriv_groups, umask=0o077)
+				returnpid=True, **spawn_kwargs)
 
 	def stop(self):
 		"""
@@ -60,6 +72,36 @@ class ProxyManager(object):
 		return self.socket_path is not None
 
 
+	@coroutine
+	def ready(self):
+		"""
+		Wait for the proxy socket to become ready. This method is a coroutine.
+		"""
+
+		while True:
+			try:
+				wait_retval = os.waitpid(self._pids[0], os.WNOHANG)
+			except OSError as e:
+				if e.errno == errno.EINTR:
+					continue
+				raise
+
+			if wait_retval is not None and wait_retval != (0, 0):
+				raise OSError(3, 'No such process')
+
+			try:
+				s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+				s.connect(self.socket_path)
+			except EnvironmentError as e:
+				if e.errno != errno.ENOENT:
+					raise
+				yield asyncio.sleep(0.2)
+			else:
+				break
+			finally:
+				s.close()
+
+
 proxy = ProxyManager()