public inbox for gentoo-commits@lists.gentoo.org
 help / color / mirror / Atom feed
* [gentoo-commits] proj/mirrorselect:ssl commit in: mirrorselect/
@ 2014-03-02  7:44 Brian Dolbec
  2014-03-02  7:44 ` [gentoo-commits] proj/mirrorselect:master " Brian Dolbec
  0 siblings, 1 reply; 7+ messages in thread
From: Brian Dolbec @ 2014-03-02  7:44 UTC (permalink / raw
  To: gentoo-commits

commit:     dc98292e2aeca8ec13aceb2efce802d9375e11aa
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Feb  1 03:49:11 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Feb  1 03:53:59 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=dc98292e

Tweak the proxy code to correctly handle the cli option.

---
 mirrorselect/extractor.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 217d1e1..eb26faf 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -58,10 +58,11 @@ class Extractor(object):
 		self.proxies = {}
 
 		for proxy in ['http_proxy', 'https_proxy']:
-			if options.proxy:
-				self.proxies[proxy.split('_')[0]] = options.proxy
+			prox = proxy.split('_')[0]
+			if options.proxy and prox + ":" in options.proxy:
+				self.proxies[prox] = options.proxy
 			elif os.getenv(proxy):
-				self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
+				self.proxies[prox] = os.getenv(proxy)
 
 		parser = MirrorParser3()
 		self.hosts = []


^ permalink raw reply related	[flat|nested] 7+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02  7:44 Brian Dolbec
  2014-01-31 15:44 ` [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
  0 siblings, 1 reply; 7+ messages in thread
From: Brian Dolbec @ 2014-03-02  7:44 UTC (permalink / raw
  To: gentoo-commits

commit:     42ad8fd9f720e52fa578dccde9591bac2ea97c7d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 31 15:36:00 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Jan 31 15:39:18 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=42ad8fd9

Indent adjustment

---
 mirrorselect/main.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 4c26c02..15b8ead 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -140,16 +140,16 @@ class MirrorSelect(object):
 		need some finishing touches.
 		"""
 		desc = "\n".join((
-				self.output.white("examples:"),
-				"",
-				self.output.white("	 automatic:"),
-				"		 # mirrorselect -s5",
-				"		 # mirrorselect -s3 -b10 -o >> /mnt/gentoo/etc/portage/make.conf",
-				"		 # mirrorselect -D -s4",
-				"",
-				self.output.white("	 interactive:"),
-				"		 # mirrorselect -i -r",
-				))
+			self.output.white("examples:"),
+			"",
+			self.output.white("	 automatic:"),
+			"		 # mirrorselect -s5",
+			"		 # mirrorselect -s3 -b10 -o >> /mnt/gentoo/etc/portage/make.conf",
+			"		 # mirrorselect -D -s4",
+			"",
+			self.output.white("	 interactive:"),
+			"		 # mirrorselect -i -r",
+			))
 		parser = OptionParser(
 			formatter=ColoredFormatter(self.output), description=desc,
 			version='Mirrorselect version: %s' % version)


^ permalink raw reply related	[flat|nested] 7+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02  7:44 Brian Dolbec
  2014-01-31 15:44 ` [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
  0 siblings, 1 reply; 7+ messages in thread
From: Brian Dolbec @ 2014-03-02  7:44 UTC (permalink / raw
  To: gentoo-commits

commit:     a3b025b7f3d5e67b1735bb773b72b5766537225f
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 23 22:53:08 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Jan 23 22:53:08 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=a3b025b7

Remove connections.py. Instead moved it to it's own ssl-fetch pkg.

---
 mirrorselect/connections.py | 182 --------------------------------------------
 1 file changed, 182 deletions(-)

diff --git a/mirrorselect/connections.py b/mirrorselect/connections.py
deleted file mode 100644
index ca4aa88..0000000
--- a/mirrorselect/connections.py
+++ /dev/null
@@ -1,182 +0,0 @@
-#-*- coding:utf-8 -*-
-
-"""Mirrorselect 2.x
- Tool for selecting Gentoo source and rsync mirrors.
-
-Copyright 2005-2012 Gentoo Foundation
-
-	Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
-	Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
-	Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
-	Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
-	Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org>
-
-Distributed under the terms of the GNU General Public License v2
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 2 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
-
-"""
-
-import sys
-import os
-
-VERIFY_SSL = False
-VERIFY_MSGS = []
-
-import requests
-from requests.exceptions import SSLError
-
-# py3.2
-if sys.hexversion >= 0x30200f0:
-	VERIFY_SSL = True
-else:
-	try: # import and enable SNI support for py2
-		from requests.packages.urllib3.contrib import pyopenssl
-		pyopenssl.inject_into_urllib3()
-		VERIFY_SSL = True
-		VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
-	except ImportError as e:
-		VERIFY_MSGS = [
-			"Failed to import and inject pyopenssl/SNI support into urllib3",
-			"Disabling certificate verification",
-			"Error was:" + e
-		]
-		VERIFY_SSL = False
-
-
-from mirrorselect.version import version
-
-
-class Connector(object):
-	"""Primary connection interface using the dev-python/requests package
-	"""
-
-	def __init__(self, output, proxies):
-		self.output = output
-		self.proxies = proxies
-		self.headers = {'Accept-Charset': 'utf-8',
-			'User-Agent': 'Mirrorselect-' + version}
-
-		if VERIFY_MSGS:
-			for msg in VERIFY_MSGS:
-				self.output.write(msg + '\n', 2)
-
-
-	def add_timestamp(self, headers, tpath=None, timestamp=None):
-		"""for possilble future caching of the list"""
-		if tpath and os.path.exists(tpath):
-			# fileopen is a layman comaptibility function not yet implemented here
-			with fileopen(tpath,'r') as previous:
-				timestamp = previous.read()
-		if timestamp:
-			headers['If-Modified-Since'] = timestamp
-			self.output.write('Current-modified: %s\n' % timestamp, 2)
-		return headers
-
-
-	def fetch_url(self, url, headers=None, timestamp=None):
-		"""Fetches the url
-
-		@param url: string
-		@param headers: dictionary, optional headers to use
-		@param tpath: string, optional filepath to a timestamp file
-					  to use in the headers
-		@param timestamp: string, optional timestamp to use in the headers
-
-		"""
-
-		if not headers:
-			headers = self.headers
-
-		if timestamp:
-			self.add_timestamp(headers, timestamp=timestamp)
-
-		verify = 'https' in url and VERIFY_SSL
-		self.output.write("Enabled ssl certificate verification: %s, for: %s\n"
-			%(str(verify), url), 3)
-
-		self.output.write('Connector.fetch_url(); headers = %s\n' %str(headers), 4)
-		self.output.write('Connector.fetch_url(); connecting to opener\n', 2)
-
-		try:
-			connection = requests.get(
-				url,
-				headers=headers,
-				verify=verify,
-				proxies=self.proxies,
-				)
-		except SSLError as error:
-			self.output.print_err('Connector.fetch_url(); Failed to update the '
-				'mirror list from: %s\nSSLError was:%s\n'
-				% (url, str(error)))
-		except Exception as error:
-			self.output.print_err('Connector.fetch_url(); Failed to retrieve '
-				'the content from: %s\nError was: %s\n'
-				% (url, str(error)))
-
-		self.output.write('Connector.fetch_url() HEADERS = %s\n' %str(connection.headers), 4)
-		self.output.write('Connector.fetch_url() Status_code = %i\n' % connection.status_code, 2)
-		return connection
-
-
-	@staticmethod
-	def normalize_headers(headers, to_lower=True):
-		""" py2, py3 compatibility function, since only py2 returns keys as lower()
-		"""
-		if to_lower:
-			return dict((x.lower(), x) for x in list(headers))
-		return dict((x.upper(), x) for x in list(headers))
-
-
-	def fetch_content(self, url, tpath=None):
-		"""Fetch the mirror list
-
-		@param url: string of the content to fetch
-		@param headers: dictionary, optional headers to use
-		@param tpath: string, optional filepath to a timestamp file
-					  to use in the headers
-		@returns (success bool, content fetched , timestamp of fetched content,
-				 content headers returned)
-		"""
-
-		fheaders = self.headers
-
-		if tpath:
-			fheaders = self.add_timestamp(fheaders, tpath)
-
-		connection = self.fetch_url(url, fheaders)
-
-		headers = self.normalize_headers(connection.headers)
-
-		if 'last-modified' in headers:
-			timestamp = headers['last-modified']
-		elif 'date' in headers:
-			timestamp = headers['date']
-		else:
-			timestamp = None
-
-		if connection.status_code in [304]:
-			self.output.write('Content already up to date: %s\n'
-				% url, 4)
-			self.output.write('Last-modified: %s\n' % timestamp, 4)
-		elif connection.status_code not in [200]:
-			self.output.print_err('Connector.fetch_content(); HTTP Status-Code was:\n'
-				'url: %s\n%s'
-				% (url, str(connection.status_code)))
-
-		if connection.status_code in [200]:
-			self.output.write('New content downloaded for: %s\n'
-				% url, 4)
-			return (True, connection.content, timestamp)
-		return (False, '', '')
-


^ permalink raw reply related	[flat|nested] 7+ messages in thread
* [gentoo-commits] proj/mirrorselect:ssl commit in: mirrorselect/
@ 2014-01-31 15:44 Brian Dolbec
  0 siblings, 0 replies; 7+ messages in thread
From: Brian Dolbec @ 2014-01-31 15:44 UTC (permalink / raw
  To: gentoo-commits

commit:     e1d0a9d4610eaafb9297e896ba4ed264e5fe939f
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 23 22:56:44 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Jan 23 22:56:44 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=e1d0a9d4

Use the new sslfetch pkg for the Connector class.

---
 mirrorselect/extractor.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index c8d5bd5..a949c75 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -30,8 +30,10 @@ Distributed under the terms of the GNU General Public License v2
 import os
 
 from mirrorselect.mirrorparser3 import MirrorParser3
-from mirrorselect.connections import Connector
+from sslfetch.connections import Connector
+from mirrorselect.version import version
 
+USERAGENT = "Mirrorselect-" + version
 
 class Extractor(object):
 	"""The Extractor employs a MirrorParser3 object to get a list of valid
@@ -101,7 +103,7 @@ class Extractor(object):
 
 		self.output.print_info('Downloading a list of mirrors...\n')
 
-		fetcher = Connector(self.output, self.proxies)
+		fetcher = Connector(self.output, self.proxies, USERAGENT)
 		success, mirrorlist, timestamp = fetcher.fetch_content(url)
 		parser.parse(mirrorlist)
 


^ permalink raw reply related	[flat|nested] 7+ messages in thread
* [gentoo-commits] proj/mirrorselect:ssl commit in: mirrorselect/
@ 2013-10-20 18:19 Brian Dolbec
  0 siblings, 0 replies; 7+ messages in thread
From: Brian Dolbec @ 2013-10-20 18:19 UTC (permalink / raw
  To: gentoo-commits

commit:     eabccfc8eeef5d0c76a444545fa390f400a9d4ed
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 05:20:39 2013 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Oct 20 18:14:21 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=eabccfc8

Work in progress for adding ssl support for downloading the mirrors lists.

add the request code in it's own file and class.

---
 mirrorselect/connections.py | 182 ++++++++++++++++++++++++++++++++++++++++++++
 mirrorselect/extractor.py   |  36 ++++-----
 2 files changed, 200 insertions(+), 18 deletions(-)

diff --git a/mirrorselect/connections.py b/mirrorselect/connections.py
new file mode 100644
index 0000000..ca4aa88
--- /dev/null
+++ b/mirrorselect/connections.py
@@ -0,0 +1,182 @@
+#-*- coding:utf-8 -*-
+
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2005-2012 Gentoo Foundation
+
+	Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
+	Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
+	Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+	Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+	Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
+
+import sys
+import os
+
+VERIFY_SSL = False
+VERIFY_MSGS = []
+
+import requests
+from requests.exceptions import SSLError
+
+# py3.2
+if sys.hexversion >= 0x30200f0:
+	VERIFY_SSL = True
+else:
+	try: # import and enable SNI support for py2
+		from requests.packages.urllib3.contrib import pyopenssl
+		pyopenssl.inject_into_urllib3()
+		VERIFY_SSL = True
+		VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
+	except ImportError as e:
+		VERIFY_MSGS = [
+			"Failed to import and inject pyopenssl/SNI support into urllib3",
+			"Disabling certificate verification",
+			"Error was:" + e
+		]
+		VERIFY_SSL = False
+
+
+from mirrorselect.version import version
+
+
+class Connector(object):
+	"""Primary connection interface using the dev-python/requests package
+	"""
+
+	def __init__(self, output, proxies):
+		self.output = output
+		self.proxies = proxies
+		self.headers = {'Accept-Charset': 'utf-8',
+			'User-Agent': 'Mirrorselect-' + version}
+
+		if VERIFY_MSGS:
+			for msg in VERIFY_MSGS:
+				self.output.write(msg + '\n', 2)
+
+
+	def add_timestamp(self, headers, tpath=None, timestamp=None):
+		"""for possilble future caching of the list"""
+		if tpath and os.path.exists(tpath):
+			# fileopen is a layman comaptibility function not yet implemented here
+			with fileopen(tpath,'r') as previous:
+				timestamp = previous.read()
+		if timestamp:
+			headers['If-Modified-Since'] = timestamp
+			self.output.write('Current-modified: %s\n' % timestamp, 2)
+		return headers
+
+
+	def fetch_url(self, url, headers=None, timestamp=None):
+		"""Fetches the url
+
+		@param url: string
+		@param headers: dictionary, optional headers to use
+		@param tpath: string, optional filepath to a timestamp file
+					  to use in the headers
+		@param timestamp: string, optional timestamp to use in the headers
+
+		"""
+
+		if not headers:
+			headers = self.headers
+
+		if timestamp:
+			self.add_timestamp(headers, timestamp=timestamp)
+
+		verify = 'https' in url and VERIFY_SSL
+		self.output.write("Enabled ssl certificate verification: %s, for: %s\n"
+			%(str(verify), url), 3)
+
+		self.output.write('Connector.fetch_url(); headers = %s\n' %str(headers), 4)
+		self.output.write('Connector.fetch_url(); connecting to opener\n', 2)
+
+		try:
+			connection = requests.get(
+				url,
+				headers=headers,
+				verify=verify,
+				proxies=self.proxies,
+				)
+		except SSLError as error:
+			self.output.print_err('Connector.fetch_url(); Failed to update the '
+				'mirror list from: %s\nSSLError was:%s\n'
+				% (url, str(error)))
+		except Exception as error:
+			self.output.print_err('Connector.fetch_url(); Failed to retrieve '
+				'the content from: %s\nError was: %s\n'
+				% (url, str(error)))
+
+		self.output.write('Connector.fetch_url() HEADERS = %s\n' %str(connection.headers), 4)
+		self.output.write('Connector.fetch_url() Status_code = %i\n' % connection.status_code, 2)
+		return connection
+
+
+	@staticmethod
+	def normalize_headers(headers, to_lower=True):
+		""" py2, py3 compatibility function, since only py2 returns keys as lower()
+		"""
+		if to_lower:
+			return dict((x.lower(), x) for x in list(headers))
+		return dict((x.upper(), x) for x in list(headers))
+
+
+	def fetch_content(self, url, tpath=None):
+		"""Fetch the mirror list
+
+		@param url: string of the content to fetch
+		@param headers: dictionary, optional headers to use
+		@param tpath: string, optional filepath to a timestamp file
+					  to use in the headers
+		@returns (success bool, content fetched , timestamp of fetched content,
+				 content headers returned)
+		"""
+
+		fheaders = self.headers
+
+		if tpath:
+			fheaders = self.add_timestamp(fheaders, tpath)
+
+		connection = self.fetch_url(url, fheaders)
+
+		headers = self.normalize_headers(connection.headers)
+
+		if 'last-modified' in headers:
+			timestamp = headers['last-modified']
+		elif 'date' in headers:
+			timestamp = headers['date']
+		else:
+			timestamp = None
+
+		if connection.status_code in [304]:
+			self.output.write('Content already up to date: %s\n'
+				% url, 4)
+			self.output.write('Last-modified: %s\n' % timestamp, 4)
+		elif connection.status_code not in [200]:
+			self.output.print_err('Connector.fetch_content(); HTTP Status-Code was:\n'
+				'url: %s\n%s'
+				% (url, str(connection.status_code)))
+
+		if connection.status_code in [200]:
+			self.output.write('New content downloaded for: %s\n'
+				% url, 4)
+			return (True, connection.content, timestamp)
+		return (False, '', '')
+

diff --git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 3a113fb..c8d5bd5 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -27,20 +27,10 @@ Distributed under the terms of the GNU General Public License v2
 
 """
 
-import sys
-
-if sys.version_info[0] >= 3:
-	import urllib.request, urllib.parse, urllib.error
-	url_parse = urllib.parse
-	url_open = urllib.request.urlopen
-else:
-	import urllib
-	import urlparse
-	url_parse = urlparse.urlparse
-	url_open = urllib.urlopen
-
+import os
 
 from mirrorselect.mirrorparser3 import MirrorParser3
+from mirrorselect.connections import Connector
 
 
 class Extractor(object):
@@ -50,6 +40,7 @@ class Extractor(object):
 
 	def __init__(self, list_url, options, output):
 		self.output = output
+		self.output.print_info('Using url: %s\n' % list_url)
 		filters = {}
 		for opt in ["country", "region"]:
 			value = getattr(options, opt)
@@ -61,6 +52,15 @@ class Extractor(object):
 			if getattr(options, opt):
 				filters["proto"] = opt
 				self.output.print_info('Limiting test to %s hosts. \n' % opt )
+
+		self.proxies = {}
+
+		for proxy in ['http_proxy', 'https_proxy']:
+			if options.proxy:
+				self.proxies[proxy.split('_')[0]] = options.proxy
+			elif os.getenv(proxy):
+				self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
+
 		parser = MirrorParser3()
 		self.hosts = []
 
@@ -99,14 +99,13 @@ class Extractor(object):
 
 		self.output.write('getlist(): fetching ' + url + '\n', 2)
 
-		self.output.print_info('Downloading a list of mirrors...')
+		self.output.print_info('Downloading a list of mirrors...\n')
 
-		try:
-			parser.parse(url_open(url).read())
-		except EnvironmentError:
-			pass
+		fetcher = Connector(self.output, self.proxies)
+		success, mirrorlist, timestamp = fetcher.fetch_content(url)
+		parser.parse(mirrorlist)
 
-		if len(parser.tuples()) == 0:
+		if (not mirrorlist) or len(parser.tuples()) == 0:
 			self.output.print_err('Could not get mirror list. '
 				'Check your internet connection.')
 
@@ -114,3 +113,4 @@ class Extractor(object):
 
 		return parser.tuples()
 
+


^ permalink raw reply related	[flat|nested] 7+ messages in thread
* [gentoo-commits] proj/mirrorselect:ssl commit in: mirrorselect/
@ 2013-10-20  8:16 Brian Dolbec
  0 siblings, 0 replies; 7+ messages in thread
From: Brian Dolbec @ 2013-10-20  8:16 UTC (permalink / raw
  To: gentoo-commits

commit:     19d6681fc169e2fdb71fbcdfc0a3ffa9f72e92a6
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 08:15:19 2013 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Oct 20 08:15:19 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=19d6681f

scrap the separate py3 standars lib method and use requests

---
 mirrorselect/extractor.py | 80 +++++++++++++++++------------------------------
 1 file changed, 28 insertions(+), 52 deletions(-)

diff --git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index ea8c503..8853d80 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -32,21 +32,20 @@ import os
 
 VERIFY_SSL = False
 VERIFY_MSGS = []
+
+import requests
+from requests.exceptions import SSLError
+
 # py3.2
 if sys.hexversion >= 0x30200f0:
 	VERIFY_SSL = True
-	from urllib import request as requests
-	from ssl import SSLError
-	PY3 = True
 else:
 	try: # import and enable SNI support for py2
 		import requests
-		from requests.exceptions import SSLError
 		from requests.packages.urllib3.contrib import pyopenssl
 		pyopenssl.inject_into_urllib3()
 		VERIFY_SSL = True
 		VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
-		PY3 = False
 	except ImportError as e:
 		VERIFY_MSGS = [
 			"Failed to import and inject pyopenssl/SNI support into urllib3",
@@ -78,7 +77,6 @@ class Extractor(object):
 			if getattr(options, opt):
 				filters["proto"] = opt
 				self.output.print_info('Limiting test to %s hosts. \n' % opt )
-		self.storage = options.cache
 
 		self.proxies = {}
 
@@ -163,15 +161,27 @@ class Extractor(object):
 		self.output.write('Extractor._fetch_url(); headers = %s\n' %str(headers))
 		self.output.write('Extractor._fetch_url(); connecting to opener\n', 2)
 
-		if PY3:
-			fetcher = self._fetch_py3
-		else:
-			fetcher = self._fetch_py2
-		content, headers, status_code = fetcher(url, headers, verify)
+		try:
+			connection = requests.get(
+				url,
+				headers=headers,
+				verify=verify,
+				proxies=self.proxies,
+				)
+		except SSLError as error:
+			self.output.print_err('Extract._fetch_url(); Failed to update the '
+				'mirror list from: %s\nSSLError was:%s\n'
+				% (url, str(error)))
+		except Exception as error:
+			self.output.print_err('Extractor._fetch_url(); Failed to retrieve '
+				'the mirror list from: %s\nError was: %s\n'
+				% (url, str(error)))
 
-		self.output.write('HEADERS = %s\n' %str(headers), 4)
-		self.output.write('Status_code = %i\n' % status_code, 2)
+		self.output.write('HEADERS = %s\n' %str(connection.headers), 4)
+		self.output.write('Status_code = %i\n' % connection.status_code, 2)
 
+		# py2, py3 compatibility, since only py2 returns keys as lower()
+		headers = dict((x.lower(), x) for x in list(connection.headers))
 		if 'last-modified' in headers:
 			timestamp = headers['last-modified']
 		elif 'date' in headers:
@@ -179,52 +189,18 @@ class Extractor(object):
 		else:
 			timestamp = None
 
-		if status_code in [304]:
+		if connection.status_code in [304]:
 			self.output.write('Remote list already up to date: %s\n'
 				% url, 4)
 			self.output.write('Last-modified: %s\n' % timestamp, 4)
-		elif status_code not in [200]:
+		elif connection.status_code not in [200]:
 			self.output.print_err('Extractor._fetch_url(); HTTP Status-Code was:\n'
 				'url: %s\n%s'
-				% (url, str(status_code)))
+				% (url, str(connection.status_code)))
 
-		if status_code in [200]:
+		if connection.status_code in [200]:
 			self.output.write('New mirror list downloaded for: %s\n'
 				% url, 4)
-			return (True, content, timestamp)
+			return (True, connection.content, timestamp)
 		return (False, '', '')
 
-
-	def _fetch_py3(self, url, headers, verify):
-		data = None
-		status = None
-		try:
-			req = urllib.request.Request(url, data, headers)
-			response = urllib.request.urlopen(req)
-			content = response.get_url()
-			res_headers = response.info()
-		except:
-			pass
-		# py2, py3 compatibility, since only py2 returns keys as lower()
-		res_headers = dict((x.lower(), x) for x in list(res_headers))
-		return content, res_headers, status
-
-
-	def _fetch_py2(self, url, headers, verify):
-		try:
-			connection = requests.get(
-				url,
-				headers=headers,
-				verify=verify,
-				proxies=self.proxies,
-				)
-		except SSLError as error:
-			self.output.print_err('Extract._fetch_url(); Failed to update the '
-				'mirror list from: %s\nSSLError was:%s\n'
-				% (url, str(error)))
-		except Exception as error:
-			self.output.print_err('Extractor._fetch_url(); Failed to retrieve '
-				'the mirror list from: %s\nError was: %s\n'
-				% (url, str(error)))
-		return connection.content, connection.headers, connection.status_code
-


^ permalink raw reply related	[flat|nested] 7+ messages in thread
end of thread, other threads:[~2014-03-02  7:44 UTC | newest]

Thread overview: 7+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2014-03-02  7:44 [gentoo-commits] proj/mirrorselect:ssl commit in: mirrorselect/ Brian Dolbec
2014-03-02  7:44 ` [gentoo-commits] proj/mirrorselect:master " Brian Dolbec
  -- strict thread matches above, loose matches on Subject: below --
2014-03-02  7:44 Brian Dolbec
2014-01-31 15:44 ` [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
2014-03-02  7:44 [gentoo-commits] proj/mirrorselect:master " Brian Dolbec
2014-01-31 15:44 ` [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
2014-01-31 15:44 Brian Dolbec
2013-10-20 18:19 Brian Dolbec
2013-10-20  8:16 Brian Dolbec

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox