From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id D47EB13873B for ; Sun, 2 Mar 2014 07:44:14 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id F3903E0AA8; Sun, 2 Mar 2014 07:44:11 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 47E4AE0AA8 for ; Sun, 2 Mar 2014 07:44:11 +0000 (UTC) Received: from spoonbill.gentoo.org (spoonbill.gentoo.org [81.93.255.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 5E82B33FB4D for ; Sun, 2 Mar 2014 07:44:10 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by spoonbill.gentoo.org (Postfix) with ESMTP id C78CC188EE for ; Sun, 2 Mar 2014 07:44:07 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1382292861.eabccfc8eeef5d0c76a444545fa390f400a9d4ed.dol-sen@gentoo> Subject: [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/ X-VCS-Repository: proj/mirrorselect X-VCS-Files: mirrorselect/connections.py mirrorselect/extractor.py X-VCS-Directories: mirrorselect/ X-VCS-Committer: dol-sen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: eabccfc8eeef5d0c76a444545fa390f400a9d4ed X-VCS-Branch: master Date: Sun, 2 Mar 2014 07:44:07 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: d85bbcf3-721b-4eef-ab5c-5cfdb0059bf0 X-Archives-Hash: d3d9d6f13b1722620cfffdff79f3e955 Message-ID: <20140302074407.IJQ-rc8YvE__KAEOqHKKyrsWXMiwgN-JZidOySsyRjQ@z> commit: eabccfc8eeef5d0c76a444545fa390f400a9d4ed Author: Brian Dolbec gentoo org> AuthorDate: Sun Oct 20 05:20:39 2013 +0000 Commit: Brian Dolbec gmail com> CommitDate: Sun Oct 20 18:14:21 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=eabccfc8 Work in progress for adding ssl support for downloading the mirrors lists. add the request code in it's own file and class. --- mirrorselect/connections.py | 182 ++++++++++++++++++++++++++++++++++++++++++++ mirrorselect/extractor.py | 36 ++++----- 2 files changed, 200 insertions(+), 18 deletions(-) diff --git a/mirrorselect/connections.py b/mirrorselect/connections.py new file mode 100644 index 0000000..ca4aa88 --- /dev/null +++ b/mirrorselect/connections.py @@ -0,0 +1,182 @@ +#-*- coding:utf-8 -*- + +"""Mirrorselect 2.x + Tool for selecting Gentoo source and rsync mirrors. + +Copyright 2005-2012 Gentoo Foundation + + Copyright (C) 2005 Colin Kingsley + Copyright (C) 2008 Zac Medico + Copyright (C) 2009 Sebastian Pipping + Copyright (C) 2009 Christian Ruppert + Copyright (C) 2012 Brian Dolbec + +Distributed under the terms of the GNU General Public License v2 + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. + +""" + +import sys +import os + +VERIFY_SSL = False +VERIFY_MSGS = [] + +import requests +from requests.exceptions import SSLError + +# py3.2 +if sys.hexversion >= 0x30200f0: + VERIFY_SSL = True +else: + try: # import and enable SNI support for py2 + from requests.packages.urllib3.contrib import pyopenssl + pyopenssl.inject_into_urllib3() + VERIFY_SSL = True + VERIFY_MSGS = ["Successfully enabled ssl certificate verification."] + except ImportError as e: + VERIFY_MSGS = [ + "Failed to import and inject pyopenssl/SNI support into urllib3", + "Disabling certificate verification", + "Error was:" + e + ] + VERIFY_SSL = False + + +from mirrorselect.version import version + + +class Connector(object): + """Primary connection interface using the dev-python/requests package + """ + + def __init__(self, output, proxies): + self.output = output + self.proxies = proxies + self.headers = {'Accept-Charset': 'utf-8', + 'User-Agent': 'Mirrorselect-' + version} + + if VERIFY_MSGS: + for msg in VERIFY_MSGS: + self.output.write(msg + '\n', 2) + + + def add_timestamp(self, headers, tpath=None, timestamp=None): + """for possilble future caching of the list""" + if tpath and os.path.exists(tpath): + # fileopen is a layman comaptibility function not yet implemented here + with fileopen(tpath,'r') as previous: + timestamp = previous.read() + if timestamp: + headers['If-Modified-Since'] = timestamp + self.output.write('Current-modified: %s\n' % timestamp, 2) + return headers + + + def fetch_url(self, url, headers=None, timestamp=None): + """Fetches the url + + @param url: string + @param headers: dictionary, optional headers to use + @param tpath: string, optional filepath to a timestamp file + to use in the headers + @param timestamp: string, optional timestamp to use in the headers + + """ + + if not headers: + headers = self.headers + + if timestamp: + self.add_timestamp(headers, timestamp=timestamp) + + verify = 'https' in url and VERIFY_SSL + self.output.write("Enabled ssl certificate verification: %s, for: %s\n" + %(str(verify), url), 3) + + self.output.write('Connector.fetch_url(); headers = %s\n' %str(headers), 4) + self.output.write('Connector.fetch_url(); connecting to opener\n', 2) + + try: + connection = requests.get( + url, + headers=headers, + verify=verify, + proxies=self.proxies, + ) + except SSLError as error: + self.output.print_err('Connector.fetch_url(); Failed to update the ' + 'mirror list from: %s\nSSLError was:%s\n' + % (url, str(error))) + except Exception as error: + self.output.print_err('Connector.fetch_url(); Failed to retrieve ' + 'the content from: %s\nError was: %s\n' + % (url, str(error))) + + self.output.write('Connector.fetch_url() HEADERS = %s\n' %str(connection.headers), 4) + self.output.write('Connector.fetch_url() Status_code = %i\n' % connection.status_code, 2) + return connection + + + @staticmethod + def normalize_headers(headers, to_lower=True): + """ py2, py3 compatibility function, since only py2 returns keys as lower() + """ + if to_lower: + return dict((x.lower(), x) for x in list(headers)) + return dict((x.upper(), x) for x in list(headers)) + + + def fetch_content(self, url, tpath=None): + """Fetch the mirror list + + @param url: string of the content to fetch + @param headers: dictionary, optional headers to use + @param tpath: string, optional filepath to a timestamp file + to use in the headers + @returns (success bool, content fetched , timestamp of fetched content, + content headers returned) + """ + + fheaders = self.headers + + if tpath: + fheaders = self.add_timestamp(fheaders, tpath) + + connection = self.fetch_url(url, fheaders) + + headers = self.normalize_headers(connection.headers) + + if 'last-modified' in headers: + timestamp = headers['last-modified'] + elif 'date' in headers: + timestamp = headers['date'] + else: + timestamp = None + + if connection.status_code in [304]: + self.output.write('Content already up to date: %s\n' + % url, 4) + self.output.write('Last-modified: %s\n' % timestamp, 4) + elif connection.status_code not in [200]: + self.output.print_err('Connector.fetch_content(); HTTP Status-Code was:\n' + 'url: %s\n%s' + % (url, str(connection.status_code))) + + if connection.status_code in [200]: + self.output.write('New content downloaded for: %s\n' + % url, 4) + return (True, connection.content, timestamp) + return (False, '', '') + diff --git a/mirrorselect/extractor.py b/mirrorselect/extractor.py index 3a113fb..c8d5bd5 100644 --- a/mirrorselect/extractor.py +++ b/mirrorselect/extractor.py @@ -27,20 +27,10 @@ Distributed under the terms of the GNU General Public License v2 """ -import sys - -if sys.version_info[0] >= 3: - import urllib.request, urllib.parse, urllib.error - url_parse = urllib.parse - url_open = urllib.request.urlopen -else: - import urllib - import urlparse - url_parse = urlparse.urlparse - url_open = urllib.urlopen - +import os from mirrorselect.mirrorparser3 import MirrorParser3 +from mirrorselect.connections import Connector class Extractor(object): @@ -50,6 +40,7 @@ class Extractor(object): def __init__(self, list_url, options, output): self.output = output + self.output.print_info('Using url: %s\n' % list_url) filters = {} for opt in ["country", "region"]: value = getattr(options, opt) @@ -61,6 +52,15 @@ class Extractor(object): if getattr(options, opt): filters["proto"] = opt self.output.print_info('Limiting test to %s hosts. \n' % opt ) + + self.proxies = {} + + for proxy in ['http_proxy', 'https_proxy']: + if options.proxy: + self.proxies[proxy.split('_')[0]] = options.proxy + elif os.getenv(proxy): + self.proxies[proxy.split('_')[0]] = os.getenv(proxy) + parser = MirrorParser3() self.hosts = [] @@ -99,14 +99,13 @@ class Extractor(object): self.output.write('getlist(): fetching ' + url + '\n', 2) - self.output.print_info('Downloading a list of mirrors...') + self.output.print_info('Downloading a list of mirrors...\n') - try: - parser.parse(url_open(url).read()) - except EnvironmentError: - pass + fetcher = Connector(self.output, self.proxies) + success, mirrorlist, timestamp = fetcher.fetch_content(url) + parser.parse(mirrorlist) - if len(parser.tuples()) == 0: + if (not mirrorlist) or len(parser.tuples()) == 0: self.output.print_err('Could not get mirror list. ' 'Check your internet connection.') @@ -114,3 +113,4 @@ class Extractor(object): return parser.tuples() +