[gentoo-commits] proj/layman:master commit in: layman/, etc/

["Brian Dolbec" <brian.dolbec@gmail.com>]

commit:     3f82940636620e5a7970eb563a8bccc0656a8f2f
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 10 17:17:07 2013 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Aug 10 17:17:07 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=3f829406

Add https connection verification.

---
 etc/layman.cfg     |   5 +-
 layman/config.py   |   3 +-
 layman/remotedb.py | 157 +++++++++++++++++++++++++++++++++++++----------------
 3 files changed, 115 insertions(+), 50 deletions(-)

diff --git a/etc/layman.cfg b/etc/layman.cfg
index 11ae1be..f1d730a 100644
--- a/etc/layman.cfg
+++ b/etc/layman.cfg
@@ -84,9 +84,10 @@ overlays  :
 
 #-----------------------------------------------------------
 # Proxy support
-# If unset, layman will use the http_proxy environment variable.
+# If unset, layman will use the http_proxy/https_proxy environment variable.
 #
-#proxy  : http://[user:pass@]www.my-proxy.org:3128
+#http_proxy  : http://[user:pass@]www.my-proxy.org:3128
+#https_proxy : https://[user:pass@]www.my-proxy.org:3128
 
 #-----------------------------------------------------------
 # Strict checking of overlay definitions

diff --git a/layman/config.py b/layman/config.py
index 77096e7..44a419d 100644
--- a/layman/config.py
+++ b/layman/config.py
@@ -101,7 +101,8 @@ class BareConfig(object):
                     'installed': '%(storage)s/installed.xml',
                     'make_conf' : '%(storage)s/make.conf',
                     'nocheck'   : 'yes',
-                    'proxy'     : '',
+                    'http_proxy'     : '',
+                    'https_proxy'     : '',
                     'umask'     : '0022',
                     'news_reporter': 'portage',
                     'custom_news_pkg': '',

diff --git a/layman/remotedb.py b/layman/remotedb.py
index cbe35b9..d623242 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -28,9 +28,29 @@ __version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
 
 import os, os.path
 import sys
-import urllib2
 import hashlib
 
+import requests
+from requests.exceptions import SSLError
+
+VERIFY_SSL = False
+# py3.2
+if sys.hexversion >= 0x30200f0:
+    VERIFY_SSL = True
+else:
+    try: # import and enable SNI support for py2
+        from requests.packages.urllib3.contrib import pyopenssl
+        pyopenssl.inject_into_urllib3()
+        VERIFY_SSL = True
+        VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
+    except ImportError as e:
+        VERIFY_MSGS = [
+            "Failed to import and inject pyopenssl/SNI support into urllib3",
+            "Disabling certificate verification",
+            "Error was:" + e
+        ]
+        VERIFY_SSL = False
+
 
 GPG_ENABLED = False
 try:
@@ -59,19 +79,19 @@ class RemoteDB(DbBase):
 
         self.proxies = {}
 
-        if config['proxy']:
-            self.proxies['http'] = config['proxy']
-        elif os.getenv('http_proxy'):
-            self.proxies['http'] = os.getenv('http_proxy')
-
-        if self.proxies:
-            proxy_handler = urllib2.ProxyHandler(self.proxies)
-            opener = urllib2.build_opener(proxy_handler)
-            urllib2.install_opener(opener)
+        for proxy in ['http_proxy', 'https_proxy']:
+            if config[proxy]:
+                self.proxies[proxy.split('_')[0]] = config[proxy]
+            elif os.getenv(proxy):
+                self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
 
         self.urls  = [i.strip()
             for i in config['overlays'].split('\n') if len(i)]
 
+        if VERIFY_MSGS:
+            for msg in VERIFY_MSGS:
+                self.output.debug(msg, 2)
+
         if GPG_ENABLED:
             self.get_gpg_urls()
         else:
@@ -158,8 +178,10 @@ class RemoteDB(DbBase):
                 self.output.debug("RemoteDB.cache() url = %s is a tuple=%s"
                     %(str(url), str(isinstance(url, tuple))),2)
                 filepath, mpath, tpath, sig = self._paths(url)
-
-                if sig:
+                if 'file://' in url:
+                    success, olist, timestamp = self._fetch_file(
+                        url, mpath, tpath)
+                elif sig:
                     success, olist, timestamp = self._fetch_url(
                         url[0], mpath, tpath)
                 else:
@@ -234,64 +256,105 @@ class RemoteDB(DbBase):
         return base + '_' + hashlib.md5(url_encoded).hexdigest()
 
 
-    def _fetch_url(self, url, mpath, tpath=None):
-        self.output.debug('RemoteDB._fetch_url() url = %s' % url, 2)
+    def _fetch_file(self, url, mpath, tpath=None):
+        self.output.debug('RemoteDB._fetch_file() url = %s' % url, 2)
         # check when the cache was last updated
         # and don't re-fetch it unless it has changed
-        request = urllib2.Request(url)
-        opener = urllib2.build_opener()
-        opener.addheaders = [('Accept-Charset', 'utf-8'),
-            ('User-Agent', 'Layman-' + VERSION)]
-        #opener.addheaders[('User-Agent', 'Layman-' + VERSION)]
+
+        filepath = url.replace('file://','')
+        url_timestamp = None
+        timestamp = ''
 
         if tpath and os.path.exists(tpath):
             with fileopen(tpath,'r') as previous:
                 timestamp = previous.read()
-            request.add_header('If-Modified-Since', timestamp)
 
         if not self.check_path([mpath]):
             return (False, '', '')
 
         try:
-            self.output.debug('RemoteDB._fetch_url() connecting to opener', 2)
-            connection = opener.open(request)
-            # py2, py3 compatibility, since only py2 returns keys as lower()
-            headers = dict((x.lower(), x) for x in connection.headers.keys())
-            if 'last-modified' in headers:
-                timestamp = connection.headers[headers['last-modified']]
-            elif 'date' in headers:
-                timestamp = connection.headers[headers['date']]
+            url_timestamp = os.stat(filepath).st_mtime
+            if url_timestamp != timestamp:
+                self.output.debug('RemoteDB._fetch_file() opening file', 2)
+                # Fetch the remote list
+                with open(filepath) as connection:
+                    olist = connection.read()
             else:
-                timestamp = None
-        except urllib2.HTTPError, e:
-            if e.code == 304:
                 self.output.info('Remote list already up to date: %s'
                     % url, 4)
                 self.output.info('Last-modified: %s' % timestamp, 4)
-            else:
-                self.output.error('RemoteDB.cache(); HTTPError was:\n'
-                    'url: %s\n%s'
-                    % (url, str(e)))
-                return (False, '', '')
-            return (False, '', '')
         except IOError, error:
-            self.output.error('RemoteDB.cache(); Failed to update the '
+            self.output.error('RemoteDB._fetch_file(); Failed to update the '
                 'overlay list from: %s\nIOError was:%s\n'
                 % (url, str(error)))
             return (False, '', '')
         else:
-            if url.startswith('file://'):
-                quieter = 1
-            else:
-                quieter = 0
+            quieter = 1
             self.output.info('Fetching new list... %s' % url, 4 + quieter)
-            if timestamp is not None:
-                self.output.info('Last-modified: %s' % timestamp, 4 + quieter)
-            # Fetch the remote list
-            olist = connection.read()
+            if url_timestamp is not None:
+                self.output.info('Last-modified: %s' % url_timestamp, 4 + quieter)
             self.output.debug('RemoteDB._fetch_url(), olist type = %s' %str(type(olist)),2)
 
-            return (True, olist, timestamp)
+            return (True, olist, url_timestamp)
+
+
+    def _fetch_url(self, url, mpath, tpath=None):
+        headers = {'Accept-Charset': 'utf-8',
+            'User-Agent': 'Layman-' + VERSION}
+
+        if tpath and os.path.exists(tpath):
+            with fileopen(tpath,'r') as previous:
+                timestamp = previous.read()
+            headers['If-Modified-Since'] = timestamp
+            self.output.info('Current-modified: %s' % timestamp, 4)
+
+        verify = 'https' in url and VERIFY_SSL
+        self.output.debug("Enabled ssl certificate verification: %s, for: %s"
+            %(str(verify), url), 3)
+
+        if not self.check_path([mpath]):
+            return (False, '', '')
+        self.output.debug('RemoteDB._fetch_url(); headers = %s' %str(headers))
+        self.output.debug('RemoteDB._fetch_url(); connecting to opener', 2)
+        try:
+            connection = requests.get(
+                url,
+                headers=headers,
+                verify=verify,
+                proxies=self.proxies,
+                )
+        except SSLError, error:
+            self.output.error('RemoteDB._fetch_url(); Failed to update the '
+                'overlay list from: %s\nSSLError was:%s\n'
+                % (url, str(error)))
+        except Exception as error:
+            self.output.error('RemoteDB._fetch_url(); Failed to update the '
+                'overlay list from: %s\nError was:%s\n'
+                % (url, str(error)))
+            # py2, py3 compatibility, since only py2 returns keys as lower()
+        headers = dict((x.lower(), x) for x in list(connection.headers))
+        self.output.info('HEADERS = %s' %str(connection.headers), 4)
+        self.output.debug('Status_code = %i' % connection.status_code)
+        if connection.status_code in [304]:
+            self.output.info('Remote list already up to date: %s'
+                % url, 4)
+            self.output.info('Last-modified: %s' % timestamp, 4)
+        elif connection.status_code not in [200]:
+            self.output.error('RemoteDB._fetch_url(); HTTP Status-Code was:\n'
+                'url: %s\n%s'
+                % (url, str(connection.status_code)))
+
+        if connection.status_code in [200]:
+            self.output.info('Remote new list downloaded for: %s'
+                % url, 4)
+            if 'last-modified' in headers:
+                timestamp = connection.headers[headers['last-modified']]
+            elif 'date' in headers:
+                timestamp = connection.headers[headers['date']]
+            else:
+                timestamp = None
+            return (True, connection.content, timestamp)
+        return (False, '', '')
 
 
     def check_path(self, paths, hint=True):