* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-14 4:01 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-14 4:01 UTC (permalink / raw
To: gentoo-commits
commit: 3796605e387a22caf80af7c81b5bc74152744fa4
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 03:56:31 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Wed May 14 03:56:31 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=3796605e
Adds unicode string compatibility support.
This commit allows string compatibility between py2 and py3 as well as
ensuring that the necessary strings that need to be unicode, stay
unicode in both py2 and py3 when running layman.
---
layman/api.py | 20 ++++++++-------
layman/db.py | 10 ++++----
layman/dbbase.py | 8 +++---
layman/makeconf.py | 22 ++++++++--------
layman/overlays/overlay.py | 62 ++++++++++++++++++++++++----------------------
layman/remotedb.py | 2 +-
layman/utils.py | 4 ++-
7 files changed, 67 insertions(+), 61 deletions(-)
diff --git a/layman/api.py b/layman/api.py
index 475691b..cbf3f76 100755
--- a/layman/api.py
+++ b/layman/api.py
@@ -13,6 +13,8 @@
# Brian Dolbec <dol-sen@sourceforge.net>
#
+from __future__ import print_function
+
import os
from layman.config import BareConfig
@@ -428,21 +430,21 @@ class LaymanAPI(object):
>>> b.close()
>>> api.get_available()
- [u'wrobel', u'wrobel-stable']
- >>> all = api.get_all_info(u'wrobel')
+ ['wrobel', 'wrobel-stable']
+ >>> all = api.get_all_info('wrobel')
>>> info = all['wrobel']
>>> info['status']
- u'official'
+ 'official'
>>> info['description']
- u'Test'
+ 'Test'
>>> info['sources']
- [(u'https://overlays.gentoo.org/svn/dev/wrobel', 'Subversion', None)]
+ [('https://overlays.gentoo.org/svn/dev/wrobel', 'Subversion', None)]
- #{u'wrobel': {'status': u'official',
- #'owner_name': None, 'description': u'Test',
+ #{'wrobel': {'status': 'official',
+ #'owner_name': None, 'description': 'Test',
#'src_uris': <generator object source_uris at 0x167c3c0>,
- #'owner_email': u'nobody@gentoo.org',
- #'quality': u'experimental', 'name': u'wrobel', 'supported': True,
+ #'owner_email': 'nobody@gentoo.org',
+ #'quality': 'experimental', 'name': 'wrobel', 'supported': True,
#'src_types': <generator object source_types at 0x167c370>,
#'official': True,
#'priority': 10, 'feeds': [], 'irc': None, 'homepage': None}}
diff --git a/layman/db.py b/layman/db.py
index 6530147..ce03e13 100644
--- a/layman/db.py
+++ b/layman/db.py
@@ -111,11 +111,11 @@ class DB(DbBase):
# * Running command "/usr/bin/rsync -rlptDvz --progress --delete --delete-after --timeout=180 --exclude="distfiles/*" --exclude="local/*" --exclude="packages/*" "rsync://gunnarwrobel.de/wrobel-stable/*" "/tmp/file.../wrobel-stable""...
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> m = MakeConf(config, b.overlays)
# >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> os.unlink(write)
>>> os.unlink(write2)
@@ -195,16 +195,16 @@ class DB(DbBase):
# * Running command "/usr/bin/svn co "https://overlays.gentoo.org/svn/dev/wrobel/" "/tmp/file.../wrobel""...
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel', u'wrobel-stable']
+ # ['wrobel', 'wrobel-stable']
# >>> b.delete(b.select('wrobel'))
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> m = MakeConf(config, b.overlays)
# >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> os.unlink(write)
>>> os.unlink(write2)
diff --git a/layman/dbbase.py b/layman/dbbase.py
index 475ce44..8fa3ba8 100644
--- a/layman/dbbase.py
+++ b/layman/dbbase.py
@@ -149,10 +149,10 @@ class DbBase(object):
>>> config = {'output': output, 'svn_command': '/usr/bin/svn', 'rsync_command':'/usr/bin/rsync'}
>>> a = DbBase(config, [here + '/tests/testfiles/global-overlays.xml', ])
>>> a.overlays.keys()
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> list(a.overlays['wrobel-stable'].source_uris())
- [u'rsync://gunnarwrobel.de/wrobel-stable']
+ ['rsync://gunnarwrobel.de/wrobel-stable']
'''
try:
document = ET.fromstring(text)
@@ -214,7 +214,7 @@ class DbBase(object):
>>> b.write(write)
>>> c = DbBase({"output": Message() }, [write,])
>>> c.overlays.keys()
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> os.unlink(write)
>>> os.rmdir(tmpdir)
@@ -245,7 +245,7 @@ class DbBase(object):
>>> config = {'output': output, 'svn_command': '/usr/bin/svn', 'rsync_command':'/usr/bin/rsync'}
>>> a = DbBase(config, [here + '/tests/testfiles/global-overlays.xml', ])
>>> list(a.select('wrobel-stable').source_uris())
- [u'rsync://gunnarwrobel.de/wrobel-stable']
+ ['rsync://gunnarwrobel.de/wrobel-stable']
'''
self.output.debug("DbBase.select(), overlay = %s" % overlay, 5)
if not overlay in self.overlays.keys():
diff --git a/layman/makeconf.py b/layman/makeconf.py
index 1abe318..52762a2 100644
--- a/layman/makeconf.py
+++ b/layman/makeconf.py
@@ -50,16 +50,16 @@ class MakeConf:
>>> a.path = write
>>> a.add(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel-stable', u'wrobel-stable']
+ ['wrobel-stable', 'wrobel-stable']
>>> a.add(b.overlays['wrobel'])
>>> [i.name for i in a.overlays]
- [u'wrobel', u'wrobel-stable', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable', 'wrobel-stable']
>>> a.delete(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel']
+ ['wrobel']
>>> a.add(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> a.delete(b.overlays['wrobel'])
>>> n_md5 = str(hashlib.md5(open(write).read()).hexdigest())
>>> o_md5 == n_md5
@@ -105,9 +105,9 @@ class MakeConf:
>>> config['make_conf'] = write
>>> b = MakeConf(config, c.overlays)
>>> [i.name for i in b.overlays]
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
@@ -139,7 +139,7 @@ class MakeConf:
>>> [i.name for i in b.overlays]
[]
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
@@ -164,9 +164,9 @@ class MakeConf:
>>> c = DB(config)
>>> a = MakeConf(config, c.overlays)
>>> [i.name for i in a.overlays]
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> a.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
'''
if os.path.isfile(self.path):
self.content()
@@ -230,9 +230,9 @@ class MakeConf:
>>> config['make_conf'] = write
>>> b = MakeConf(config, c.overlays)
>>> [i.name for i in b.overlays]
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
diff --git a/layman/overlays/overlay.py b/layman/overlays/overlay.py
index 0c2d042..174518b 100755
--- a/layman/overlays/overlay.py
+++ b/layman/overlays/overlay.py
@@ -20,6 +20,8 @@
#
''' Basic overlay class.'''
+from __future__ import unicode_literals
+
__version__ = "0.2"
#===============================================================================
@@ -85,15 +87,15 @@ class Overlay(object):
>>> output = Message()
>>> a = Overlay({'output': output}, overlays[0])
>>> a.name
- u'wrobel'
+ 'wrobel'
>>> a.is_official()
True
>>> list(a.source_uris())
- [u'https://overlays.gentoo.org/svn/dev/wrobel']
+ ['https://overlays.gentoo.org/svn/dev/wrobel']
>>> a.owner_email
- u'nobody@gentoo.org'
+ 'nobody@gentoo.org'
>>> a.description
- u'Test'
+ 'Test'
>>> a.priority
10
>>> b = Overlay({'output': output}, overlays[1])
@@ -208,7 +210,7 @@ class Overlay(object):
else:
self.status = None
- self.quality = u'experimental'
+ self.quality = 'experimental'
if 'quality' in xml.attrib:
if xml.attrib['quality'] in set(QUALITY_LEVELS):
self.quality = encode(xml.attrib['quality'])
@@ -304,7 +306,7 @@ class Overlay(object):
else:
self.status = None
- self.quality = u'experimental'
+ self.quality = 'experimental'
if len(overlay['quality']):
if overlay['quality'] in set(QUALITY_LEVELS):
self.quality = encode(overlay['quality'])
@@ -453,55 +455,55 @@ class Overlay(object):
<BLANKLINE>
'''
- result = u''
+ result = ''
- result += self.name + u'\n' + (len(self.name) * u'~')
+ result += self.name + '\n' + (len(self.name) * '~')
if len(self.sources) == 1:
- result += u'\nSource : ' + self.sources[0].src
+ result += '\nSource : ' + self.sources[0].src
else:
- result += u'\nSources:'
+ result += '\nSources:'
for i, v in enumerate(self.sources):
result += '\n %d. %s' % (i + 1, v.src)
result += '\n'
if self.owner_name != None:
- result += u'\nContact : %s <%s>' \
+ result += '\nContact : %s <%s>' \
% (self.owner_name, self.owner_email)
else:
- result += u'\nContact : ' + self.owner_email
+ result += '\nContact : ' + self.owner_email
if len(self.sources) == 1:
- result += u'\nType : ' + self.sources[0].type
+ result += '\nType : ' + self.sources[0].type
else:
- result += u'\nType : ' + '/'.join(
+ result += '\nType : ' + '/'.join(
sorted(set(e.type for e in self.sources)))
- result += u'; Priority: ' + str(self.priority) + u'\n'
- result += u'Quality : ' + self.quality + u'\n'
+ result += '; Priority: ' + str(self.priority) + '\n'
+ result += 'Quality : ' + self.quality + '\n'
description = self.description
- description = re.compile(u' +').sub(u' ', description)
- description = re.compile(u'\n ').sub(u'\n', description)
- result += u'\nDescription:'
- result += u'\n '.join((u'\n' + description).split(u'\n'))
- result += u'\n'
+ description = re.compile(' +').sub(' ', description)
+ description = re.compile('\n ').sub('\n', description)
+ result += '\nDescription:'
+ result += '\n '.join(('\n' + description).split('\n'))
+ result += '\n'
if self.homepage != None:
link = self.homepage
- link = re.compile(u' +').sub(u' ', link)
- link = re.compile(u'\n ').sub(u'\n', link)
- result += u'\nLink:'
- result += u'\n '.join((u'\n' + link).split(u'\n'))
- result += u'\n'
+ link = re.compile(' +').sub(' ', link)
+ link = re.compile('\n ').sub('\n', link)
+ result += '\nLink:'
+ result += '\n '.join(('\n' + link).split('\n'))
+ result += '\n'
if self.irc != None:
- result += u'\nIRC : ' + self.irc + u'\n'
+ result += '\nIRC : ' + self.irc + '\n'
if len(self.feeds):
- result += u'\n%s:' % ((len(self.feeds) == 1) and "Feed" or "Feeds")
+ result += '\n%s:' % ((len(self.feeds) == 1) and "Feed" or "Feeds")
for i in self.feeds:
- result += u'\n %s' % i
- result += u'\n'
+ result += '\n %s' % i
+ result += '\n'
return encoder(result, self._encoding_)
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 38d9fa8..b010e51 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -160,7 +160,7 @@ class RemoteDB(DbBase):
>>> os.unlink(a.filepath(config['overlays'])+'.xml')
>>> a.overlays.keys()
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> import shutil
>>> shutil.rmtree(tmpdir)
diff --git a/layman/utils.py b/layman/utils.py
index f174215..efb3231 100644
--- a/layman/utils.py
+++ b/layman/utils.py
@@ -21,6 +21,8 @@
'''Utility functions to deal with xml nodes. '''
+from __future__ import unicode_literals
+
__version__ = '$Id: utils.py 236 2006-09-05 20:39:37Z wrobel $'
#===============================================================================
@@ -197,7 +199,7 @@ def create_overlay_dict(**kwargs):
'feeds': [],
'sources': [('','','')],
'priority': 50,
- 'quality': u'experimental',
+ 'quality': 'experimental',
'status': '',
'official': False,
'supported': False,
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-14 17:32 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-14 17:32 UTC (permalink / raw
To: gentoo-commits
commit: 134801fe7e132ca667d8a74161f850a01cceeaa8
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 00:35:07 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Wed May 14 17:27:29 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=134801fe
layman/{*.py, /overlays/*.py}: "Exception, error" to "Exception as error"
This changes the all instances of the phrase "Exception, error" to
"Exception as error" to allow py2/py3 compatibility.
---
layman/api.py | 16 ++++++++--------
layman/cli.py | 2 +-
layman/dbbase.py | 4 ++--
layman/makeconf.py | 4 ++--
layman/overlays/overlay.py | 2 +-
layman/overlays/source.py | 2 +-
layman/overlays/tar.py | 10 +++++-----
layman/remotedb.py | 10 +++++-----
layman/updater.py | 2 +-
layman/utils.py | 2 +-
10 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/layman/api.py b/layman/api.py
index 11534fd..475691b 100755
--- a/layman/api.py
+++ b/layman/api.py
@@ -113,7 +113,7 @@ class LaymanAPI(object):
try:
self._get_installed_db().delete(
self._get_installed_db().select(ovl))
- except Exception, e:
+ except Exception as e:
self._error(
"Exception caught disabling repository '"+ovl+
"':\n"+str(e))
@@ -147,7 +147,7 @@ class LaymanAPI(object):
try:
success = self._get_installed_db().add(
self._get_remote_db().select(ovl))
- except Exception, e:
+ except Exception as e:
self._error("Exception caught enabling repository '"+ovl+
"' : "+str(e))
results.append(success)
@@ -201,7 +201,7 @@ class LaymanAPI(object):
continue
try:
overlay = db.select(ovl)
- except UnknownOverlayException, error:
+ except UnknownOverlayException as error:
self._error(error)
result[ovl] = ('', False, False)
else:
@@ -255,7 +255,7 @@ class LaymanAPI(object):
overlay = db.select(ovl)
#print "overlay = ", ovl
#print "!!!", overlay
- except UnknownOverlayException, error:
+ except UnknownOverlayException as error:
#print "ERRORS", str(error)
self._error(error)
result[ovl] = ('', False, False)
@@ -311,7 +311,7 @@ class LaymanAPI(object):
#self.output.debug("API.sync(); selecting %s, db = %s" % (ovl, str(db)), 5)
odb = db.select(ovl)
self.output.debug("API.sync(); %s now selected" %ovl, 5)
- except UnknownOverlayException, error:
+ except UnknownOverlayException as error:
#self.output.debug("API.sync(); UnknownOverlayException selecting %s" %ovl, 5)
#self._error(str(error))
fatals.append((ovl,
@@ -364,7 +364,7 @@ class LaymanAPI(object):
self.output.debug("API.sync(); starting db.sync(ovl)", 5)
db.sync(ovl)
success.append((ovl,'Successfully synchronized overlay "' + ovl + '".'))
- except Exception, error:
+ except Exception as error:
fatals.append((ovl,
'Failed to sync overlay "' + ovl + '".\nError was: '
+ str(error)))
@@ -457,7 +457,7 @@ class LaymanAPI(object):
self.output.debug(
'LaymanAPI.fetch_remote_list(); cache updated = %s'
% str(dbreload),8)
- except Exception, error:
+ except Exception as error:
self.output.error('Failed to fetch overlay list!\n Original Error was: '
+ str(error))
return False
@@ -585,7 +585,7 @@ class LaymanAPI(object):
elif self.config['news_reporter'] == 'pkgcore':
# pkgcore is not yet capable
return
- except Exception, err:
+ except Exception as err:
msg = "update_news() failed running %s news reporter function\n" +\
"Error was; %s"
self._error(msg % (self.config['news_reporter'], err))
diff --git a/layman/cli.py b/layman/cli.py
index a5d2799..258e3b9 100644
--- a/layman/cli.py
+++ b/layman/cli.py
@@ -173,7 +173,7 @@ class Main(object):
try:
new_umask = int(umask, 8)
old_umask = os.umask(new_umask)
- except Exception, error:
+ except Exception as error:
self.output.die('Failed setting to umask "' + umask +
'"!\nError was: ' + str(error))
diff --git a/layman/dbbase.py b/layman/dbbase.py
index e5c0fd9..8d57a9f 100644
--- a/layman/dbbase.py
+++ b/layman/dbbase.py
@@ -124,7 +124,7 @@ class DbBase(object):
df = fileopen(path, 'r')
document = df.read()
- except Exception, error:
+ except Exception as error:
if not self.ignore_init_read_errors:
self.output.error('Failed to read the overlay list at ("'
+ path + '")')
@@ -231,7 +231,7 @@ class DbBase(object):
""")
tree.write(f, encoding='utf-8')
f.close()
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to write to local overlays file: '
+ path + '\nError was:\n' + str(error))
diff --git a/layman/makeconf.py b/layman/makeconf.py
index 9740072..1abe318 100644
--- a/layman/makeconf.py
+++ b/layman/makeconf.py
@@ -273,7 +273,7 @@ class MakeConf:
make_conf.close()
- except Exception, error:
+ except Exception as error:
self.output.error('MakeConf: write(); Failed to write "'
+ self.path + '".\nError was:\n' + str(error))
return False
@@ -290,7 +290,7 @@ class MakeConf:
make_conf.close()
- except Exception, error:
+ except Exception as error:
self.output.error('MakeConf: content(); Failed to read "' +
self.path + '".\nError was:\n' + str(error))
raise error
diff --git a/layman/overlays/overlay.py b/layman/overlays/overlay.py
index 65c148b..0c2d042 100755
--- a/layman/overlays/overlay.py
+++ b/layman/overlays/overlay.py
@@ -414,7 +414,7 @@ class Overlay(object):
# Worked, throw other sources away
self.sources = [s]
break
- except Exception, error:
+ except Exception as error:
self.output.warn(str(error), 4)
first_s = False
return res
diff --git a/layman/overlays/source.py b/layman/overlays/source.py
index 2b79f2c..69c59bb 100644
--- a/layman/overlays/source.py
+++ b/layman/overlays/source.py
@@ -177,7 +177,7 @@ class OverlaySource(object):
self.output.warn("Checking for cleanup actions to perform", 4)
self.cleanup()
result = 1
- except Exception, err:
+ except Exception as err:
self.output.error(
'Unknown exception running command: %s' % command_repr)
self.output.error('Original error was: %s' % str(err))
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index c6f2443..e57096d 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -111,7 +111,7 @@ class TarOverlay(OverlaySource):
try:
tar = urllib2.urlopen(tar_url).read()
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to fetch the tar package from: '
+ self.src + '\nError was:' + str(error))
@@ -121,7 +121,7 @@ class TarOverlay(OverlaySource):
out_file = open(pkg, 'w+b')
out_file.write(tar)
out_file.close()
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to store tar package in '
+ pkg + '\nError was:' + str(error))
@@ -140,7 +140,7 @@ class TarOverlay(OverlaySource):
try:
self.output.info('Deleting directory "%s"' % folder, 2)
shutil.rmtree(folder)
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to remove unnecessary tar structure "'
+ folder + '"\nError was:' + str(error))
@@ -149,7 +149,7 @@ class TarOverlay(OverlaySource):
try:
result = self._extract(base=base, tar_url=self.src,
dest_dir=temp_path)
- except Exception, error:
+ except Exception as error:
try_to_wipe(temp_path)
raise error
@@ -165,7 +165,7 @@ class TarOverlay(OverlaySource):
try:
os.rename(source, final_path)
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to rename tar subdirectory ' +
source + ' to ' + final_path +
'\nError was:' + str(error))
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 98dffbb..38d9fa8 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -245,7 +245,7 @@ class RemoteDB(DbBase):
if not os.path.exists(os.path.dirname(mpath)):
try:
os.makedirs(os.path.dirname(mpath))
- except OSError, error:
+ except OSError as error:
raise OSError('Failed to create layman storage directory ' +
os.path.dirname(mpath) + '\n' +
'Error was:' + str(error))
@@ -290,7 +290,7 @@ class RemoteDB(DbBase):
self.output.info('Remote list already up to date: %s'
% url, 4)
self.output.info('Last-modified: %s' % timestamp, 4)
- except IOError, error:
+ except IOError as error:
self.output.error('RemoteDB._fetch_file(); Failed to update the '
'overlay list from: %s\nIOError was:%s\n'
% (url, str(error)))
@@ -333,7 +333,7 @@ class RemoteDB(DbBase):
verify=verify,
proxies=self.proxies,
)
- except SSLError, error:
+ except SSLError as error:
self.output.error('RemoteDB._fetch_url(); Failed to update the '
'overlay list from: %s\nSSLError was:%s\n'
% (url, str(error)))
@@ -388,7 +388,7 @@ class RemoteDB(DbBase):
try:
self.read(olist, origin=url)
- except Exception, error:
+ except Exception as error:
self.output.debug("RemoteDB._check_download(), url=%s \nolist:\n"
% url,2)
self.output.debug(olist, 2)
@@ -419,7 +419,7 @@ class RemoteDB(DbBase):
has_updates = True
- except Exception, error:
+ except Exception as error:
raise IOError('Failed to temporarily cache overlays list in'
' ' + mpath + '\nError was:\n' + str(error))
return has_updates
diff --git a/layman/updater.py b/layman/updater.py
index f7d91e6..7c46707 100644
--- a/layman/updater.py
+++ b/layman/updater.py
@@ -23,7 +23,7 @@ def rename_db(config, newname, output):
"name is...: %s" %newname, 2)
output.notice('')
return
- except OSError, err:
+ except OSError as err:
output.error(" Automatic db rename failed:\n%s" %str(err))
else:
output.info(" Automatic db rename, failed access to: %s"
diff --git a/layman/utils.py b/layman/utils.py
index ab9904f..f174215 100644
--- a/layman/utils.py
+++ b/layman/utils.py
@@ -174,7 +174,7 @@ def delete_empty_directory(mdir, output=None):
output.info('Deleting _empty_ directory "%s"' % mdir, 2)
try:
os.rmdir(mdir)
- except OSError, error:
+ except OSError as error:
output.warn(str(error))
else:
output.warn('Insufficient permissions to delete _empty_ folder "%s".' % mdir)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-14 17:32 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-14 17:32 UTC (permalink / raw
To: gentoo-commits
commit: 6d1c59d9c9da4d975019f6bc0a243361c8987e76
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 03:56:31 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Wed May 14 17:31:29 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=6d1c59d9
Adds unicode string compatibility support.
This commit allows string compatibility between py2 and py3 as well as
ensuring that the necessary strings that need to be unicode, stay
unicode in both py2 and py3 when running layman.
---
layman/api.py | 20 ++++++++-------
layman/db.py | 10 ++++----
layman/dbbase.py | 8 +++---
layman/makeconf.py | 22 ++++++++--------
layman/overlays/overlay.py | 62 ++++++++++++++++++++++++----------------------
layman/remotedb.py | 2 +-
layman/utils.py | 4 ++-
7 files changed, 67 insertions(+), 61 deletions(-)
diff --git a/layman/api.py b/layman/api.py
index 475691b..cbf3f76 100755
--- a/layman/api.py
+++ b/layman/api.py
@@ -13,6 +13,8 @@
# Brian Dolbec <dol-sen@sourceforge.net>
#
+from __future__ import print_function
+
import os
from layman.config import BareConfig
@@ -428,21 +430,21 @@ class LaymanAPI(object):
>>> b.close()
>>> api.get_available()
- [u'wrobel', u'wrobel-stable']
- >>> all = api.get_all_info(u'wrobel')
+ ['wrobel', 'wrobel-stable']
+ >>> all = api.get_all_info('wrobel')
>>> info = all['wrobel']
>>> info['status']
- u'official'
+ 'official'
>>> info['description']
- u'Test'
+ 'Test'
>>> info['sources']
- [(u'https://overlays.gentoo.org/svn/dev/wrobel', 'Subversion', None)]
+ [('https://overlays.gentoo.org/svn/dev/wrobel', 'Subversion', None)]
- #{u'wrobel': {'status': u'official',
- #'owner_name': None, 'description': u'Test',
+ #{'wrobel': {'status': 'official',
+ #'owner_name': None, 'description': 'Test',
#'src_uris': <generator object source_uris at 0x167c3c0>,
- #'owner_email': u'nobody@gentoo.org',
- #'quality': u'experimental', 'name': u'wrobel', 'supported': True,
+ #'owner_email': 'nobody@gentoo.org',
+ #'quality': 'experimental', 'name': 'wrobel', 'supported': True,
#'src_types': <generator object source_types at 0x167c370>,
#'official': True,
#'priority': 10, 'feeds': [], 'irc': None, 'homepage': None}}
diff --git a/layman/db.py b/layman/db.py
index 6530147..ce03e13 100644
--- a/layman/db.py
+++ b/layman/db.py
@@ -111,11 +111,11 @@ class DB(DbBase):
# * Running command "/usr/bin/rsync -rlptDvz --progress --delete --delete-after --timeout=180 --exclude="distfiles/*" --exclude="local/*" --exclude="packages/*" "rsync://gunnarwrobel.de/wrobel-stable/*" "/tmp/file.../wrobel-stable""...
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> m = MakeConf(config, b.overlays)
# >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> os.unlink(write)
>>> os.unlink(write2)
@@ -195,16 +195,16 @@ class DB(DbBase):
# * Running command "/usr/bin/svn co "https://overlays.gentoo.org/svn/dev/wrobel/" "/tmp/file.../wrobel""...
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel', u'wrobel-stable']
+ # ['wrobel', 'wrobel-stable']
# >>> b.delete(b.select('wrobel'))
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> m = MakeConf(config, b.overlays)
# >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> os.unlink(write)
>>> os.unlink(write2)
diff --git a/layman/dbbase.py b/layman/dbbase.py
index 475ce44..8fa3ba8 100644
--- a/layman/dbbase.py
+++ b/layman/dbbase.py
@@ -149,10 +149,10 @@ class DbBase(object):
>>> config = {'output': output, 'svn_command': '/usr/bin/svn', 'rsync_command':'/usr/bin/rsync'}
>>> a = DbBase(config, [here + '/tests/testfiles/global-overlays.xml', ])
>>> a.overlays.keys()
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> list(a.overlays['wrobel-stable'].source_uris())
- [u'rsync://gunnarwrobel.de/wrobel-stable']
+ ['rsync://gunnarwrobel.de/wrobel-stable']
'''
try:
document = ET.fromstring(text)
@@ -214,7 +214,7 @@ class DbBase(object):
>>> b.write(write)
>>> c = DbBase({"output": Message() }, [write,])
>>> c.overlays.keys()
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> os.unlink(write)
>>> os.rmdir(tmpdir)
@@ -245,7 +245,7 @@ class DbBase(object):
>>> config = {'output': output, 'svn_command': '/usr/bin/svn', 'rsync_command':'/usr/bin/rsync'}
>>> a = DbBase(config, [here + '/tests/testfiles/global-overlays.xml', ])
>>> list(a.select('wrobel-stable').source_uris())
- [u'rsync://gunnarwrobel.de/wrobel-stable']
+ ['rsync://gunnarwrobel.de/wrobel-stable']
'''
self.output.debug("DbBase.select(), overlay = %s" % overlay, 5)
if not overlay in self.overlays.keys():
diff --git a/layman/makeconf.py b/layman/makeconf.py
index 1abe318..52762a2 100644
--- a/layman/makeconf.py
+++ b/layman/makeconf.py
@@ -50,16 +50,16 @@ class MakeConf:
>>> a.path = write
>>> a.add(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel-stable', u'wrobel-stable']
+ ['wrobel-stable', 'wrobel-stable']
>>> a.add(b.overlays['wrobel'])
>>> [i.name for i in a.overlays]
- [u'wrobel', u'wrobel-stable', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable', 'wrobel-stable']
>>> a.delete(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel']
+ ['wrobel']
>>> a.add(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> a.delete(b.overlays['wrobel'])
>>> n_md5 = str(hashlib.md5(open(write).read()).hexdigest())
>>> o_md5 == n_md5
@@ -105,9 +105,9 @@ class MakeConf:
>>> config['make_conf'] = write
>>> b = MakeConf(config, c.overlays)
>>> [i.name for i in b.overlays]
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
@@ -139,7 +139,7 @@ class MakeConf:
>>> [i.name for i in b.overlays]
[]
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
@@ -164,9 +164,9 @@ class MakeConf:
>>> c = DB(config)
>>> a = MakeConf(config, c.overlays)
>>> [i.name for i in a.overlays]
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> a.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
'''
if os.path.isfile(self.path):
self.content()
@@ -230,9 +230,9 @@ class MakeConf:
>>> config['make_conf'] = write
>>> b = MakeConf(config, c.overlays)
>>> [i.name for i in b.overlays]
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
diff --git a/layman/overlays/overlay.py b/layman/overlays/overlay.py
index 0c2d042..174518b 100755
--- a/layman/overlays/overlay.py
+++ b/layman/overlays/overlay.py
@@ -20,6 +20,8 @@
#
''' Basic overlay class.'''
+from __future__ import unicode_literals
+
__version__ = "0.2"
#===============================================================================
@@ -85,15 +87,15 @@ class Overlay(object):
>>> output = Message()
>>> a = Overlay({'output': output}, overlays[0])
>>> a.name
- u'wrobel'
+ 'wrobel'
>>> a.is_official()
True
>>> list(a.source_uris())
- [u'https://overlays.gentoo.org/svn/dev/wrobel']
+ ['https://overlays.gentoo.org/svn/dev/wrobel']
>>> a.owner_email
- u'nobody@gentoo.org'
+ 'nobody@gentoo.org'
>>> a.description
- u'Test'
+ 'Test'
>>> a.priority
10
>>> b = Overlay({'output': output}, overlays[1])
@@ -208,7 +210,7 @@ class Overlay(object):
else:
self.status = None
- self.quality = u'experimental'
+ self.quality = 'experimental'
if 'quality' in xml.attrib:
if xml.attrib['quality'] in set(QUALITY_LEVELS):
self.quality = encode(xml.attrib['quality'])
@@ -304,7 +306,7 @@ class Overlay(object):
else:
self.status = None
- self.quality = u'experimental'
+ self.quality = 'experimental'
if len(overlay['quality']):
if overlay['quality'] in set(QUALITY_LEVELS):
self.quality = encode(overlay['quality'])
@@ -453,55 +455,55 @@ class Overlay(object):
<BLANKLINE>
'''
- result = u''
+ result = ''
- result += self.name + u'\n' + (len(self.name) * u'~')
+ result += self.name + '\n' + (len(self.name) * '~')
if len(self.sources) == 1:
- result += u'\nSource : ' + self.sources[0].src
+ result += '\nSource : ' + self.sources[0].src
else:
- result += u'\nSources:'
+ result += '\nSources:'
for i, v in enumerate(self.sources):
result += '\n %d. %s' % (i + 1, v.src)
result += '\n'
if self.owner_name != None:
- result += u'\nContact : %s <%s>' \
+ result += '\nContact : %s <%s>' \
% (self.owner_name, self.owner_email)
else:
- result += u'\nContact : ' + self.owner_email
+ result += '\nContact : ' + self.owner_email
if len(self.sources) == 1:
- result += u'\nType : ' + self.sources[0].type
+ result += '\nType : ' + self.sources[0].type
else:
- result += u'\nType : ' + '/'.join(
+ result += '\nType : ' + '/'.join(
sorted(set(e.type for e in self.sources)))
- result += u'; Priority: ' + str(self.priority) + u'\n'
- result += u'Quality : ' + self.quality + u'\n'
+ result += '; Priority: ' + str(self.priority) + '\n'
+ result += 'Quality : ' + self.quality + '\n'
description = self.description
- description = re.compile(u' +').sub(u' ', description)
- description = re.compile(u'\n ').sub(u'\n', description)
- result += u'\nDescription:'
- result += u'\n '.join((u'\n' + description).split(u'\n'))
- result += u'\n'
+ description = re.compile(' +').sub(' ', description)
+ description = re.compile('\n ').sub('\n', description)
+ result += '\nDescription:'
+ result += '\n '.join(('\n' + description).split('\n'))
+ result += '\n'
if self.homepage != None:
link = self.homepage
- link = re.compile(u' +').sub(u' ', link)
- link = re.compile(u'\n ').sub(u'\n', link)
- result += u'\nLink:'
- result += u'\n '.join((u'\n' + link).split(u'\n'))
- result += u'\n'
+ link = re.compile(' +').sub(' ', link)
+ link = re.compile('\n ').sub('\n', link)
+ result += '\nLink:'
+ result += '\n '.join(('\n' + link).split('\n'))
+ result += '\n'
if self.irc != None:
- result += u'\nIRC : ' + self.irc + u'\n'
+ result += '\nIRC : ' + self.irc + '\n'
if len(self.feeds):
- result += u'\n%s:' % ((len(self.feeds) == 1) and "Feed" or "Feeds")
+ result += '\n%s:' % ((len(self.feeds) == 1) and "Feed" or "Feeds")
for i in self.feeds:
- result += u'\n %s' % i
- result += u'\n'
+ result += '\n %s' % i
+ result += '\n'
return encoder(result, self._encoding_)
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 38d9fa8..b010e51 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -160,7 +160,7 @@ class RemoteDB(DbBase):
>>> os.unlink(a.filepath(config['overlays'])+'.xml')
>>> a.overlays.keys()
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> import shutil
>>> shutil.rmtree(tmpdir)
diff --git a/layman/utils.py b/layman/utils.py
index f174215..efb3231 100644
--- a/layman/utils.py
+++ b/layman/utils.py
@@ -21,6 +21,8 @@
'''Utility functions to deal with xml nodes. '''
+from __future__ import unicode_literals
+
__version__ = '$Id: utils.py 236 2006-09-05 20:39:37Z wrobel $'
#===============================================================================
@@ -197,7 +199,7 @@ def create_overlay_dict(**kwargs):
'feeds': [],
'sources': [('','','')],
'priority': 50,
- 'quality': u'experimental',
+ 'quality': 'experimental',
'status': '',
'official': False,
'supported': False,
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-14 23:10 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-14 23:10 UTC (permalink / raw
To: gentoo-commits
commit: 6fb6999fce0f2257ea54a4e385100c00e286d11d
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 23:07:30 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Wed May 14 23:07:30 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=6fb6999f
layman/{remotedb, overlays/tar}.py: Implements ssl-fetch code.
To clean up no longer necessary functions and ensure py2/py3
compatibility, functions from ssh-fetch were implemented throughout
both remotedb.py and overlays/tar.py.
---
layman/overlays/tar.py | 30 ++++++++++----
layman/remotedb.py | 108 +++++++------------------------------------------
2 files changed, 37 insertions(+), 101 deletions(-)
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 9cb65a2..94fefb1 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -24,12 +24,16 @@ __version__ = "$Id: tar.py 310 2007-04-09 16:30:40Z wrobel $"
#
#-------------------------------------------------------------------------------
-import os, os.path, sys, urllib2, shutil, tempfile
+import os, os.path, sys, shutil
+
import xml.etree.ElementTree as ET # Python 2.5
from layman.utils import path
-#from layman.debug import OUT
from layman.overlays.source import OverlaySource, require_supported
+from layman.version import VERSION
+from sslfetch.connections import Connector
+
+USERAGENT = "Layman" + VERSION
#===============================================================================
#
@@ -78,6 +82,15 @@ class TarOverlay(OverlaySource):
def __init__(self, parent, config, _location, ignore = 0):
+ self.proxies = {}
+
+ for proxy in ['http_proxy', 'https_proxy']:
+ if config[proxy]:
+ self.proxies[proxy.split('_')[0]] = config[proxy]
+ elif os.getenv(proxy):
+ self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
+
+
super(TarOverlay, self).__init__(parent,
config, _location, ignore)
@@ -109,11 +122,14 @@ class TarOverlay(OverlaySource):
ext = candidate_ext
break
- try:
- tar = urllib2.urlopen(tar_url).read()
- except Exception as error:
- raise Exception('Failed to fetch the tar package from: '
- + self.src + '\nError was:' + str(error))
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
+
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
+
+ success, tar, timestamp = fetcher.fetch_content(tar_url)
pkg = path([base, self.parent.name + ext])
diff --git a/layman/remotedb.py b/layman/remotedb.py
index b010e51..24ee8b8 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -17,6 +17,7 @@
'''Handles different storage files.'''
from __future__ import with_statement
+from __future__ import unicode_literals
__version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
@@ -30,28 +31,6 @@ import os, os.path
import sys
import hashlib
-import requests
-from requests.exceptions import SSLError
-
-VERIFY_SSL = False
-# py3.2
-if sys.hexversion >= 0x30200f0:
- VERIFY_SSL = True
-else:
- try: # import and enable SNI support for py2
- from requests.packages.urllib3.contrib import pyopenssl
- pyopenssl.inject_into_urllib3()
- VERIFY_SSL = True
- VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
- except ImportError as e:
- VERIFY_MSGS = [
- "Failed to import and inject pyopenssl/SNI support into urllib3",
- "Disabling certificate verification",
- "Error was:" + e
- ]
- VERIFY_SSL = False
-
-
GPG_ENABLED = False
try:
from pygpg.config import GPGConfig
@@ -64,8 +43,10 @@ except ImportError:
from layman.utils import encoder
from layman.dbbase import DbBase
from layman.version import VERSION
-from layman.compatibility import fileopen
+from layman.compatibility import fileopen
+from sslfetch.connections import Connector
+USERAGENT = "Layman-" + VERSION
class RemoteDB(DbBase):
'''Handles fetching the remote overlay list.'''
@@ -88,10 +69,6 @@ class RemoteDB(DbBase):
self.urls = [i.strip()
for i in config['overlays'].split('\n') if len(i)]
- if VERIFY_MSGS:
- for msg in VERIFY_MSGS:
- self.output.debug(msg, 2)
-
if GPG_ENABLED:
self.get_gpg_urls()
else:
@@ -113,7 +90,6 @@ class RemoteDB(DbBase):
self.output.debug('RemoteDB.__init__(), paths to load = %s' %str(paths),
2)
-
if config['nocheck']:
ignore = 2
else:
@@ -171,6 +147,12 @@ class RemoteDB(DbBase):
succeeded = True
url_lists = [self.urls, self.detached_urls, self.signed_urls]
need_gpg = [False, True, True]
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
+
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
urls = url_lists[index]
@@ -187,11 +169,11 @@ class RemoteDB(DbBase):
success, olist, timestamp = self._fetch_file(
url, mpath, tpath)
elif sig:
- success, olist, timestamp = self._fetch_url(
- url[0], mpath, tpath)
+ success, olist, timestamp = fetcher.fetch_content(
+ url[0], tpath)
else:
- success, olist, timestamp = self._fetch_url(
- url, mpath, tpath)
+ success, olist, timestamp = fetcher.fetch_content(
+ url, tpath)
if not success:
#succeeded = False
continue
@@ -262,7 +244,6 @@ class RemoteDB(DbBase):
return base + '_' + hashlib.md5(url_encoded).hexdigest()
-
def _fetch_file(self, url, mpath, tpath=None):
self.output.debug('RemoteDB._fetch_file() url = %s' % url, 2)
# check when the cache was last updated
@@ -306,67 +287,6 @@ class RemoteDB(DbBase):
return (True, olist, url_timestamp)
-
- def _fetch_url(self, url, mpath, tpath=None):
- headers = {'Accept-Charset': 'utf-8',
- 'User-Agent': 'Layman-' + VERSION}
-
- if tpath and os.path.exists(tpath):
- with fileopen(tpath,'r') as previous:
- timestamp = previous.read()
- headers['If-Modified-Since'] = timestamp
- self.output.info('Current-modified: %s' % timestamp, 4)
-
- verify = 'https' in url and VERIFY_SSL
- self.output.debug("Enabled ssl certificate verification: %s, for: %s"
- %(str(verify), url), 3)
-
- if not self.check_path([mpath]):
- return (False, '', '')
- self.output.debug('RemoteDB._fetch_url(); headers = %s'
- % str(headers), 2)
- self.output.debug('RemoteDB._fetch_url(); connecting to opener', 2)
- try:
- connection = requests.get(
- url,
- headers=headers,
- verify=verify,
- proxies=self.proxies,
- )
- except SSLError as error:
- self.output.error('RemoteDB._fetch_url(); Failed to update the '
- 'overlay list from: %s\nSSLError was:%s\n'
- % (url, str(error)))
- except Exception as error:
- self.output.error('RemoteDB._fetch_url(); Failed to update the '
- 'overlay list from: %s\nError was:%s\n'
- % (url, str(error)))
- # py2, py3 compatibility, since only py2 returns keys as lower()
- headers = dict((x.lower(), x) for x in list(connection.headers))
- self.output.info('HEADERS = %s' %str(connection.headers), 4)
- self.output.debug('Status_code = %i' % connection.status_code, 2)
- if connection.status_code in [304]:
- self.output.info('Remote list already up to date: %s'
- % url, 4)
- self.output.info('Last-modified: %s' % timestamp, 4)
- elif connection.status_code not in [200]:
- self.output.error('RemoteDB._fetch_url(); HTTP Status-Code was:\n'
- 'url: %s\n%s'
- % (url, str(connection.status_code)))
-
- if connection.status_code in [200]:
- self.output.info('Remote new list downloaded for: %s'
- % url, 4)
- if 'last-modified' in headers:
- timestamp = connection.headers[headers['last-modified']]
- elif 'date' in headers:
- timestamp = connection.headers[headers['date']]
- else:
- timestamp = None
- return (True, connection.content, timestamp)
- return (False, '', '')
-
-
def check_path(self, paths, hint=True):
'''Check for sufficient privileges'''
self.output.debug('RemoteDB.check_path; paths = ' + str(paths), 8)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-14 23:36 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-14 23:36 UTC (permalink / raw
To: gentoo-commits
commit: 807f7afc4398d84692a0e5f738b35b463b435dda
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 23:07:30 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Wed May 14 23:33:25 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=807f7afc
remotedb.py, tar.py: Implements ssl-fetch code.
Migrate to using ssl-fetch (urllib3 based) for downloading files.
---
layman/overlays/tar.py | 30 ++++++++++----
layman/remotedb.py | 108 +++++++------------------------------------------
2 files changed, 37 insertions(+), 101 deletions(-)
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 9cb65a2..94fefb1 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -24,12 +24,16 @@ __version__ = "$Id: tar.py 310 2007-04-09 16:30:40Z wrobel $"
#
#-------------------------------------------------------------------------------
-import os, os.path, sys, urllib2, shutil, tempfile
+import os, os.path, sys, shutil
+
import xml.etree.ElementTree as ET # Python 2.5
from layman.utils import path
-#from layman.debug import OUT
from layman.overlays.source import OverlaySource, require_supported
+from layman.version import VERSION
+from sslfetch.connections import Connector
+
+USERAGENT = "Layman" + VERSION
#===============================================================================
#
@@ -78,6 +82,15 @@ class TarOverlay(OverlaySource):
def __init__(self, parent, config, _location, ignore = 0):
+ self.proxies = {}
+
+ for proxy in ['http_proxy', 'https_proxy']:
+ if config[proxy]:
+ self.proxies[proxy.split('_')[0]] = config[proxy]
+ elif os.getenv(proxy):
+ self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
+
+
super(TarOverlay, self).__init__(parent,
config, _location, ignore)
@@ -109,11 +122,14 @@ class TarOverlay(OverlaySource):
ext = candidate_ext
break
- try:
- tar = urllib2.urlopen(tar_url).read()
- except Exception as error:
- raise Exception('Failed to fetch the tar package from: '
- + self.src + '\nError was:' + str(error))
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
+
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
+
+ success, tar, timestamp = fetcher.fetch_content(tar_url)
pkg = path([base, self.parent.name + ext])
diff --git a/layman/remotedb.py b/layman/remotedb.py
index b010e51..24ee8b8 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -17,6 +17,7 @@
'''Handles different storage files.'''
from __future__ import with_statement
+from __future__ import unicode_literals
__version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
@@ -30,28 +31,6 @@ import os, os.path
import sys
import hashlib
-import requests
-from requests.exceptions import SSLError
-
-VERIFY_SSL = False
-# py3.2
-if sys.hexversion >= 0x30200f0:
- VERIFY_SSL = True
-else:
- try: # import and enable SNI support for py2
- from requests.packages.urllib3.contrib import pyopenssl
- pyopenssl.inject_into_urllib3()
- VERIFY_SSL = True
- VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
- except ImportError as e:
- VERIFY_MSGS = [
- "Failed to import and inject pyopenssl/SNI support into urllib3",
- "Disabling certificate verification",
- "Error was:" + e
- ]
- VERIFY_SSL = False
-
-
GPG_ENABLED = False
try:
from pygpg.config import GPGConfig
@@ -64,8 +43,10 @@ except ImportError:
from layman.utils import encoder
from layman.dbbase import DbBase
from layman.version import VERSION
-from layman.compatibility import fileopen
+from layman.compatibility import fileopen
+from sslfetch.connections import Connector
+USERAGENT = "Layman-" + VERSION
class RemoteDB(DbBase):
'''Handles fetching the remote overlay list.'''
@@ -88,10 +69,6 @@ class RemoteDB(DbBase):
self.urls = [i.strip()
for i in config['overlays'].split('\n') if len(i)]
- if VERIFY_MSGS:
- for msg in VERIFY_MSGS:
- self.output.debug(msg, 2)
-
if GPG_ENABLED:
self.get_gpg_urls()
else:
@@ -113,7 +90,6 @@ class RemoteDB(DbBase):
self.output.debug('RemoteDB.__init__(), paths to load = %s' %str(paths),
2)
-
if config['nocheck']:
ignore = 2
else:
@@ -171,6 +147,12 @@ class RemoteDB(DbBase):
succeeded = True
url_lists = [self.urls, self.detached_urls, self.signed_urls]
need_gpg = [False, True, True]
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
+
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
urls = url_lists[index]
@@ -187,11 +169,11 @@ class RemoteDB(DbBase):
success, olist, timestamp = self._fetch_file(
url, mpath, tpath)
elif sig:
- success, olist, timestamp = self._fetch_url(
- url[0], mpath, tpath)
+ success, olist, timestamp = fetcher.fetch_content(
+ url[0], tpath)
else:
- success, olist, timestamp = self._fetch_url(
- url, mpath, tpath)
+ success, olist, timestamp = fetcher.fetch_content(
+ url, tpath)
if not success:
#succeeded = False
continue
@@ -262,7 +244,6 @@ class RemoteDB(DbBase):
return base + '_' + hashlib.md5(url_encoded).hexdigest()
-
def _fetch_file(self, url, mpath, tpath=None):
self.output.debug('RemoteDB._fetch_file() url = %s' % url, 2)
# check when the cache was last updated
@@ -306,67 +287,6 @@ class RemoteDB(DbBase):
return (True, olist, url_timestamp)
-
- def _fetch_url(self, url, mpath, tpath=None):
- headers = {'Accept-Charset': 'utf-8',
- 'User-Agent': 'Layman-' + VERSION}
-
- if tpath and os.path.exists(tpath):
- with fileopen(tpath,'r') as previous:
- timestamp = previous.read()
- headers['If-Modified-Since'] = timestamp
- self.output.info('Current-modified: %s' % timestamp, 4)
-
- verify = 'https' in url and VERIFY_SSL
- self.output.debug("Enabled ssl certificate verification: %s, for: %s"
- %(str(verify), url), 3)
-
- if not self.check_path([mpath]):
- return (False, '', '')
- self.output.debug('RemoteDB._fetch_url(); headers = %s'
- % str(headers), 2)
- self.output.debug('RemoteDB._fetch_url(); connecting to opener', 2)
- try:
- connection = requests.get(
- url,
- headers=headers,
- verify=verify,
- proxies=self.proxies,
- )
- except SSLError as error:
- self.output.error('RemoteDB._fetch_url(); Failed to update the '
- 'overlay list from: %s\nSSLError was:%s\n'
- % (url, str(error)))
- except Exception as error:
- self.output.error('RemoteDB._fetch_url(); Failed to update the '
- 'overlay list from: %s\nError was:%s\n'
- % (url, str(error)))
- # py2, py3 compatibility, since only py2 returns keys as lower()
- headers = dict((x.lower(), x) for x in list(connection.headers))
- self.output.info('HEADERS = %s' %str(connection.headers), 4)
- self.output.debug('Status_code = %i' % connection.status_code, 2)
- if connection.status_code in [304]:
- self.output.info('Remote list already up to date: %s'
- % url, 4)
- self.output.info('Last-modified: %s' % timestamp, 4)
- elif connection.status_code not in [200]:
- self.output.error('RemoteDB._fetch_url(); HTTP Status-Code was:\n'
- 'url: %s\n%s'
- % (url, str(connection.status_code)))
-
- if connection.status_code in [200]:
- self.output.info('Remote new list downloaded for: %s'
- % url, 4)
- if 'last-modified' in headers:
- timestamp = connection.headers[headers['last-modified']]
- elif 'date' in headers:
- timestamp = connection.headers[headers['date']]
- else:
- timestamp = None
- return (True, connection.content, timestamp)
- return (False, '', '')
-
-
def check_path(self, paths, hint=True):
'''Check for sufficient privileges'''
self.output.debug('RemoteDB.check_path; paths = ' + str(paths), 8)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-14 23:49 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-14 23:49 UTC (permalink / raw
To: gentoo-commits
commit: 16687673a416f392347d1665dad61b0dd8c5aab3
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 00:35:07 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Wed May 14 23:44:30 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=16687673
Alters "Exception, error" to "Exception as error"
This changes the all instances of the phrase "Exception, error" to
"Exception as error" to allow py2/py3 compatibility.
---
layman/api.py | 16 ++++++++--------
layman/cli.py | 2 +-
layman/dbbase.py | 4 ++--
layman/makeconf.py | 4 ++--
layman/overlays/overlay.py | 2 +-
layman/overlays/source.py | 2 +-
layman/overlays/tar.py | 10 +++++-----
layman/remotedb.py | 10 +++++-----
layman/updater.py | 2 +-
layman/utils.py | 2 +-
10 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/layman/api.py b/layman/api.py
index 11534fd..475691b 100755
--- a/layman/api.py
+++ b/layman/api.py
@@ -113,7 +113,7 @@ class LaymanAPI(object):
try:
self._get_installed_db().delete(
self._get_installed_db().select(ovl))
- except Exception, e:
+ except Exception as e:
self._error(
"Exception caught disabling repository '"+ovl+
"':\n"+str(e))
@@ -147,7 +147,7 @@ class LaymanAPI(object):
try:
success = self._get_installed_db().add(
self._get_remote_db().select(ovl))
- except Exception, e:
+ except Exception as e:
self._error("Exception caught enabling repository '"+ovl+
"' : "+str(e))
results.append(success)
@@ -201,7 +201,7 @@ class LaymanAPI(object):
continue
try:
overlay = db.select(ovl)
- except UnknownOverlayException, error:
+ except UnknownOverlayException as error:
self._error(error)
result[ovl] = ('', False, False)
else:
@@ -255,7 +255,7 @@ class LaymanAPI(object):
overlay = db.select(ovl)
#print "overlay = ", ovl
#print "!!!", overlay
- except UnknownOverlayException, error:
+ except UnknownOverlayException as error:
#print "ERRORS", str(error)
self._error(error)
result[ovl] = ('', False, False)
@@ -311,7 +311,7 @@ class LaymanAPI(object):
#self.output.debug("API.sync(); selecting %s, db = %s" % (ovl, str(db)), 5)
odb = db.select(ovl)
self.output.debug("API.sync(); %s now selected" %ovl, 5)
- except UnknownOverlayException, error:
+ except UnknownOverlayException as error:
#self.output.debug("API.sync(); UnknownOverlayException selecting %s" %ovl, 5)
#self._error(str(error))
fatals.append((ovl,
@@ -364,7 +364,7 @@ class LaymanAPI(object):
self.output.debug("API.sync(); starting db.sync(ovl)", 5)
db.sync(ovl)
success.append((ovl,'Successfully synchronized overlay "' + ovl + '".'))
- except Exception, error:
+ except Exception as error:
fatals.append((ovl,
'Failed to sync overlay "' + ovl + '".\nError was: '
+ str(error)))
@@ -457,7 +457,7 @@ class LaymanAPI(object):
self.output.debug(
'LaymanAPI.fetch_remote_list(); cache updated = %s'
% str(dbreload),8)
- except Exception, error:
+ except Exception as error:
self.output.error('Failed to fetch overlay list!\n Original Error was: '
+ str(error))
return False
@@ -585,7 +585,7 @@ class LaymanAPI(object):
elif self.config['news_reporter'] == 'pkgcore':
# pkgcore is not yet capable
return
- except Exception, err:
+ except Exception as err:
msg = "update_news() failed running %s news reporter function\n" +\
"Error was; %s"
self._error(msg % (self.config['news_reporter'], err))
diff --git a/layman/cli.py b/layman/cli.py
index a5d2799..258e3b9 100644
--- a/layman/cli.py
+++ b/layman/cli.py
@@ -173,7 +173,7 @@ class Main(object):
try:
new_umask = int(umask, 8)
old_umask = os.umask(new_umask)
- except Exception, error:
+ except Exception as error:
self.output.die('Failed setting to umask "' + umask +
'"!\nError was: ' + str(error))
diff --git a/layman/dbbase.py b/layman/dbbase.py
index e5c0fd9..8d57a9f 100644
--- a/layman/dbbase.py
+++ b/layman/dbbase.py
@@ -124,7 +124,7 @@ class DbBase(object):
df = fileopen(path, 'r')
document = df.read()
- except Exception, error:
+ except Exception as error:
if not self.ignore_init_read_errors:
self.output.error('Failed to read the overlay list at ("'
+ path + '")')
@@ -231,7 +231,7 @@ class DbBase(object):
""")
tree.write(f, encoding='utf-8')
f.close()
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to write to local overlays file: '
+ path + '\nError was:\n' + str(error))
diff --git a/layman/makeconf.py b/layman/makeconf.py
index 9740072..1abe318 100644
--- a/layman/makeconf.py
+++ b/layman/makeconf.py
@@ -273,7 +273,7 @@ class MakeConf:
make_conf.close()
- except Exception, error:
+ except Exception as error:
self.output.error('MakeConf: write(); Failed to write "'
+ self.path + '".\nError was:\n' + str(error))
return False
@@ -290,7 +290,7 @@ class MakeConf:
make_conf.close()
- except Exception, error:
+ except Exception as error:
self.output.error('MakeConf: content(); Failed to read "' +
self.path + '".\nError was:\n' + str(error))
raise error
diff --git a/layman/overlays/overlay.py b/layman/overlays/overlay.py
index 65c148b..0c2d042 100755
--- a/layman/overlays/overlay.py
+++ b/layman/overlays/overlay.py
@@ -414,7 +414,7 @@ class Overlay(object):
# Worked, throw other sources away
self.sources = [s]
break
- except Exception, error:
+ except Exception as error:
self.output.warn(str(error), 4)
first_s = False
return res
diff --git a/layman/overlays/source.py b/layman/overlays/source.py
index 2b79f2c..69c59bb 100644
--- a/layman/overlays/source.py
+++ b/layman/overlays/source.py
@@ -177,7 +177,7 @@ class OverlaySource(object):
self.output.warn("Checking for cleanup actions to perform", 4)
self.cleanup()
result = 1
- except Exception, err:
+ except Exception as err:
self.output.error(
'Unknown exception running command: %s' % command_repr)
self.output.error('Original error was: %s' % str(err))
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index c6f2443..e57096d 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -111,7 +111,7 @@ class TarOverlay(OverlaySource):
try:
tar = urllib2.urlopen(tar_url).read()
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to fetch the tar package from: '
+ self.src + '\nError was:' + str(error))
@@ -121,7 +121,7 @@ class TarOverlay(OverlaySource):
out_file = open(pkg, 'w+b')
out_file.write(tar)
out_file.close()
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to store tar package in '
+ pkg + '\nError was:' + str(error))
@@ -140,7 +140,7 @@ class TarOverlay(OverlaySource):
try:
self.output.info('Deleting directory "%s"' % folder, 2)
shutil.rmtree(folder)
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to remove unnecessary tar structure "'
+ folder + '"\nError was:' + str(error))
@@ -149,7 +149,7 @@ class TarOverlay(OverlaySource):
try:
result = self._extract(base=base, tar_url=self.src,
dest_dir=temp_path)
- except Exception, error:
+ except Exception as error:
try_to_wipe(temp_path)
raise error
@@ -165,7 +165,7 @@ class TarOverlay(OverlaySource):
try:
os.rename(source, final_path)
- except Exception, error:
+ except Exception as error:
raise Exception('Failed to rename tar subdirectory ' +
source + ' to ' + final_path +
'\nError was:' + str(error))
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 98dffbb..38d9fa8 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -245,7 +245,7 @@ class RemoteDB(DbBase):
if not os.path.exists(os.path.dirname(mpath)):
try:
os.makedirs(os.path.dirname(mpath))
- except OSError, error:
+ except OSError as error:
raise OSError('Failed to create layman storage directory ' +
os.path.dirname(mpath) + '\n' +
'Error was:' + str(error))
@@ -290,7 +290,7 @@ class RemoteDB(DbBase):
self.output.info('Remote list already up to date: %s'
% url, 4)
self.output.info('Last-modified: %s' % timestamp, 4)
- except IOError, error:
+ except IOError as error:
self.output.error('RemoteDB._fetch_file(); Failed to update the '
'overlay list from: %s\nIOError was:%s\n'
% (url, str(error)))
@@ -333,7 +333,7 @@ class RemoteDB(DbBase):
verify=verify,
proxies=self.proxies,
)
- except SSLError, error:
+ except SSLError as error:
self.output.error('RemoteDB._fetch_url(); Failed to update the '
'overlay list from: %s\nSSLError was:%s\n'
% (url, str(error)))
@@ -388,7 +388,7 @@ class RemoteDB(DbBase):
try:
self.read(olist, origin=url)
- except Exception, error:
+ except Exception as error:
self.output.debug("RemoteDB._check_download(), url=%s \nolist:\n"
% url,2)
self.output.debug(olist, 2)
@@ -419,7 +419,7 @@ class RemoteDB(DbBase):
has_updates = True
- except Exception, error:
+ except Exception as error:
raise IOError('Failed to temporarily cache overlays list in'
' ' + mpath + '\nError was:\n' + str(error))
return has_updates
diff --git a/layman/updater.py b/layman/updater.py
index f7d91e6..7c46707 100644
--- a/layman/updater.py
+++ b/layman/updater.py
@@ -23,7 +23,7 @@ def rename_db(config, newname, output):
"name is...: %s" %newname, 2)
output.notice('')
return
- except OSError, err:
+ except OSError as err:
output.error(" Automatic db rename failed:\n%s" %str(err))
else:
output.info(" Automatic db rename, failed access to: %s"
diff --git a/layman/utils.py b/layman/utils.py
index ab9904f..f174215 100644
--- a/layman/utils.py
+++ b/layman/utils.py
@@ -174,7 +174,7 @@ def delete_empty_directory(mdir, output=None):
output.info('Deleting _empty_ directory "%s"' % mdir, 2)
try:
os.rmdir(mdir)
- except OSError, error:
+ except OSError as error:
output.warn(str(error))
else:
output.warn('Insufficient permissions to delete _empty_ folder "%s".' % mdir)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-14 23:49 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-14 23:49 UTC (permalink / raw
To: gentoo-commits
commit: c5e2e8c437b0bb06b74cf8a0881508f4e47bf321
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 03:56:31 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Wed May 14 23:46:30 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=c5e2e8c4
Adds unicode string compatibility support
Ensures that the necessary strings that need to be unicode, stay
unicode in both py2 and py3 when running layman.
---
layman/api.py | 20 ++++++++-------
layman/db.py | 10 ++++----
layman/dbbase.py | 8 +++---
layman/makeconf.py | 22 ++++++++--------
layman/overlays/overlay.py | 62 ++++++++++++++++++++++++----------------------
layman/remotedb.py | 2 +-
layman/utils.py | 4 ++-
7 files changed, 67 insertions(+), 61 deletions(-)
diff --git a/layman/api.py b/layman/api.py
index 475691b..cbf3f76 100755
--- a/layman/api.py
+++ b/layman/api.py
@@ -13,6 +13,8 @@
# Brian Dolbec <dol-sen@sourceforge.net>
#
+from __future__ import print_function
+
import os
from layman.config import BareConfig
@@ -428,21 +430,21 @@ class LaymanAPI(object):
>>> b.close()
>>> api.get_available()
- [u'wrobel', u'wrobel-stable']
- >>> all = api.get_all_info(u'wrobel')
+ ['wrobel', 'wrobel-stable']
+ >>> all = api.get_all_info('wrobel')
>>> info = all['wrobel']
>>> info['status']
- u'official'
+ 'official'
>>> info['description']
- u'Test'
+ 'Test'
>>> info['sources']
- [(u'https://overlays.gentoo.org/svn/dev/wrobel', 'Subversion', None)]
+ [('https://overlays.gentoo.org/svn/dev/wrobel', 'Subversion', None)]
- #{u'wrobel': {'status': u'official',
- #'owner_name': None, 'description': u'Test',
+ #{'wrobel': {'status': 'official',
+ #'owner_name': None, 'description': 'Test',
#'src_uris': <generator object source_uris at 0x167c3c0>,
- #'owner_email': u'nobody@gentoo.org',
- #'quality': u'experimental', 'name': u'wrobel', 'supported': True,
+ #'owner_email': 'nobody@gentoo.org',
+ #'quality': 'experimental', 'name': 'wrobel', 'supported': True,
#'src_types': <generator object source_types at 0x167c370>,
#'official': True,
#'priority': 10, 'feeds': [], 'irc': None, 'homepage': None}}
diff --git a/layman/db.py b/layman/db.py
index 6530147..ce03e13 100644
--- a/layman/db.py
+++ b/layman/db.py
@@ -111,11 +111,11 @@ class DB(DbBase):
# * Running command "/usr/bin/rsync -rlptDvz --progress --delete --delete-after --timeout=180 --exclude="distfiles/*" --exclude="local/*" --exclude="packages/*" "rsync://gunnarwrobel.de/wrobel-stable/*" "/tmp/file.../wrobel-stable""...
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> m = MakeConf(config, b.overlays)
# >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> os.unlink(write)
>>> os.unlink(write2)
@@ -195,16 +195,16 @@ class DB(DbBase):
# * Running command "/usr/bin/svn co "https://overlays.gentoo.org/svn/dev/wrobel/" "/tmp/file.../wrobel""...
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel', u'wrobel-stable']
+ # ['wrobel', 'wrobel-stable']
# >>> b.delete(b.select('wrobel'))
# >>> c = DbBase([write, ], dict())
# >>> c.overlays.keys()
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> m = MakeConf(config, b.overlays)
# >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
- # [u'wrobel-stable']
+ # ['wrobel-stable']
# >>> os.unlink(write)
>>> os.unlink(write2)
diff --git a/layman/dbbase.py b/layman/dbbase.py
index 475ce44..8fa3ba8 100644
--- a/layman/dbbase.py
+++ b/layman/dbbase.py
@@ -149,10 +149,10 @@ class DbBase(object):
>>> config = {'output': output, 'svn_command': '/usr/bin/svn', 'rsync_command':'/usr/bin/rsync'}
>>> a = DbBase(config, [here + '/tests/testfiles/global-overlays.xml', ])
>>> a.overlays.keys()
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> list(a.overlays['wrobel-stable'].source_uris())
- [u'rsync://gunnarwrobel.de/wrobel-stable']
+ ['rsync://gunnarwrobel.de/wrobel-stable']
'''
try:
document = ET.fromstring(text)
@@ -214,7 +214,7 @@ class DbBase(object):
>>> b.write(write)
>>> c = DbBase({"output": Message() }, [write,])
>>> c.overlays.keys()
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> os.unlink(write)
>>> os.rmdir(tmpdir)
@@ -245,7 +245,7 @@ class DbBase(object):
>>> config = {'output': output, 'svn_command': '/usr/bin/svn', 'rsync_command':'/usr/bin/rsync'}
>>> a = DbBase(config, [here + '/tests/testfiles/global-overlays.xml', ])
>>> list(a.select('wrobel-stable').source_uris())
- [u'rsync://gunnarwrobel.de/wrobel-stable']
+ ['rsync://gunnarwrobel.de/wrobel-stable']
'''
self.output.debug("DbBase.select(), overlay = %s" % overlay, 5)
if not overlay in self.overlays.keys():
diff --git a/layman/makeconf.py b/layman/makeconf.py
index 1abe318..52762a2 100644
--- a/layman/makeconf.py
+++ b/layman/makeconf.py
@@ -50,16 +50,16 @@ class MakeConf:
>>> a.path = write
>>> a.add(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel-stable', u'wrobel-stable']
+ ['wrobel-stable', 'wrobel-stable']
>>> a.add(b.overlays['wrobel'])
>>> [i.name for i in a.overlays]
- [u'wrobel', u'wrobel-stable', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable', 'wrobel-stable']
>>> a.delete(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel']
+ ['wrobel']
>>> a.add(b.overlays['wrobel-stable'])
>>> [i.name for i in a.overlays]
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> a.delete(b.overlays['wrobel'])
>>> n_md5 = str(hashlib.md5(open(write).read()).hexdigest())
>>> o_md5 == n_md5
@@ -105,9 +105,9 @@ class MakeConf:
>>> config['make_conf'] = write
>>> b = MakeConf(config, c.overlays)
>>> [i.name for i in b.overlays]
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
@@ -139,7 +139,7 @@ class MakeConf:
>>> [i.name for i in b.overlays]
[]
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
@@ -164,9 +164,9 @@ class MakeConf:
>>> c = DB(config)
>>> a = MakeConf(config, c.overlays)
>>> [i.name for i in a.overlays]
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> a.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
'''
if os.path.isfile(self.path):
self.content()
@@ -230,9 +230,9 @@ class MakeConf:
>>> config['make_conf'] = write
>>> b = MakeConf(config, c.overlays)
>>> [i.name for i in b.overlays]
- [u'wrobel-stable']
+ ['wrobel-stable']
>>> b.extra
- [u'/usr/local/portage/ebuilds/testing', u'/usr/local/portage/ebuilds/stable', u'/usr/local/portage/kolab2', u'/usr/local/portage/gentoo-webapps-overlay/experimental', u'/usr/local/portage/gentoo-webapps-overlay/production-ready']
+ ['/usr/local/portage/ebuilds/testing', '/usr/local/portage/ebuilds/stable', '/usr/local/portage/kolab2', '/usr/local/portage/gentoo-webapps-overlay/experimental', '/usr/local/portage/gentoo-webapps-overlay/production-ready']
>>> os.unlink(write)
>>> import shutil
diff --git a/layman/overlays/overlay.py b/layman/overlays/overlay.py
index 0c2d042..174518b 100755
--- a/layman/overlays/overlay.py
+++ b/layman/overlays/overlay.py
@@ -20,6 +20,8 @@
#
''' Basic overlay class.'''
+from __future__ import unicode_literals
+
__version__ = "0.2"
#===============================================================================
@@ -85,15 +87,15 @@ class Overlay(object):
>>> output = Message()
>>> a = Overlay({'output': output}, overlays[0])
>>> a.name
- u'wrobel'
+ 'wrobel'
>>> a.is_official()
True
>>> list(a.source_uris())
- [u'https://overlays.gentoo.org/svn/dev/wrobel']
+ ['https://overlays.gentoo.org/svn/dev/wrobel']
>>> a.owner_email
- u'nobody@gentoo.org'
+ 'nobody@gentoo.org'
>>> a.description
- u'Test'
+ 'Test'
>>> a.priority
10
>>> b = Overlay({'output': output}, overlays[1])
@@ -208,7 +210,7 @@ class Overlay(object):
else:
self.status = None
- self.quality = u'experimental'
+ self.quality = 'experimental'
if 'quality' in xml.attrib:
if xml.attrib['quality'] in set(QUALITY_LEVELS):
self.quality = encode(xml.attrib['quality'])
@@ -304,7 +306,7 @@ class Overlay(object):
else:
self.status = None
- self.quality = u'experimental'
+ self.quality = 'experimental'
if len(overlay['quality']):
if overlay['quality'] in set(QUALITY_LEVELS):
self.quality = encode(overlay['quality'])
@@ -453,55 +455,55 @@ class Overlay(object):
<BLANKLINE>
'''
- result = u''
+ result = ''
- result += self.name + u'\n' + (len(self.name) * u'~')
+ result += self.name + '\n' + (len(self.name) * '~')
if len(self.sources) == 1:
- result += u'\nSource : ' + self.sources[0].src
+ result += '\nSource : ' + self.sources[0].src
else:
- result += u'\nSources:'
+ result += '\nSources:'
for i, v in enumerate(self.sources):
result += '\n %d. %s' % (i + 1, v.src)
result += '\n'
if self.owner_name != None:
- result += u'\nContact : %s <%s>' \
+ result += '\nContact : %s <%s>' \
% (self.owner_name, self.owner_email)
else:
- result += u'\nContact : ' + self.owner_email
+ result += '\nContact : ' + self.owner_email
if len(self.sources) == 1:
- result += u'\nType : ' + self.sources[0].type
+ result += '\nType : ' + self.sources[0].type
else:
- result += u'\nType : ' + '/'.join(
+ result += '\nType : ' + '/'.join(
sorted(set(e.type for e in self.sources)))
- result += u'; Priority: ' + str(self.priority) + u'\n'
- result += u'Quality : ' + self.quality + u'\n'
+ result += '; Priority: ' + str(self.priority) + '\n'
+ result += 'Quality : ' + self.quality + '\n'
description = self.description
- description = re.compile(u' +').sub(u' ', description)
- description = re.compile(u'\n ').sub(u'\n', description)
- result += u'\nDescription:'
- result += u'\n '.join((u'\n' + description).split(u'\n'))
- result += u'\n'
+ description = re.compile(' +').sub(' ', description)
+ description = re.compile('\n ').sub('\n', description)
+ result += '\nDescription:'
+ result += '\n '.join(('\n' + description).split('\n'))
+ result += '\n'
if self.homepage != None:
link = self.homepage
- link = re.compile(u' +').sub(u' ', link)
- link = re.compile(u'\n ').sub(u'\n', link)
- result += u'\nLink:'
- result += u'\n '.join((u'\n' + link).split(u'\n'))
- result += u'\n'
+ link = re.compile(' +').sub(' ', link)
+ link = re.compile('\n ').sub('\n', link)
+ result += '\nLink:'
+ result += '\n '.join(('\n' + link).split('\n'))
+ result += '\n'
if self.irc != None:
- result += u'\nIRC : ' + self.irc + u'\n'
+ result += '\nIRC : ' + self.irc + '\n'
if len(self.feeds):
- result += u'\n%s:' % ((len(self.feeds) == 1) and "Feed" or "Feeds")
+ result += '\n%s:' % ((len(self.feeds) == 1) and "Feed" or "Feeds")
for i in self.feeds:
- result += u'\n %s' % i
- result += u'\n'
+ result += '\n %s' % i
+ result += '\n'
return encoder(result, self._encoding_)
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 38d9fa8..b010e51 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -160,7 +160,7 @@ class RemoteDB(DbBase):
>>> os.unlink(a.filepath(config['overlays'])+'.xml')
>>> a.overlays.keys()
- [u'wrobel', u'wrobel-stable']
+ ['wrobel', 'wrobel-stable']
>>> import shutil
>>> shutil.rmtree(tmpdir)
diff --git a/layman/utils.py b/layman/utils.py
index f174215..efb3231 100644
--- a/layman/utils.py
+++ b/layman/utils.py
@@ -21,6 +21,8 @@
'''Utility functions to deal with xml nodes. '''
+from __future__ import unicode_literals
+
__version__ = '$Id: utils.py 236 2006-09-05 20:39:37Z wrobel $'
#===============================================================================
@@ -197,7 +199,7 @@ def create_overlay_dict(**kwargs):
'feeds': [],
'sources': [('','','')],
'priority': 50,
- 'quality': u'experimental',
+ 'quality': 'experimental',
'status': '',
'official': False,
'supported': False,
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-14 23:49 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-14 23:49 UTC (permalink / raw
To: gentoo-commits
commit: 846c2efd8958bc0b5ac26dc1bf597912c04fae87
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 23:07:30 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Wed May 14 23:49:28 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=846c2efd
remotedb.py, tar.py: Implements ssl-fetch code
Migrate to using ssl-fetch (urllib3 based) for downloading files.
---
layman/overlays/tar.py | 30 ++++++++++----
layman/remotedb.py | 108 +++++++------------------------------------------
2 files changed, 37 insertions(+), 101 deletions(-)
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 9cb65a2..94fefb1 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -24,12 +24,16 @@ __version__ = "$Id: tar.py 310 2007-04-09 16:30:40Z wrobel $"
#
#-------------------------------------------------------------------------------
-import os, os.path, sys, urllib2, shutil, tempfile
+import os, os.path, sys, shutil
+
import xml.etree.ElementTree as ET # Python 2.5
from layman.utils import path
-#from layman.debug import OUT
from layman.overlays.source import OverlaySource, require_supported
+from layman.version import VERSION
+from sslfetch.connections import Connector
+
+USERAGENT = "Layman" + VERSION
#===============================================================================
#
@@ -78,6 +82,15 @@ class TarOverlay(OverlaySource):
def __init__(self, parent, config, _location, ignore = 0):
+ self.proxies = {}
+
+ for proxy in ['http_proxy', 'https_proxy']:
+ if config[proxy]:
+ self.proxies[proxy.split('_')[0]] = config[proxy]
+ elif os.getenv(proxy):
+ self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
+
+
super(TarOverlay, self).__init__(parent,
config, _location, ignore)
@@ -109,11 +122,14 @@ class TarOverlay(OverlaySource):
ext = candidate_ext
break
- try:
- tar = urllib2.urlopen(tar_url).read()
- except Exception as error:
- raise Exception('Failed to fetch the tar package from: '
- + self.src + '\nError was:' + str(error))
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
+
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
+
+ success, tar, timestamp = fetcher.fetch_content(tar_url)
pkg = path([base, self.parent.name + ext])
diff --git a/layman/remotedb.py b/layman/remotedb.py
index b010e51..24ee8b8 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -17,6 +17,7 @@
'''Handles different storage files.'''
from __future__ import with_statement
+from __future__ import unicode_literals
__version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
@@ -30,28 +31,6 @@ import os, os.path
import sys
import hashlib
-import requests
-from requests.exceptions import SSLError
-
-VERIFY_SSL = False
-# py3.2
-if sys.hexversion >= 0x30200f0:
- VERIFY_SSL = True
-else:
- try: # import and enable SNI support for py2
- from requests.packages.urllib3.contrib import pyopenssl
- pyopenssl.inject_into_urllib3()
- VERIFY_SSL = True
- VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
- except ImportError as e:
- VERIFY_MSGS = [
- "Failed to import and inject pyopenssl/SNI support into urllib3",
- "Disabling certificate verification",
- "Error was:" + e
- ]
- VERIFY_SSL = False
-
-
GPG_ENABLED = False
try:
from pygpg.config import GPGConfig
@@ -64,8 +43,10 @@ except ImportError:
from layman.utils import encoder
from layman.dbbase import DbBase
from layman.version import VERSION
-from layman.compatibility import fileopen
+from layman.compatibility import fileopen
+from sslfetch.connections import Connector
+USERAGENT = "Layman-" + VERSION
class RemoteDB(DbBase):
'''Handles fetching the remote overlay list.'''
@@ -88,10 +69,6 @@ class RemoteDB(DbBase):
self.urls = [i.strip()
for i in config['overlays'].split('\n') if len(i)]
- if VERIFY_MSGS:
- for msg in VERIFY_MSGS:
- self.output.debug(msg, 2)
-
if GPG_ENABLED:
self.get_gpg_urls()
else:
@@ -113,7 +90,6 @@ class RemoteDB(DbBase):
self.output.debug('RemoteDB.__init__(), paths to load = %s' %str(paths),
2)
-
if config['nocheck']:
ignore = 2
else:
@@ -171,6 +147,12 @@ class RemoteDB(DbBase):
succeeded = True
url_lists = [self.urls, self.detached_urls, self.signed_urls]
need_gpg = [False, True, True]
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
+
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
urls = url_lists[index]
@@ -187,11 +169,11 @@ class RemoteDB(DbBase):
success, olist, timestamp = self._fetch_file(
url, mpath, tpath)
elif sig:
- success, olist, timestamp = self._fetch_url(
- url[0], mpath, tpath)
+ success, olist, timestamp = fetcher.fetch_content(
+ url[0], tpath)
else:
- success, olist, timestamp = self._fetch_url(
- url, mpath, tpath)
+ success, olist, timestamp = fetcher.fetch_content(
+ url, tpath)
if not success:
#succeeded = False
continue
@@ -262,7 +244,6 @@ class RemoteDB(DbBase):
return base + '_' + hashlib.md5(url_encoded).hexdigest()
-
def _fetch_file(self, url, mpath, tpath=None):
self.output.debug('RemoteDB._fetch_file() url = %s' % url, 2)
# check when the cache was last updated
@@ -306,67 +287,6 @@ class RemoteDB(DbBase):
return (True, olist, url_timestamp)
-
- def _fetch_url(self, url, mpath, tpath=None):
- headers = {'Accept-Charset': 'utf-8',
- 'User-Agent': 'Layman-' + VERSION}
-
- if tpath and os.path.exists(tpath):
- with fileopen(tpath,'r') as previous:
- timestamp = previous.read()
- headers['If-Modified-Since'] = timestamp
- self.output.info('Current-modified: %s' % timestamp, 4)
-
- verify = 'https' in url and VERIFY_SSL
- self.output.debug("Enabled ssl certificate verification: %s, for: %s"
- %(str(verify), url), 3)
-
- if not self.check_path([mpath]):
- return (False, '', '')
- self.output.debug('RemoteDB._fetch_url(); headers = %s'
- % str(headers), 2)
- self.output.debug('RemoteDB._fetch_url(); connecting to opener', 2)
- try:
- connection = requests.get(
- url,
- headers=headers,
- verify=verify,
- proxies=self.proxies,
- )
- except SSLError as error:
- self.output.error('RemoteDB._fetch_url(); Failed to update the '
- 'overlay list from: %s\nSSLError was:%s\n'
- % (url, str(error)))
- except Exception as error:
- self.output.error('RemoteDB._fetch_url(); Failed to update the '
- 'overlay list from: %s\nError was:%s\n'
- % (url, str(error)))
- # py2, py3 compatibility, since only py2 returns keys as lower()
- headers = dict((x.lower(), x) for x in list(connection.headers))
- self.output.info('HEADERS = %s' %str(connection.headers), 4)
- self.output.debug('Status_code = %i' % connection.status_code, 2)
- if connection.status_code in [304]:
- self.output.info('Remote list already up to date: %s'
- % url, 4)
- self.output.info('Last-modified: %s' % timestamp, 4)
- elif connection.status_code not in [200]:
- self.output.error('RemoteDB._fetch_url(); HTTP Status-Code was:\n'
- 'url: %s\n%s'
- % (url, str(connection.status_code)))
-
- if connection.status_code in [200]:
- self.output.info('Remote new list downloaded for: %s'
- % url, 4)
- if 'last-modified' in headers:
- timestamp = connection.headers[headers['last-modified']]
- elif 'date' in headers:
- timestamp = connection.headers[headers['date']]
- else:
- timestamp = None
- return (True, connection.content, timestamp)
- return (False, '', '')
-
-
def check_path(self, paths, hint=True):
'''Check for sufficient privileges'''
self.output.debug('RemoteDB.check_path; paths = ' + str(paths), 8)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-15 20:28 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-15 20:28 UTC (permalink / raw
To: gentoo-commits
commit: 07dcf53ca7717085af91bc391b08074a2cd88a8f
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Thu May 15 20:25:37 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Thu May 15 20:25:37 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=07dcf53c
output.py: Migrates output mappings.
remotedb.py, tar.py: Removes output mappings necessary for ssl-fetch
usage in favor of mapping them in output.py to avoid mapping the
functions in more than one place.
---
layman/output.py | 11 +++++++++++
layman/overlays/tar.py | 4 ++--
layman/remotedb.py | 4 ++--
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/layman/output.py b/layman/output.py
index ef348f5..0c19ab9 100644
--- a/layman/output.py
+++ b/layman/output.py
@@ -124,6 +124,11 @@ class Message(MessageBase):
MessageBase.__init__(self, out, err, info_level, warn_level,
note_level, col, error_callback)
+
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
## Output Functions
@@ -161,6 +166,9 @@ class Message(MessageBase):
for i in info.split('\n'):
print(" %s %s" % (self.color_func('green', '*'),i), file=self.std_out)
+ # Maps info function to write function
+ # for ssl-fetch usage.
+ write = info
def status (self, message, status, info = 'ignored'):
@@ -218,6 +226,9 @@ class Message(MessageBase):
self.std_out.flush()
self.do_error_callback(error)
+ # Maps error function with print_err function
+ # for ssl-fetch usage.
+ print_err = error
def die (self, error):
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 94fefb1..8574f1a 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -126,8 +126,8 @@ class TarOverlay(OverlaySource):
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
success, tar, timestamp = fetcher.fetch_content(tar_url)
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 24ee8b8..959b090 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -150,8 +150,8 @@ class RemoteDB(DbBase):
fetcher = Connector(self.output, self.proxies, USERAGENT)
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-15 20:37 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-15 20:37 UTC (permalink / raw
To: gentoo-commits
commit: 9b206b1c201d8c36f503e64588a431411270c1d7
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Thu May 15 20:25:37 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Thu May 15 20:35:49 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=9b206b1c
output.py: Migrates output mappings
remotedb.py, tar.py: Removes output mappings necessary for ssl-fetch
usage in favor of mapping them in output.py to avoid mapping the
functions in more than one place.
---
layman/output.py | 9 +++++++--
layman/overlays/tar.py | 4 ++--
layman/remotedb.py | 4 ++--
3 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/layman/output.py b/layman/output.py
index ef348f5..a6c2c20 100644
--- a/layman/output.py
+++ b/layman/output.py
@@ -124,8 +124,7 @@ class Message(MessageBase):
MessageBase.__init__(self, out, err, info_level, warn_level,
note_level, col, error_callback)
-
-
+
## Output Functions
def debug(self, info, level = OFF):
@@ -161,6 +160,9 @@ class Message(MessageBase):
for i in info.split('\n'):
print(" %s %s" % (self.color_func('green', '*'),i), file=self.std_out)
+ # Maps info function to write function
+ # for ssl-fetch usage.
+ write = info
def status (self, message, status, info = 'ignored'):
@@ -218,6 +220,9 @@ class Message(MessageBase):
self.std_out.flush()
self.do_error_callback(error)
+ # Maps error function with print_err function
+ # for ssl-fetch usage.
+ print_err = error
def die (self, error):
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 94fefb1..8574f1a 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -126,8 +126,8 @@ class TarOverlay(OverlaySource):
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
success, tar, timestamp = fetcher.fetch_content(tar_url)
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 24ee8b8..959b090 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -150,8 +150,8 @@ class RemoteDB(DbBase):
fetcher = Connector(self.output, self.proxies, USERAGENT)
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-15 20:46 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-15 20:46 UTC (permalink / raw
To: gentoo-commits
commit: bf10389bf1c5e767ce66f34d38aefdb53b6a966c
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Thu May 15 20:25:37 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Thu May 15 20:45:50 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=bf10389b
output.py: Migrates output mappings
remotedb.py, tar.py: Removes output mappings necessary for ssl-fetch
usage in favor of mapping them in output.py to avoid mapping the
functions in more than one place.
---
layman/output.py | 9 +++++++--
layman/overlays/tar.py | 4 ++--
layman/remotedb.py | 4 ++--
3 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/layman/output.py b/layman/output.py
index ef348f5..a6c2c20 100644
--- a/layman/output.py
+++ b/layman/output.py
@@ -124,8 +124,7 @@ class Message(MessageBase):
MessageBase.__init__(self, out, err, info_level, warn_level,
note_level, col, error_callback)
-
-
+
## Output Functions
def debug(self, info, level = OFF):
@@ -161,6 +160,9 @@ class Message(MessageBase):
for i in info.split('\n'):
print(" %s %s" % (self.color_func('green', '*'),i), file=self.std_out)
+ # Maps info function to write function
+ # for ssl-fetch usage.
+ write = info
def status (self, message, status, info = 'ignored'):
@@ -218,6 +220,9 @@ class Message(MessageBase):
self.std_out.flush()
self.do_error_callback(error)
+ # Maps error function with print_err function
+ # for ssl-fetch usage.
+ print_err = error
def die (self, error):
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 94fefb1..8574f1a 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -126,8 +126,8 @@ class TarOverlay(OverlaySource):
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
success, tar, timestamp = fetcher.fetch_content(tar_url)
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 24ee8b8..959b090 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -150,8 +150,8 @@ class RemoteDB(DbBase):
fetcher = Connector(self.output, self.proxies, USERAGENT)
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-16 0:58 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-16 0:58 UTC (permalink / raw
To: gentoo-commits
commit: f4f4defbcd5365ae362675805faad4df4d642c3f
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Thu May 15 20:25:37 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Fri May 16 00:58:20 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=f4f4defb
output.py: Migrates output mappings
remotedb.py, tar.py: Removes output mappings necessary for ssl-fetch
usage in favor of mapping them in output.py to avoid mapping the
functions in more than one place.
---
layman/output.py | 9 +++++++--
layman/overlays/tar.py | 4 ++--
layman/remotedb.py | 4 ++--
3 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/layman/output.py b/layman/output.py
index ef348f5..a6c2c20 100644
--- a/layman/output.py
+++ b/layman/output.py
@@ -124,8 +124,7 @@ class Message(MessageBase):
MessageBase.__init__(self, out, err, info_level, warn_level,
note_level, col, error_callback)
-
-
+
## Output Functions
def debug(self, info, level = OFF):
@@ -161,6 +160,9 @@ class Message(MessageBase):
for i in info.split('\n'):
print(" %s %s" % (self.color_func('green', '*'),i), file=self.std_out)
+ # Maps info function to write function
+ # for ssl-fetch usage.
+ write = info
def status (self, message, status, info = 'ignored'):
@@ -218,6 +220,9 @@ class Message(MessageBase):
self.std_out.flush()
self.do_error_callback(error)
+ # Maps error function with print_err function
+ # for ssl-fetch usage.
+ print_err = error
def die (self, error):
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 94fefb1..8574f1a 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -126,8 +126,8 @@ class TarOverlay(OverlaySource):
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
success, tar, timestamp = fetcher.fetch_content(tar_url)
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 24ee8b8..959b090 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -150,8 +150,8 @@ class RemoteDB(DbBase):
fetcher = Connector(self.output, self.proxies, USERAGENT)
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-16 1:07 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-16 1:07 UTC (permalink / raw
To: gentoo-commits
commit: d827daeb694f4d91fdeff15143b738438c1ce415
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed May 14 23:07:30 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Fri May 16 01:06:46 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=d827daeb
remotedb.py, tar.py: Implements ssl-fetch code
Migrate to using ssl-fetch (urllib3 based) for downloading files.
---
layman/overlays/tar.py | 30 ++++++++++----
layman/remotedb.py | 108 +++++++------------------------------------------
2 files changed, 37 insertions(+), 101 deletions(-)
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 9cb65a2..94fefb1 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -24,12 +24,16 @@ __version__ = "$Id: tar.py 310 2007-04-09 16:30:40Z wrobel $"
#
#-------------------------------------------------------------------------------
-import os, os.path, sys, urllib2, shutil, tempfile
+import os, os.path, sys, shutil
+
import xml.etree.ElementTree as ET # Python 2.5
from layman.utils import path
-#from layman.debug import OUT
from layman.overlays.source import OverlaySource, require_supported
+from layman.version import VERSION
+from sslfetch.connections import Connector
+
+USERAGENT = "Layman" + VERSION
#===============================================================================
#
@@ -78,6 +82,15 @@ class TarOverlay(OverlaySource):
def __init__(self, parent, config, _location, ignore = 0):
+ self.proxies = {}
+
+ for proxy in ['http_proxy', 'https_proxy']:
+ if config[proxy]:
+ self.proxies[proxy.split('_')[0]] = config[proxy]
+ elif os.getenv(proxy):
+ self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
+
+
super(TarOverlay, self).__init__(parent,
config, _location, ignore)
@@ -109,11 +122,14 @@ class TarOverlay(OverlaySource):
ext = candidate_ext
break
- try:
- tar = urllib2.urlopen(tar_url).read()
- except Exception as error:
- raise Exception('Failed to fetch the tar package from: '
- + self.src + '\nError was:' + str(error))
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
+
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
+
+ success, tar, timestamp = fetcher.fetch_content(tar_url)
pkg = path([base, self.parent.name + ext])
diff --git a/layman/remotedb.py b/layman/remotedb.py
index b010e51..24ee8b8 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -17,6 +17,7 @@
'''Handles different storage files.'''
from __future__ import with_statement
+from __future__ import unicode_literals
__version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
@@ -30,28 +31,6 @@ import os, os.path
import sys
import hashlib
-import requests
-from requests.exceptions import SSLError
-
-VERIFY_SSL = False
-# py3.2
-if sys.hexversion >= 0x30200f0:
- VERIFY_SSL = True
-else:
- try: # import and enable SNI support for py2
- from requests.packages.urllib3.contrib import pyopenssl
- pyopenssl.inject_into_urllib3()
- VERIFY_SSL = True
- VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
- except ImportError as e:
- VERIFY_MSGS = [
- "Failed to import and inject pyopenssl/SNI support into urllib3",
- "Disabling certificate verification",
- "Error was:" + e
- ]
- VERIFY_SSL = False
-
-
GPG_ENABLED = False
try:
from pygpg.config import GPGConfig
@@ -64,8 +43,10 @@ except ImportError:
from layman.utils import encoder
from layman.dbbase import DbBase
from layman.version import VERSION
-from layman.compatibility import fileopen
+from layman.compatibility import fileopen
+from sslfetch.connections import Connector
+USERAGENT = "Layman-" + VERSION
class RemoteDB(DbBase):
'''Handles fetching the remote overlay list.'''
@@ -88,10 +69,6 @@ class RemoteDB(DbBase):
self.urls = [i.strip()
for i in config['overlays'].split('\n') if len(i)]
- if VERIFY_MSGS:
- for msg in VERIFY_MSGS:
- self.output.debug(msg, 2)
-
if GPG_ENABLED:
self.get_gpg_urls()
else:
@@ -113,7 +90,6 @@ class RemoteDB(DbBase):
self.output.debug('RemoteDB.__init__(), paths to load = %s' %str(paths),
2)
-
if config['nocheck']:
ignore = 2
else:
@@ -171,6 +147,12 @@ class RemoteDB(DbBase):
succeeded = True
url_lists = [self.urls, self.detached_urls, self.signed_urls]
need_gpg = [False, True, True]
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
+ # Maps output functions for compatibility with ssl-fetch
+ # output calls.
+ self.output.write = self.output.info
+ self.output.print_err = self.output.error
+
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
urls = url_lists[index]
@@ -187,11 +169,11 @@ class RemoteDB(DbBase):
success, olist, timestamp = self._fetch_file(
url, mpath, tpath)
elif sig:
- success, olist, timestamp = self._fetch_url(
- url[0], mpath, tpath)
+ success, olist, timestamp = fetcher.fetch_content(
+ url[0], tpath)
else:
- success, olist, timestamp = self._fetch_url(
- url, mpath, tpath)
+ success, olist, timestamp = fetcher.fetch_content(
+ url, tpath)
if not success:
#succeeded = False
continue
@@ -262,7 +244,6 @@ class RemoteDB(DbBase):
return base + '_' + hashlib.md5(url_encoded).hexdigest()
-
def _fetch_file(self, url, mpath, tpath=None):
self.output.debug('RemoteDB._fetch_file() url = %s' % url, 2)
# check when the cache was last updated
@@ -306,67 +287,6 @@ class RemoteDB(DbBase):
return (True, olist, url_timestamp)
-
- def _fetch_url(self, url, mpath, tpath=None):
- headers = {'Accept-Charset': 'utf-8',
- 'User-Agent': 'Layman-' + VERSION}
-
- if tpath and os.path.exists(tpath):
- with fileopen(tpath,'r') as previous:
- timestamp = previous.read()
- headers['If-Modified-Since'] = timestamp
- self.output.info('Current-modified: %s' % timestamp, 4)
-
- verify = 'https' in url and VERIFY_SSL
- self.output.debug("Enabled ssl certificate verification: %s, for: %s"
- %(str(verify), url), 3)
-
- if not self.check_path([mpath]):
- return (False, '', '')
- self.output.debug('RemoteDB._fetch_url(); headers = %s'
- % str(headers), 2)
- self.output.debug('RemoteDB._fetch_url(); connecting to opener', 2)
- try:
- connection = requests.get(
- url,
- headers=headers,
- verify=verify,
- proxies=self.proxies,
- )
- except SSLError as error:
- self.output.error('RemoteDB._fetch_url(); Failed to update the '
- 'overlay list from: %s\nSSLError was:%s\n'
- % (url, str(error)))
- except Exception as error:
- self.output.error('RemoteDB._fetch_url(); Failed to update the '
- 'overlay list from: %s\nError was:%s\n'
- % (url, str(error)))
- # py2, py3 compatibility, since only py2 returns keys as lower()
- headers = dict((x.lower(), x) for x in list(connection.headers))
- self.output.info('HEADERS = %s' %str(connection.headers), 4)
- self.output.debug('Status_code = %i' % connection.status_code, 2)
- if connection.status_code in [304]:
- self.output.info('Remote list already up to date: %s'
- % url, 4)
- self.output.info('Last-modified: %s' % timestamp, 4)
- elif connection.status_code not in [200]:
- self.output.error('RemoteDB._fetch_url(); HTTP Status-Code was:\n'
- 'url: %s\n%s'
- % (url, str(connection.status_code)))
-
- if connection.status_code in [200]:
- self.output.info('Remote new list downloaded for: %s'
- % url, 4)
- if 'last-modified' in headers:
- timestamp = connection.headers[headers['last-modified']]
- elif 'date' in headers:
- timestamp = connection.headers[headers['date']]
- else:
- timestamp = None
- return (True, connection.content, timestamp)
- return (False, '', '')
-
-
def check_path(self, paths, hint=True):
'''Check for sufficient privileges'''
self.output.debug('RemoteDB.check_path; paths = ' + str(paths), 8)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-05-16 1:07 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-05-16 1:07 UTC (permalink / raw
To: gentoo-commits
commit: 4ea75fa8beb75e7e9be74b2dc4bebca0a808ad37
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Thu May 15 20:25:37 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Fri May 16 01:06:46 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=4ea75fa8
output.py: Migrates output mappings
remotedb.py, tar.py: Removes output mappings necessary for ssl-fetch
usage in favor of mapping them in output.py to avoid mapping the
functions in more than one place.
---
layman/output.py | 9 +++++++--
layman/overlays/tar.py | 4 ++--
layman/remotedb.py | 4 ++--
3 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/layman/output.py b/layman/output.py
index ef348f5..a6c2c20 100644
--- a/layman/output.py
+++ b/layman/output.py
@@ -124,8 +124,7 @@ class Message(MessageBase):
MessageBase.__init__(self, out, err, info_level, warn_level,
note_level, col, error_callback)
-
-
+
## Output Functions
def debug(self, info, level = OFF):
@@ -161,6 +160,9 @@ class Message(MessageBase):
for i in info.split('\n'):
print(" %s %s" % (self.color_func('green', '*'),i), file=self.std_out)
+ # Maps info function to write function
+ # for ssl-fetch usage.
+ write = info
def status (self, message, status, info = 'ignored'):
@@ -218,6 +220,9 @@ class Message(MessageBase):
self.std_out.flush()
self.do_error_callback(error)
+ # Maps error function with print_err function
+ # for ssl-fetch usage.
+ print_err = error
def die (self, error):
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 94fefb1..8574f1a 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -126,8 +126,8 @@ class TarOverlay(OverlaySource):
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
success, tar, timestamp = fetcher.fetch_content(tar_url)
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 24ee8b8..959b090 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -150,8 +150,8 @@ class RemoteDB(DbBase):
fetcher = Connector(self.output, self.proxies, USERAGENT)
# Maps output functions for compatibility with ssl-fetch
# output calls.
- self.output.write = self.output.info
- self.output.print_err = self.output.error
+ #self.output.write = self.output.info
+ #self.output.print_err = self.output.error
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
2014-06-16 3:40 [gentoo-commits] proj/layman:master " Brian Dolbec
@ 2014-06-16 3:37 ` Brian Dolbec
0 siblings, 0 replies; 20+ messages in thread
From: Brian Dolbec @ 2014-06-16 3:37 UTC (permalink / raw
To: gentoo-commits
commit: 0764934ac9670197a051af437f586d5594235c80
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Fri May 23 21:25:08 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Jun 12 21:11:49 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=0764934a
overlay.py: adds update() function
To update local database and overlay source urls, this function
has been created to allow the overlay.type functions to update
their source urls in their own methods.
source.py: Adds a stub update() function for overlay types
that don't have their own update() function.
config.py: Adds a list of supported overlay types that have their
own update() methods.
---
layman/config.py | 1 +
layman/overlays/overlay.py | 34 ++++++++++++++++++++++++++++++++++
layman/overlays/source.py | 8 ++++++++
3 files changed, 43 insertions(+)
diff --git a/layman/config.py b/layman/config.py
index c8fdf4f..40ee90a 100644
--- a/layman/config.py
+++ b/layman/config.py
@@ -165,6 +165,7 @@ class BareConfig(object):
'g-sorcery_postsync' : '',
'git_user': 'layman',
'git_email': 'layman@localhost',
+ 'support_url_updates': ['Bzr', 'cvs', 'Git', 'Mercurial', 'Subversion'],
}
self._options = {
'config': config if config else self._defaults['config'],
diff --git a/layman/overlays/overlay.py b/layman/overlays/overlay.py
index 174518b..d3d81e2 100755
--- a/layman/overlays/overlay.py
+++ b/layman/overlays/overlay.py
@@ -422,6 +422,40 @@ class Overlay(object):
return res
+ def update(self, base, available_srcs):
+ res = 1
+ first_src = True
+ result = False
+
+ if isinstance(available_srcs, str):
+ available_srcs = [available_srcs]
+
+ if self.sources[0].type in self.config.get_option('support_url_updates'):
+ for src in available_srcs:
+ if not first_src:
+ self.output.info("\nTrying next source of listed sources...", 4)
+ try:
+ res = self.sources[0].update(base, src)
+ if res == 0:
+ # Updating it worked, no need to bother
+ # checking other sources.
+ self.sources[0].src = src
+ result = True
+ break
+ except Exception as error:
+ self.output.warn(str(error), 4)
+ first_s = False
+ else:
+ # Update the overlay source with the remote
+ # source, assuming that it's telling the truth
+ # so it can be written to the installed.xml.
+ self.output.debug("overlay.update(); type: %s does not support"\
+ " source URL updating" % self.sources[0].type, 4)
+ self.sources[0].src = available_srcs.pop()
+ result = True
+ return (self.sources, result)
+
+
def sync(self, base):
self.output.debug("overlay.sync(); name = %s" % self.name, 4)
assert len(self.sources) == 1
diff --git a/layman/overlays/source.py b/layman/overlays/source.py
index 5044156..7c9674b 100644
--- a/layman/overlays/source.py
+++ b/layman/overlays/source.py
@@ -95,6 +95,14 @@ class OverlaySource(object):
os.makedirs(mdir)
return True
+ def update(self, src):
+ '''
+ Updates the overlay source url.
+
+ @params src: source URL.
+ '''
+ pass
+
def sync(self, base):
'''Sync the overlay.'''
pass
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
2014-06-16 3:40 [gentoo-commits] proj/layman:master " Brian Dolbec
@ 2014-06-16 3:37 ` Brian Dolbec
0 siblings, 0 replies; 20+ messages in thread
From: Brian Dolbec @ 2014-06-16 3:37 UTC (permalink / raw
To: gentoo-commits
commit: fda51375fed8b54e264e4577fb3d72385bbffa17
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Fri May 23 18:37:51 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Jun 12 21:11:50 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=fda51375
config.py: Adds proxies() @property function
Since remotedb.py and tar.py both made use of the same code
that determined available proxies, the code has been moved to a
function in config.py and both files have been changed to make use
of this function as a value.
---
layman/config.py | 23 ++++++++++++++++++++++-
layman/overlays/tar.py | 10 +---------
layman/remotedb.py | 11 ++---------
3 files changed, 25 insertions(+), 19 deletions(-)
diff --git a/layman/config.py b/layman/config.py
index 40ee90a..9d2bc7f 100644
--- a/layman/config.py
+++ b/layman/config.py
@@ -66,7 +66,6 @@ def read_layman_config(config=None, defaults=None, output=None):
overlays.update(["file://" + _path])
config.set('MAIN', 'overlays', '\n'.join(overlays))
-
# establish the eprefix, initially set so eprefixify can
# set it on install
EPREFIX = "@GENTOO_PORTAGE_EPREFIX@"
@@ -280,6 +279,28 @@ class BareConfig(object):
"""
return option.lower() in ['yes', 'true', 'y', 't']
+ @property
+ def proxies(self):
+ """
+ Reads the config options to determine the available proxies.
+
+ @param config: config options dict.
+ @rtype dict
+ """
+ proxies = {}
+
+ for proxy in ['http_proxy', 'https_proxy']:
+ if self.config and self.config.get('MAIN', proxy):
+ proxies[proxy.split('_')[0]] = self.config.get('MAIN', proxy)
+ elif self.get_option(proxy):
+ proxies[proxy.split('_')[0]] = self.get_option(proxy)
+ elif os.getenv(proxy):
+ proxies[proxy.split('_')[0]] = os.getenv(proxy)
+ if self.config and proxies == {}:
+ self.output.debug("Warning: unable to determine proxies.", 6)
+
+ return proxies
+
class OptionConfig(BareConfig):
"""This subclasses BareCongig adding functions to make overriding
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index d2c1cc9..a819475 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -88,19 +88,11 @@ class TarOverlay(OverlaySource):
def __init__(self, parent, config, _location, ignore = 0):
- self.proxies = {}
-
- for proxy in ['http_proxy', 'https_proxy']:
- if config[proxy]:
- self.proxies[proxy.split('_')[0]] = config[proxy]
- elif os.getenv(proxy):
- self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
-
-
super(TarOverlay, self).__init__(parent,
config, _location, ignore)
self.output = config['output']
+ self.proxies = config.proxies
self.subpath = None
def __eq__(self, other):
diff --git a/layman/remotedb.py b/layman/remotedb.py
index 175aeb5..f883799 100644
--- a/layman/remotedb.py
+++ b/layman/remotedb.py
@@ -57,15 +57,8 @@ class RemoteDB(DbBase):
self.output = config['output']
self.detached_urls = []
self.signed_urls = []
-
- self.proxies = {}
-
- for proxy in ['http_proxy', 'https_proxy']:
- if config[proxy]:
- self.proxies[proxy.split('_')[0]] = config[proxy]
- elif os.getenv(proxy):
- self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
-
+ self.proxies = config.proxies
+
self.urls = [i.strip()
for i in config['overlays'].split('\n') if len(i)]
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-06-16 3:37 Brian Dolbec
0 siblings, 0 replies; 20+ messages in thread
From: Brian Dolbec @ 2014-06-16 3:37 UTC (permalink / raw
To: gentoo-commits
commit: e5aeb45de0db5912b57f1574562fc7d474afaa67
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 6 21:18:14 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Jun 14 22:07:34 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=e5aeb45d
Various syntax fixes
Fixes to docstring indentation, output messages, and function
definition syntax.
---
layman/api.py | 6 +++---
layman/overlays/bzr.py | 8 ++++----
layman/overlays/git.py | 8 ++++----
layman/overlays/mercurial.py | 8 ++++----
layman/overlays/svn.py | 8 ++++----
layman/repoconfmanager.py | 30 +++++++++++++++---------------
6 files changed, 34 insertions(+), 34 deletions(-)
diff --git a/layman/api.py b/layman/api.py
index d9d3eb1..1d50adf 100755
--- a/layman/api.py
+++ b/layman/api.py
@@ -333,7 +333,7 @@ class LaymanAPI(object):
'\n'\
' %(remote_type)s\n'\
'\n'\
- 'the overlay will be readded using %(remote_name)s' %
+ 'the overlay will be readded using %(remote_name)s' %\
({
'repo_name': odb.name,
'current_type': current_type,
@@ -342,7 +342,7 @@ class LaymanAPI(object):
return True, msg
return False, ''
- def _verify_overlay_source(self, odb, ordb)
+ def _verify_overlay_source(self, odb, ordb):
"""
Verifies the overlay source url against the source url(s)
reported by the remote database.
@@ -367,7 +367,7 @@ class LaymanAPI(object):
msg = 'The source of the overlay "%(repo_name)s" seems to have changed.\n'\
'You currently sync from\n'\
'\n'\
- ' %(current_src)s\n'
+ ' %(current_src)s\n'\
'\n'\
'while the remote lists report\n'\
'\n'\
diff --git a/layman/overlays/bzr.py b/layman/overlays/bzr.py
index 20d2491..162ba40 100644
--- a/layman/overlays/bzr.py
+++ b/layman/overlays/bzr.py
@@ -50,11 +50,11 @@ class BzrOverlay(OverlaySource):
self.subpath = None
def _fix_bzr_source(self, source):
- '''
- Adds trailing slash to source URL if needed.
+ '''
+ Adds trailing slash to source URL if needed.
- @params source: source URL, string.
- '''
+ @params source: source URL, string.
+ '''
if source.endswith("/"):
return source
return source + '/'
diff --git a/layman/overlays/git.py b/layman/overlays/git.py
index ee8c53b..5e99a65 100644
--- a/layman/overlays/git.py
+++ b/layman/overlays/git.py
@@ -47,11 +47,11 @@ class GitOverlay(OverlaySource):
self.subpath = None
def _fix_git_source(self, source):
- '''
- Adds trailing slash to http sources
+ '''
+ Adds trailing slash to http sources
- @param source: source URL, string.
- '''
+ @param source: source URL, string.
+ '''
# http:// should get trailing slash, other protocols shouldn't
if source.split(':')[:1] == 'http':
if not source.endswith('/'):
diff --git a/layman/overlays/mercurial.py b/layman/overlays/mercurial.py
index 74f1934..9f7d45c 100644
--- a/layman/overlays/mercurial.py
+++ b/layman/overlays/mercurial.py
@@ -51,11 +51,11 @@ class MercurialOverlay(OverlaySource):
self.subpath = None
def _fix_mercurial_source(self, source):
- '''
- Adds trailing slash to source URL if needed.
+ '''
+ Adds trailing slash to source URL if needed.
- @params source: source URL, string.
- '''
+ @params source: source URL, string.
+ '''
if source.endswith("/"):
return source
return source + '/'
diff --git a/layman/overlays/svn.py b/layman/overlays/svn.py
index f494eba..41dd862 100644
--- a/layman/overlays/svn.py
+++ b/layman/overlays/svn.py
@@ -54,11 +54,11 @@ class SvnOverlay(OverlaySource):
self.subpath = None
def _fix_svn_source(self, source):
- '''
- Adds @ to all sources that don't already include it.
+ '''
+ Adds @ to all sources that don't already include it.
- @params source: source URL, string.
- '''
+ @params source: source URL, string.
+ '''
if source.endswith("/"):
source = source + '@'
else:
diff --git a/layman/repoconfmanager.py b/layman/repoconfmanager.py
index 502dc2a..415a454 100644
--- a/layman/repoconfmanager.py
+++ b/layman/repoconfmanager.py
@@ -44,12 +44,12 @@ class RepoConfManager:
def add(self, overlay):
- '''
- Adds overlay information to the specified config type(s).
+ '''
+ Adds overlay information to the specified config type(s).
- @param overlay: layman.overlay.Overlay instance.
- @return boolean: represents success or failure.
- '''
+ @param overlay: layman.overlay.Overlay instance.
+ @return boolean: represents success or failure.
+ '''
if self.config['require_repoconfig']:
for types in self.conf_types:
conf = getattr(self.modules[types][0],
@@ -59,12 +59,12 @@ class RepoConfManager:
return True
def delete(self, overlay):
- '''
- Deletes overlay information from the specified config type(s).
+ '''
+ Deletes overlay information from the specified config type(s).
- @param overlay: layman.overlay.Overlay instance.
- @return boolean: represents success or failure.
- '''
+ @param overlay: layman.overlay.Overlay instance.
+ @return boolean: represents success or failure.
+ '''
if self.config['require_repoconfig']:
for types in self.conf_types:
conf = getattr(self.modules[types][0],
@@ -75,12 +75,12 @@ class RepoConfManager:
def update(self, overlay):
- '''
- Updates the source URL for the specified config type(s).
+ '''
+ Updates the source URL for the specified config type(s).
- @param overlay: layman.overlay.Overlay instance.
- @return boolean: represents success or failure.
- '''
+ @param overlay: layman.overlay.Overlay instance.
+ @return boolean: represents success or failure.
+ '''
if self.config['require_repoconfig']:
for types in self.conf_types:
conf = getattr(self.modules[types][0],
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-08-15 22:32 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-08-15 22:32 UTC (permalink / raw
To: gentoo-commits
commit: 3ea8512b470742610c3e450cbab20fc9e5609bb2
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Tue Aug 12 04:09:34 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 15 21:42:40 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=3ea8512b
Migrates run_command to utily.py
---
layman/overlays/bzr.py | 9 ++--
layman/overlays/cvs.py | 13 +++---
layman/overlays/darcs.py | 7 ++--
layman/overlays/g_common.py | 8 ++--
layman/overlays/g_sorcery.py | 8 ++--
layman/overlays/git.py | 16 +++++---
layman/overlays/mercurial.py | 9 ++--
layman/overlays/rsync.py | 4 +-
layman/overlays/source.py | 95 ++----------------------------------------
layman/overlays/svn.py | 16 ++++----
layman/overlays/tar.py | 3 +-
layman/utils.py | 98 ++++++++++++++++++++++++++++++++++++++++++--
12 files changed, 152 insertions(+), 134 deletions(-)
diff --git a/layman/overlays/bzr.py b/layman/overlays/bzr.py
index ea3e787..614f816 100644
--- a/layman/overlays/bzr.py
+++ b/layman/overlays/bzr.py
@@ -28,7 +28,7 @@ __version__ = "$Id: bzr.py 236 2006-09-05 20:39:37Z wrobel $"
#
#-------------------------------------------------------------------------------
-from layman.utils import path
+from layman.utils import path, run_command
from layman.overlays.source import OverlaySource, require_supported
#===============================================================================
@@ -77,7 +77,7 @@ class BzrOverlay(OverlaySource):
else:
args = ['branch', src, target]
return self.postsync(
- self.run_command(self.command(), args, cmd=self.type),
+ run_command(self.config, self.command(), args, cmd=self.type),
cwd=target)
def update(self, base, src):
@@ -98,7 +98,7 @@ class BzrOverlay(OverlaySource):
if self.config['quiet']:
args.append('--quiet')
return self.postsync(
- self.run_command(self.command(), args, cmd=self.type),
+ run_command(self.config, self.command(), args, cmd=self.type),
cwd=target)
def sync(self, base):
@@ -116,7 +116,8 @@ class BzrOverlay(OverlaySource):
else:
args = ['pull', '--overwrite', self.src]
return self.postsync(
- self.run_command(self.command(), args, cwd=target, cmd=self.type),
+ run_command(self.config, self.command(), args, cwd=target,
+ cmd=self.type),
cwd=target)
def supported(self):
diff --git a/layman/overlays/cvs.py b/layman/overlays/cvs.py
index 70638e3..62ad156 100644
--- a/layman/overlays/cvs.py
+++ b/layman/overlays/cvs.py
@@ -29,7 +29,7 @@ __version__ = "$Id$"
import xml.etree.ElementTree as ET # Python 2.5
import re
-from layman.utils import path
+from layman.utils import path, run_command
from layman.overlays.source import OverlaySource, require_supported
#===============================================================================
@@ -71,7 +71,7 @@ class CvsOverlay(OverlaySource):
args.append(self.branch)
return self.postsync(
- self.run_command(self.command(), args, cwd=base,
+ run_command(self.config, self.command(), args, cwd=base,
env=dict(CVSROOT=self.src), cmd=self.type),
cwd=target)
@@ -90,7 +90,8 @@ class CvsOverlay(OverlaySource):
# First echo the new repository to CVS/Root
args = [src, '>', '/CVS/Root']
- result = self.run_command('echo', args, cmd='echo', cwd=target)
+ result = run_command(self.config, 'echo', args, cmd='echo',
+ cwd=target)
if result == 0:
location = src.split(':')[3]
@@ -106,7 +107,8 @@ class CvsOverlay(OverlaySource):
# sed -i 's/<old_location>/<new_location>/ <target>/CVS/Repository
args = ['-i', expression, '/CVS/Repository']
- return self.run_command('sed', args, cmd='sed', cwd=target)
+ return run_command(self.config, 'sed', args, cmd='sed',
+ cwd=target)
return result
@@ -129,7 +131,8 @@ class CvsOverlay(OverlaySource):
if len(cfg_opts):
args.append(cfg_opts)
return self.postsync(
- self.run_command(self.command(), args, cwd=target, cmd=self.type),
+ run_command(self.config, self.command(), args, cwd=target,
+ cmd=self.type),
cwd=target)
def supported(self):
diff --git a/layman/overlays/darcs.py b/layman/overlays/darcs.py
index 53966c6..9e951da 100644
--- a/layman/overlays/darcs.py
+++ b/layman/overlays/darcs.py
@@ -27,7 +27,7 @@ __version__ = "$Id: darcs.py 236 2006-09-05 20:39:37Z wrobel $"
#
#-------------------------------------------------------------------------------
-from layman.utils import path
+from layman.utils import path, run_command
from layman.overlays.source import OverlaySource, require_supported
#===============================================================================
@@ -71,7 +71,7 @@ class DarcsOverlay(OverlaySource):
src, target]
return self.postsync(
- self.run_command(self.command(), args, cmd=self.type),
+ run_command(self.config, self.command(), args, cmd=self.type),
cwd=target)
def sync(self, base):
@@ -89,7 +89,8 @@ class DarcsOverlay(OverlaySource):
else:
args = ['pull', '--all', self.src]
return self.postsync(
- self.run_command(self.command(), args, cwd=target, cmd=self.type),
+ run_command(self.config, self.command(), args, cwd=target,
+ cmd=self.type),
cwd=target)
def supported(self):
diff --git a/layman/overlays/g_common.py b/layman/overlays/g_common.py
index f3eb58d..9c5f6b7 100644
--- a/layman/overlays/g_common.py
+++ b/layman/overlays/g_common.py
@@ -25,7 +25,7 @@ from __future__ import unicode_literals
#-------------------------------------------------------------------------------
import os
-from layman.utils import path
+from layman.utils import path, run_command
from layman.overlays.source import OverlaySource, require_supported
#===============================================================================
@@ -69,12 +69,14 @@ class GCommonOverlay(OverlaySource):
target = path([base, self.parent.name])
args = [target, 'sync', self.driver, self.remote_uri]
- returncode = self.run_command(self.command(), args, cwd=target)
+ returncode = run_command(self.config, self.command(), args,
+ cwd=target)
if returncode:
return returncode
args = [target, 'generate-tree']
return self.postsync(
- self.run_command(self.command(), args, cwd=target, cmd=self.type),
+ run_command(self.config, self.command(), args, cwd=target,
+ cmd=self.type),
cwd=target)
def supported(self):
diff --git a/layman/overlays/g_sorcery.py b/layman/overlays/g_sorcery.py
index 19bfe18..cc84aeb 100644
--- a/layman/overlays/g_sorcery.py
+++ b/layman/overlays/g_sorcery.py
@@ -26,7 +26,7 @@ from __future__ import unicode_literals
#-------------------------------------------------------------------------------
import os
-from layman.utils import path
+from layman.utils import path, run_command
from layman.overlays.source import OverlaySource, require_supported
#===============================================================================
@@ -70,12 +70,14 @@ class GSorceryOverlay(OverlaySource):
target = path([base, self.parent.name])
args = [self.backend, '-o', target, '-r', self.repository, 'sync']
- returncode = self.run_command(self.command(), args, cwd=target)
+ returncode = run_command(self.config, self.command(), args,
+ cwd=target)
if returncode:
return returncode
args = [self.backend, '-o', target, 'generate-tree']
return self.postsync(
- self.run_command(self.command(), args, cwd=target, cmd=self.type),
+ run_command(self.config, self.command(), args, cwd=target,
+ cmd=self.type),
cwd=target)
def supported(self):
diff --git a/layman/overlays/git.py b/layman/overlays/git.py
index f3ada66..f647139 100644
--- a/layman/overlays/git.py
+++ b/layman/overlays/git.py
@@ -28,7 +28,7 @@ __version__ = "$Id: git.py 146 2006-05-27 09:52:36Z wrobel $"
import xml.etree.ElementTree as ET
-from layman.utils import path
+from layman.utils import path, run_command
from layman.overlays.source import OverlaySource, require_supported
#===============================================================================
@@ -86,7 +86,8 @@ class GitOverlay(OverlaySource):
# adding cwd=base due to a new git bug in selinux due to
# not having user_home_dir_t and portage_fetch_t permissions
# but changing dir works around it.
- success = self.run_command(self.command(), args, cmd=self.type, cwd=base)
+ success = run_command(self.config, self.command(), args,cmd=self.type,
+ cwd=base)
self.output.debug("cloned git repo...success=%s" % str(success), 8)
success = self.set_user(target)
return self.postsync(success, cwd=target)
@@ -97,13 +98,14 @@ class GitOverlay(OverlaySource):
email = '"%s"' % self.config['git_email']
args = ['config', 'user.name', user]
self.output.debug("set git user info...args=%s" % ' '.join(args), 8)
- failure = self.run_command(self.command(), args, cmd=self.type, cwd=target)
+ failure = run_command(self.config, self.command(), args, cmd=self.type, cwd=target)
if failure:
self.output.debug("set git user info...failure setting name")
return failure
args = ['config', 'user.email', email]
self.output.debug("set git user info...args=%s" % ' '.join(args), 8)
- return self.run_command(self.command(), args, cmd=self.type, cwd=target)
+ return run_command(self.config, self.command(), args, cmd=self.type,
+ cwd=target)
def update(self, base, src):
'''
@@ -118,7 +120,8 @@ class GitOverlay(OverlaySource):
# git remote set-url <name> <newurl> <oldurl>
args = ['remote', 'set-url', 'origin', self._fix_git_source(src), self._fix_git_source(self.src)]
- return self.run_command(self.command(), args, cmd=self.type, cwd=target)
+ return run_command(self.config, self.command(), args, cmd=self.type,
+ cwd=target)
def sync(self, base):
'''Sync overlay.'''
@@ -137,7 +140,8 @@ class GitOverlay(OverlaySource):
args.append(cfg_opts)
return self.postsync(
- self.run_command(self.command(), args, cwd=target, cmd=self.type),
+ run_command(self.config, self.command(), args, cwd=target,
+ cmd=self.type),
cwd=target)
def supported(self):
diff --git a/layman/overlays/mercurial.py b/layman/overlays/mercurial.py
index a1b7846..fa32555 100644
--- a/layman/overlays/mercurial.py
+++ b/layman/overlays/mercurial.py
@@ -30,7 +30,7 @@ __version__ = "$Id: mercurial.py 236 2006-09-05 20:39:37Z wrobel $"
import re
import xml.etree.ElementTree as ET
-from layman.utils import path
+from layman.utils import path, run_command
from layman.overlays.source import OverlaySource, require_supported
#===============================================================================
@@ -85,7 +85,7 @@ class MercurialOverlay(OverlaySource):
args.append(self.branch)
return self.postsync(
- self.run_command(self.command(), args, cmd=self.type),
+ run_command(self.config, self.command(), args, cmd=self.type),
cwd=target)
def update(self, base, src):
@@ -110,7 +110,7 @@ class MercurialOverlay(OverlaySource):
args = ['-i', expression, hgrc]
# Run sed.
- return self.run_command('sed', args, cmd='sed', cwd=target)
+ return run_command(self.config, 'sed', args, cmd='sed', cwd=target)
def sync(self, base):
'''Sync overlay.'''
@@ -128,7 +128,8 @@ class MercurialOverlay(OverlaySource):
args = ['pull', '-u', self.src]
return self.postsync(
- self.run_command(self.command(), args, cwd=target, cmd=self.type),
+ run_command(self.config, self.command(), args, cwd=target,
+ cmd=self.type),
cwd=target)
def supported(self):
diff --git a/layman/overlays/rsync.py b/layman/overlays/rsync.py
index 407a09e..f540b76 100644
--- a/layman/overlays/rsync.py
+++ b/layman/overlays/rsync.py
@@ -26,7 +26,7 @@ __version__ = "$Id: rsync.py 236 2006-09-05 20:39:37Z wrobel $"
#
#-------------------------------------------------------------------------------
-from layman.utils import path
+from layman.utils import path, run_command
from layman.overlays.source import OverlaySource, require_supported
#===============================================================================
@@ -80,7 +80,7 @@ class RsyncOverlay(OverlaySource):
args.append(target)
return self.postsync(
- self.run_command(self.command(), args, cmd=self.type),
+ run_command(self.config, self.command(), args, cmd=self.type),
cwd=target)
def supported(self):
diff --git a/layman/overlays/source.py b/layman/overlays/source.py
index 7c9674b..baabad3 100644
--- a/layman/overlays/source.py
+++ b/layman/overlays/source.py
@@ -20,7 +20,7 @@ import copy
import sys
import shutil
import subprocess
-from layman.utils import path
+from layman.utils import path, resolve_command, run_command
supported_cache = {}
@@ -33,27 +33,10 @@ def _supported(key, check_supported=None):
supported_cache[key] = check_supported()
return supported_cache[key]
-def _resolve_command(command, _output):
- if os.path.isabs(command):
- if not os.path.exists(command):
- _output('Program "%s" not found' % command)
- return ('File', None)
- return ('File', command)
- else:
- kind = 'Command'
- env_path = os.environ['PATH']
- for d in env_path.split(os.pathsep):
- f = os.path.join(d, command)
- if os.path.exists(f):
- return ('Command', f)
- _output('Cound not resolve command ' +\
- '"%s" based on PATH "%s"' % (command, env_path))
- return ('Command', None)
-
def require_supported(binaries, _output):
for command, mtype, package in binaries:
- kind, path = _resolve_command(command, _output)
+ kind, path = resolve_command(command, _output)
if not path:
if _output:
_output(kind + ' ' + command + ' seems to be missing!'
@@ -134,73 +117,6 @@ class OverlaySource(object):
def command(self):
return self.config['%s_command' % self.__class__.type_key]
- def run_command(self, command, args, **kwargs):
- self.output.debug("OverlaySource.run_command(): " + command, 6)
- file_to_run = _resolve_command(command, self.output.error)[1]
- args = [file_to_run] + args
- assert('pwd' not in kwargs) # Bug detector
-
- self.output.debug("OverlaySource.run_command(): cleared 'assert'", 7)
- cwd = kwargs.get('cwd', None)
- env = None
- env_updates = None
- if 'env' in kwargs:
- # Build actual env from surrounding plus updates
- env_updates = kwargs['env']
- env = copy.copy(os.environ)
- env.update(env_updates)
-
- command_repr = ' '.join(args)
- if env_updates is not None:
- command_repr = '%s %s' % (' '.join('%s=%s' % (k, v) for (k, v)
- in sorted(env_updates.items())), command_repr)
- if cwd is not None:
- command_repr = '( cd %s && %s )' % (cwd, command_repr)
-
- cmd = kwargs.get('cmd', '')
- self.output.info('Running %s... # %s' % (cmd, command_repr), 2)
-
- if self.config['quiet']:
-
- input_source = subprocess.PIPE
- output_target = open('/dev/null', 'w')
- else:
- # Re-use parent file descriptors
- input_source = None
- output_target = None
-
- proc = subprocess.Popen(args,
- stdin=input_source,
- stdout=output_target,
- stderr=self.config['stderr'],
- cwd=cwd,
- env=env)
-
- if self.config['quiet']:
- # Make child non-interactive
- proc.stdin.close()
-
- try:
- result = proc.wait()
- except KeyboardInterrupt:
- self.output.info('Interrupted manually', 2)
- self.output.warn("Checking for cleanup actions to perform", 4)
- self.cleanup()
- result = 1
- except Exception as err:
- self.output.error(
- 'Unknown exception running command: %s' % command_repr)
- self.output.error('Original error was: %s' % str(err))
- result = 1
-
- if self.config['quiet']:
- output_target.close()
-
- if result:
- self.output.info('Failure result returned from %s' % cmd , 2)
-
- return result
-
def postsync(self, failed_sync, **kwargs):
"""Runs any repo specific postsync operations
"""
@@ -215,14 +131,9 @@ class OverlaySource(object):
kwargs.get('cwd', '')).split()
command = _opt[0]
args = _opt[1:]
- return self.run_command(command, args,
+ return run_command(self.config, command, args,
cmd='%s_postsync' % self.__class__.type_key)
return failed_sync
def to_xml_hook(self, repo_elem):
pass
-
- def cleanup(self):
- '''cleanup a failed/interrupted process
- overridden in subclass if it is needed.'''
- pass
diff --git a/layman/overlays/svn.py b/layman/overlays/svn.py
index c97195b..d5e598e 100644
--- a/layman/overlays/svn.py
+++ b/layman/overlays/svn.py
@@ -30,9 +30,8 @@ from subprocess import PIPE, Popen
#
#------------------------------------------------------------------------------
-from layman.utils import path
-from layman.overlays.source import (OverlaySource, require_supported,
- _resolve_command)
+from layman.utils import path, resolve_command, run_command
+from layman.overlays.source import (OverlaySource, require_supported)
#==============================================================================
#
@@ -87,7 +86,7 @@ class SvnOverlay(OverlaySource):
args.append(self.target)
return self.postsync(
- self.run_command(self.command(), args, cmd=self.type),
+ run_command(self.config, self.command(), args, cmd=self.type),
cwd=self.target)
def update(self, base, src):
@@ -105,7 +104,7 @@ class SvnOverlay(OverlaySource):
args = ['switch', '--relocate', self._fix_svn_source(self.src), self._fix_svn_source(src)]
return self.postsync(
- self.run_command(self.command(), args, cmd=self.type),
+ run_command(self.config, self.command(), args, cmd=self.type),
cwd=target)
@@ -139,7 +138,7 @@ class SvnOverlay(OverlaySource):
args.append(self.target)
return self.postsync(
- self.run_command(self.command(), args, cmd=self.type),
+ run_command(self.config, self.command(), args, cmd=self.type),
cwd=self.target)
def supported(self):
@@ -156,7 +155,8 @@ class SvnOverlay(OverlaySource):
self.output.warn("SVN: preparing to run cleanup()", 2)
args = ["cleanup"]
args.append(self.target)
- cleanup = self.run_command(self.command(), args, cmd="svn cleanup")
+ cleanup = run_command(self.config, self.command(), args,
+ cmd="svn cleanup")
return
def check_upgrade(self, target):
@@ -164,7 +164,7 @@ class SvnOverlay(OverlaySource):
than checking if it does need an upgrade if it is
actually needed.
'''
- file_to_run = _resolve_command(self.command(), self.output.error)[1]
+ file_to_run = resolve_command(self.command(), self.output.error)[1]
args = " ".join([file_to_run, " upgrade", target])
pipe = Popen(args, shell=True, stdout=PIPE, stderr=PIPE)
if pipe:
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 7d7bf89..bcba32c 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -31,6 +31,7 @@ import sys
from layman.constants import FILE_EXTENSIONS
from layman.overlays.archive import ArchiveOverlay
from layman.overlays.source import require_supported
+from layman.utils import run_command
#===============================================================================
#
@@ -78,7 +79,7 @@ class TarOverlay(ArchiveOverlay):
'''
# tar -v -x -f SOURCE -C TARGET
args = ['-v', '-x', '-f', pkg, '-C', dest_dir]
- result = self.run_command(self.command(), args, cmd=self.type)
+ result = run_command(self.config, self.command(), args, cmd=self.type)
return result
diff --git a/layman/utils.py b/layman/utils.py
index 33d2c30..b769302 100644
--- a/layman/utils.py
+++ b/layman/utils.py
@@ -31,10 +31,14 @@ __version__ = '$Id: utils.py 236 2006-09-05 20:39:37Z wrobel $'
#
#-------------------------------------------------------------------------------
-import types, re, os
-import sys
-import locale
import codecs
+import copy
+import locale
+import os
+import re
+import subprocess
+import sys
+import types
from layman.output import Message
@@ -184,6 +188,7 @@ def path(path_elements):
return pathname
+
def reload_config(config):
'''
Rereads the layman config.
@@ -196,6 +201,93 @@ def reload_config(config):
config.update_defaults({'config': defaults['config']})
config.read_config(defaults)
+
+def resolve_command(command, output):
+ if os.path.isabs(command):
+ if not os.path.exists(command):
+ output('Program "%s" not found' % command)
+ return ('File', None)
+ return ('File', command)
+ else:
+ kind = 'Command'
+ env_path = os.environ['PATH']
+ for d in env_path.split(os.pathsep):
+ f = os.path.join(d, command)
+ if os.path.exists(f):
+ return ('Command', f)
+ output('Cound not resolve command ' +\
+ '"%s" based on PATH "%s"' % (command, env_path))
+ return ('Command', None)
+
+
+def run_command(config, command, args, **kwargs):
+ output = config['output']
+ output.debug("Utils.run_command(): " + command, 6)
+
+ file_to_run = resolve_command(command, output.error)[1]
+ args = [file_to_run] + args
+ assert('pwd' not in kwargs) # Bug detector
+
+ output.debug("OverlaySource.run_command(): cleared 'assert'", 7)
+ cwd = kwargs.get('cwd', None)
+ env = None
+ env_updates = None
+ if 'env' in kwargs:
+ # Build actual env from surrounding plus updates
+ env_updates = kwargs['env']
+ env = copy.copy(os.environ)
+ env.update(env_updates)
+
+ command_repr = ' '.join(args)
+ if env_updates is not None:
+ command_repr = '%s %s' % (' '.join('%s=%s' % (k, v) for (k, v)
+ in sorted(env_updates.items())), command_repr)
+ if cwd is not None:
+ command_repr = '( cd %s && %s )' % (cwd, command_repr)
+
+ cmd = kwargs.get('cmd', '')
+ output.info('Running %s... # %s' % (cmd, command_repr), 2)
+
+ if config['quiet']:
+
+ input_source = subprocess.PIPE
+ output_target = open('/dev/null', 'w')
+ else:
+ # Re-use parent file descriptors
+ input_source = None
+ output_target = None
+
+ proc = subprocess.Popen(args,
+ stdin=input_source,
+ stdout=output_target,
+ stderr=config['stderr'],
+ cwd=cwd,
+ env=env)
+
+ if config['quiet']:
+ # Make child non-interactive
+ proc.stdin.close()
+
+ try:
+ result = proc.wait()
+ except KeyboardInterrupt:
+ output.info('Interrupted manually', 2)
+ result = 1
+ except Exception as err:
+ output.error(
+ 'Unknown exception running command: %s' % command_repr)
+ output.error('Original error was: %s' % str(err))
+ result = 1
+
+ if config['quiet']:
+ output_target.close()
+
+ if result:
+ output.info('Failure result returned from %s' % cmd , 2)
+
+ return result
+
+
def verify_overlay_src(current_src, remote_srcs):
'''
Verifies that the src-url of the overlay in
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-08-15 22:32 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-08-15 22:32 UTC (permalink / raw
To: gentoo-commits
commit: 691dc445082cc455159d699498a714d40754fedd
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 7 18:27:24 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 15 20:57:38 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=691dc445
Centralizes common archive overlay code
This commit brings archive overlay code to one common place, in
an attempt to make maintaining and adding archive overlay types
to layman much easier.
config.py: Alters the clean_tar option to clean_archive.
tar.py: Removes doctest tests from tar.py as there is already an
existing external.py test.
---
layman/config.py | 2 +-
layman/constants.py | 11 +++
layman/overlays/archive.py | 197 +++++++++++++++++++++++++++++++++++++++++++++
layman/overlays/tar.py | 179 ++++++----------------------------------
4 files changed, 235 insertions(+), 154 deletions(-)
diff --git a/layman/config.py b/layman/config.py
index 4849d5e..91c179e 100644
--- a/layman/config.py
+++ b/layman/config.py
@@ -100,7 +100,7 @@ class BareConfig(object):
'check_official': 'Yes',
'conf_type': 'make.conf',
'require_repoconfig': 'Yes',
- 'clean_tar': 'yes',
+ 'clean_archive': 'yes',
'make_conf' : '%(storage)s/make.conf',
'repos_conf': path([self.root, EPREFIX,'/etc/portage/repos.conf/layman.conf']),
'conf_module': ['make_conf', 'repos_conf'],
diff --git a/layman/constants.py b/layman/constants.py
index 24d1f36..c526cb6 100644
--- a/layman/constants.py
+++ b/layman/constants.py
@@ -66,3 +66,14 @@ SUCCEED = 0
COMPONENT_DEFAULTS = ['name', 'descriptions', 'owner', 'type', 'sources']
POSSIBLE_COMPONENTS = ['name', 'descriptions', 'homepage', 'owner', 'quality',
'priority', 'sources', 'branch', 'irc', 'feeds']
+
+
+###############################################################################
+##
+## Archive overlay possible file extensions
+##
+###############################################################################
+
+FILE_EXTENSIONS = {'Tar': ('bz2', 'gz', 'lzma', 'xz', 'Z', 'tgz', 'tbz', 'taz',
+ 'tlz', 'txz')
+ }
diff --git a/layman/overlays/archive.py b/layman/overlays/archive.py
new file mode 100644
index 0000000..68c8b47
--- /dev/null
+++ b/layman/overlays/archive.py
@@ -0,0 +1,197 @@
+#!/usr/bin/python
+from __future__ import unicode_literals
+
+import os
+import sys
+import shutil
+import tempfile
+
+import xml.etree.ElementTree as ET # Python 2.5
+
+from layman.constants import MOUNT_TYPES
+from layman.compatibility import fileopen
+from layman.overlays.source import OverlaySource, require_supported
+from layman.utils import path
+from layman.version import VERSION
+from sslfetch.connections import Connector
+
+USERAGENT = "Layman-" + VERSION
+
+class ArchiveOverlay(OverlaySource):
+
+ type = 'Archive'
+ type_key = 'archive'
+
+ def __init__(self, parent, config, _location, ignore = 0):
+
+ super(ArchiveOverlay, self).__init__(parent,
+ config, _location, ignore)
+
+ self.clean_archive = config['clean_archive']
+ self.output = config['output']
+ self.proxies = config.proxies
+ self.branch = self.parent.branch
+ self.mount_me = bool(self.type in MOUNT_TYPES)
+
+
+ def _fetch(self, base, archive_url, dest_dir):
+ '''
+ Fetches overlay source archive.
+
+ @params base: string of directory base for installed overlays.
+ @params archive_url: string of URL where archive is located.
+ @params dest_dir: string of destination of extracted archive.
+ @rtype tuple (str of package location, bool to clean_archive)
+ '''
+ ext = self.get_extension()
+
+ if 'file://' not in archive_url:
+ # set up ssl-fetch output map
+ connector_output = {
+ 'info': self.output.debug,
+ 'error': self.output.error,
+ 'kwargs-info': {'level': 2},
+ 'kwargs-error': {'level': None},
+ }
+
+ fetcher = Connector(connector_output, self.proxies, USERAGENT)
+
+ success, archive, timestamp = fetcher.fetch_content(archive_url)
+
+ pkg = path([base, self.parent.name + ext])
+
+ try:
+ with fileopen(pkg, 'w+b') as out_file:
+ out_file.write(archive)
+
+ except Exception as error:
+ raise Exception('Failed to store archive package in '\
+ '%(pkg)s\nError was: %(error)s'\
+ % ({'pkg': pkg, 'error': error}))
+
+ else:
+ self.clean_archive = False
+ pkg = archive_url.replace('file://', '')
+
+ return pkg
+
+
+ def _add_unchecked(self, base):
+ def try_to_wipe(folder):
+ if not os.path.exists(folder):
+ return
+
+ try:
+ self.output.info('Deleting directory %(dir)s'\
+ % ({'dir': folder}), 2)
+ shutil.rmtree(folder)
+ except Exception as error:
+ raise Exception('Failed to remove unneccessary archive '\
+ 'structure %(dir)s\nError was: %(err)s'\
+ % ({'dir': folder, 'err': error}))
+
+ final_path = path([base, self.parent.name])
+ try:
+ if not self.mount_me:
+ temp_path = tempfile.mkdtemp(dir=base)
+ else:
+ temp_path = final_path
+ if not os.path.exists(temp_path):
+ os.mkdir(temp_path)
+ pkg = self._fetch(base=base, archive_url=self.src,
+ dest_dir=temp_path)
+ result = self.post_fetch(pkg, temp_path)
+ if self.clean_archive:
+ os.unlink(pkg)
+ except Exception as error:
+ try_to_wipe(temp_path)
+ raise error
+
+ if result == 0 and not self.mount_me:
+ if self.branch:
+ source = temp_path + os.path.sep + self.branch
+ else:
+ source = temp_path
+
+ if os.path.exists(source):
+ if os.path.exists(final_path):
+ self.delete(base)
+
+ try:
+ os.rename(source, final_path)
+ except Exception as error:
+ raise Exception('Failed to rename archive subdirectory '\
+ '%(src)s to %(path)s\nError was: %(err)s'\
+ % ({'src': source, 'path': final_path, 'err': error}))
+ os.chmod(final_path, 0o755)
+ else:
+ raise Exception('The given path (branch setting in the xml)\n'\
+ ' %(src)s does not exist in this archive package!'\
+ % ({'src': source}))
+
+ if not self.mount_me:
+ try_to_wipe(temp_path)
+
+ return result
+
+
+ def add(self, base):
+ '''
+ Add overlay.
+
+ @params base: string location where overlays are installed.
+ @rtype bool
+ '''
+
+ if not self.supported():
+ return 1
+
+ target = path([base, self.parent.name])
+
+ if os.path.exists(target):
+ raise Exception('Directory %(dir)s already exists. Will not '\
+ 'overwrite its contents!' % ({'dir': target}))
+
+ return self.postsync(
+ self._add_unchecked(base),
+ cwd=target)
+
+
+ def sync(self, base):
+ '''
+ Sync overlay.
+
+ @params base: string location where overlays are installed.
+ @rtype bool
+ '''
+
+ if not self.supported():
+ return 1
+
+ target = path([base, self.parent.name])
+
+ return self.postsync(
+ self._add_unchecked(base),
+ cwd=target)
+
+
+ def supported(self):
+ '''
+ Determines if overlay type is supported.
+
+ @rtype bool
+ '''
+
+ return self.is_supported()
+
+
+if __name__ == '__main__':
+ import doctest
+
+ # Ignore warnings here. We are just testing.
+ from warnings import filterwarnings, resetwarnings
+ filterwarnings('ignore')
+
+ doctest.testmod(sys.modules[__name__])
+
+ resetwarnings()
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 4999e20..7d7bf89 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -26,21 +26,11 @@ __version__ = "$Id: tar.py 310 2007-04-09 16:30:40Z wrobel $"
#
#-------------------------------------------------------------------------------
-import os
-import os.path
import sys
-import shutil
-import tempfile
-import xml.etree.ElementTree as ET # Python 2.5
-
-from layman.compatibility import fileopen
-from layman.overlays.source import OverlaySource, require_supported
-from layman.utils import path
-from layman.version import VERSION
-from sslfetch.connections import Connector
-
-USERAGENT = "Layman" + VERSION
+from layman.constants import FILE_EXTENSIONS
+from layman.overlays.archive import ArchiveOverlay
+from layman.overlays.source import require_supported
#===============================================================================
#
@@ -48,174 +38,57 @@ USERAGENT = "Layman" + VERSION
#
#-------------------------------------------------------------------------------
-class TarOverlay(OverlaySource):
- ''' Handles tar overlays.
-
- >>> from layman.output import Message
- >>> import xml.etree.ElementTree as ET # Python 2.5
- >>> repo = ET.Element('repo')
- >>> repo_name = ET.Element('name')
- >>> repo_name.text = 'dummy'
- >>> desc = ET.Element('description')
- >>> desc.text = 'Dummy description'
- >>> owner = ET.Element('owner')
- >>> owner_email = ET.Element('email')
- >>> owner_email.text = 'dummy@example.org'
- >>> owner[:] = [owner_email]
- >>> source = ET.Element('source', type='tar')
- >>> here = os.path.dirname(os.path.realpath(__file__))
- >>> source.text = 'file://' + here + '/../tests/testfiles/layman-test.tar.bz2'
- >>> branch = ET.Element('branch')
- >>> branch.text = 'layman-test'
- >>> repo[:] = [repo_name, desc, owner, source, branch]
- >>> from layman.config import BareConfig
- >>> config = BareConfig()
- >>> import tempfile
- >>> testdir = tempfile.mkdtemp(prefix="laymantmp_")
- >>> from layman.overlays.overlay import Overlay
- >>> a = Overlay(config, repo)
- >>> config['output'].set_colorize(False)
- >>> a.add(testdir)
- 0
- >>> os.listdir(testdir + '/dummy/')
- ['layman-test']
- >>> sorted(os.listdir(testdir + '/dummy/layman-test/'))
- ['app-admin', 'app-portage']
- >>> shutil.rmtree(testdir)
- '''
+class TarOverlay(ArchiveOverlay):
+ '''Handles tar overlays.'''
type = 'Tar'
type_key = 'tar'
- def __init__(self, parent, config, _location, ignore = 0):
+ def __init__(self, parent, config, _location, ignore=0):
super(TarOverlay, self).__init__(parent,
config, _location, ignore)
self.output = config['output']
- self.proxies = config.proxies
- self.branch = self.parent.branch
- def _extract(self, base, tar_url, dest_dir):
+ def get_extension(self):
+ '''
+ Determines tar file extension.
+
+ @rtype str
+ '''
ext = '.tar.noidea'
- clean_tar = self.config['clean_tar']
- for i in [('tar.%s' % e) for e in ('bz2', 'gz', 'lzma', 'xz', 'Z')] \
- + ['tgz', 'tbz', 'taz', 'tlz', 'txz']:
+ for i in [('tar.%s' % e) for e in FILE_EXTENSIONS[self.type]:
candidate_ext = '.%s' % i
if self.src.endswith(candidate_ext):
ext = candidate_ext
break
- if 'file://' not in tar_url:
- # setup the ssl-fetch output map
- connector_output = {
- 'info': self.output.debug,
- 'error': self.output.error,
- 'kwargs-info': {'level': 2},
- 'kwargs-error':{'level': None},
- }
-
- fetcher = Connector(connector_output, self.proxies, USERAGENT)
+ return ext
- success, tar, timestamp = fetcher.fetch_content(tar_url)
- pkg = path([base, self.parent.name + ext])
-
- try:
- with fileopen(pkg, 'w+b') as out_file:
- out_file.write(tar)
-
- except Exception as error:
- raise Exception('Failed to store tar package in '
- + pkg + '\nError was:' + str(error))
- else:
- clean_tar = False
- pkg = tar_url.replace('file://', '')
+ def post_fetch(self, pkg, dest_dir):
+ '''
+ Extracts tar archive.
+ @params pkg: string location where tar archive is located.
+ @params dest_dir: string of destination of extracted archive.
+ @rtype bool
+ '''
# tar -v -x -f SOURCE -C TARGET
args = ['-v', '-x', '-f', pkg, '-C', dest_dir]
result = self.run_command(self.command(), args, cmd=self.type)
- if clean_tar:
- os.unlink(pkg)
- return result
-
- def _add_unchecked(self, base):
- def try_to_wipe(folder):
- if not os.path.exists(folder):
- return
-
- try:
- self.output.info('Deleting directory "%s"' % folder, 2)
- shutil.rmtree(folder)
- except Exception as error:
- raise Exception('Failed to remove unnecessary tar structure "'
- + folder + '"\nError was:' + str(error))
-
- final_path = path([base, self.parent.name])
- temp_path = tempfile.mkdtemp(dir=base)
- try:
- result = self._extract(base=base, tar_url=self.src,
- dest_dir=temp_path)
- except Exception as error:
- try_to_wipe(temp_path)
- raise error
-
- if result == 0:
- if self.branch:
- source = temp_path + '/' + self.branch
- else:
- source = temp_path
-
- if os.path.exists(source):
- if os.path.exists(final_path):
- self.delete(base)
-
- try:
- os.rename(source, final_path)
- except Exception as error:
- raise Exception('Failed to rename tar subdirectory ' +
- source + ' to ' + final_path +
- '\nError was:' + str(error))
- os.chmod(final_path, 0o755)
- else:
- raise Exception('The given path (branch setting in the xml)\n' + \
- '"%(source)s" does not exist in the tar package!' % ({'source': source}))
-
- try_to_wipe(temp_path)
return result
- def add(self, base):
- '''Add overlay.'''
-
- if not self.supported():
- return 1
-
- target = path([base, self.parent.name])
-
- if os.path.exists(target):
- raise Exception('Directory ' + target + ' already exists.' +\
- ' Will not overwrite its contents!')
-
- return self.postsync(
- self._add_unchecked(base),
- cwd=target)
-
- def sync(self, base):
- '''Sync overlay.'''
-
- if not self.supported():
- return 1
-
- target = path([base, self.parent.name])
- return self.postsync(
- self._add_unchecked(base),
- cwd=target)
+ def is_supported(self):
+ '''
+ Determines if overlay type is supported.
- def supported(self):
- '''Overlay type supported?'''
+ @rtype bool
+ '''
return require_supported(
[(self.command(), 'tar', 'app-arch/tar'), ],
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/
@ 2014-08-15 22:32 Devan Franchini
0 siblings, 0 replies; 20+ messages in thread
From: Devan Franchini @ 2014-08-15 22:32 UTC (permalink / raw
To: gentoo-commits
commit: a4cacd7fec67b377ca7ce882c3257326e0259d2b
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 13 23:24:01 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 15 21:42:41 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=a4cacd7f
Adds Squashfs overlay support
constants.py: Adds Squashfs overlay type to MOUNT_TYPES constants,
as well as different possible file extensions to FILE_EXTENSIONS
constants.
---
layman/constants.py | 7 ++-
layman/overlays/squashfs.py | 148 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 152 insertions(+), 3 deletions(-)
diff --git a/layman/constants.py b/layman/constants.py
index 7379429..787b610 100644
--- a/layman/constants.py
+++ b/layman/constants.py
@@ -73,8 +73,9 @@ POSSIBLE_COMPONENTS = ['name', 'descriptions', 'homepage', 'owner', 'quality',
##
###############################################################################
-FILE_EXTENSIONS = {'Tar': ('bz2', 'gz', 'lzma', 'xz', 'Z', 'tgz', 'tbz', 'taz',
- 'tlz', 'txz')
+FILE_EXTENSIONS = {'Squashfs': ('.squashfs', '.squash', '.sqfs', '.sfs'),
+ 'Tar': ('bz2', 'gz', 'lzma', 'xz', 'Z', 'tgz', 'tbz', 'taz',
+ 'tlz', 'txz'),
}
###################################################################################
@@ -83,4 +84,4 @@ FILE_EXTENSIONS = {'Tar': ('bz2', 'gz', 'lzma', 'xz', 'Z', 'tgz', 'tbz', 'taz',
##
####################################################################################
-MOUNT_TYPES = []
+MOUNT_TYPES = ['Squashfs']
diff --git a/layman/overlays/squashfs.py b/layman/overlays/squashfs.py
new file mode 100644
index 0000000..48711ce
--- /dev/null
+++ b/layman/overlays/squashfs.py
@@ -0,0 +1,148 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+################################################################################
+# LAYMAN SQUASHFS OVERLAY HANDLER
+################################################################################
+# File: squashfs.py
+#
+# Handles squashfs overlays
+#
+# Copyright:
+# (c) 2014 Devan Franchini
+# Distributed under the terms of the GNU General Public License v2
+#
+# Author(s):
+# Devan Franchini <twitch153@gentoo.org>
+#
+''' Squashfs overlay support.'''
+
+from __future__ import unicode_literals
+
+#===============================================================================
+#
+# Dependencies
+#
+#-------------------------------------------------------------------------------
+
+import os
+import shutil
+import sys
+
+from layman.constants import FILE_EXTENSIONS
+from layman.overlays.archive import ArchiveOverlay
+from layman.overlays.source import require_supported
+from layman.utils import path
+
+#===============================================================================
+#
+# Class SquashfsOverlay
+#
+#-------------------------------------------------------------------------------
+
+class SquashfsOverlay(ArchiveOverlay):
+ ''' Handles squashfs overlays.'''
+
+ type = 'Squashfs'
+ type_key = 'squashfs'
+
+ def __init__(self, parent, config, _location, ignore=0):
+ super(SquashfsOverlay, self).__init__(parent,
+ config, _location, ignore)
+ self.mounter = config.get_option('mounts')
+
+
+ def get_extension(self):
+ '''
+ Determines squashfs file extension.
+
+ @rtype str
+ '''
+ ext = ''
+ for i in FILE_EXTENSIONS[self.type]:
+ candidate_ext = i
+ if self.src.endswith(candidate_ext):
+ ext = candidate_ext
+ break
+
+ return ext
+
+
+ def delete(self, base):
+ '''
+ Deletes the selected overlay.
+
+ @params base: Base dir where the overlay is installed.
+ @rtype bool
+ '''
+ mdir = path([base, self.parent.name])
+
+ source = self.src
+ if 'file://' in source:
+ pkg = source.replace('file://', '')
+ else:
+ pkg_name = self.parent.name + self.get_extension()
+ pkg = path([self.config['storage'], pkg_name])
+
+ if os.path.ismount(mdir):
+ result = self.mounter.umount([self.parent.name])
+ else:
+ result = 1
+
+ shutil.rmtree(mdir)
+ if self.clean_archive:
+ if os.path.exists(pkg):
+ os.unlink(pkg)
+
+ return result
+
+
+ def post_fetch(self, pkg, dest_dir):
+ '''
+ Runs initial squashfs archive mounting.
+
+ @params pkg: string location where squashfs archive is located.
+ @params dest_dir: string of mounting destination.
+ @rtype bool
+ '''
+ result = self.mounter.mount([self.parent.name],
+ dest_dir,
+ install=True,
+ ovl_type=self.type,
+ pkg=pkg)
+ # Warn the user that pkg is mounted r/o and will have to be mounted
+ # again using the layman-mounter tool on the next boot.
+ if result == 0:
+ warning_1 = 'Overlay "%(ovl)s" has been mounted read-only at'\
+ ' %(loc)s.' % {'loc': dest_dir,
+ 'ovl': self.parent.name}
+ warning_2 = 'Upon reboot the overlay will not be accessible until'\
+ ' you mount it either manually or via the'\
+ ' layman-mounter tool.'
+
+ self.output.warn(warning_1)
+ self.output.warn(warning_2)
+
+ return result
+
+
+ def is_supported(self):
+ '''
+ Determines if the overlay type is supported.
+
+ @rtype bool
+ '''
+ return require_supported(
+ [(self.command(), 'squashfs', 'sys-fs/squashfs-tools'), ],
+ self.output.warn)
+
+
+if __name__ == '__main__':
+ import doctest
+
+ # Ignore warnings here. We are just testing
+ from warnings import filterwarnings, resetwarnings
+ filterwarnings('ignore')
+
+ doctest.testmod(sys.modules[__name__])
+
+ resetwarnings()
^ permalink raw reply related [flat|nested] 20+ messages in thread
end of thread, other threads:[~2014-08-15 22:32 UTC | newest]
Thread overview: 20+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2014-05-15 20:28 [gentoo-commits] proj/layman:gsoc2014 commit in: layman/overlays/, layman/ Devan Franchini
-- strict thread matches above, loose matches on Subject: below --
2014-08-15 22:32 Devan Franchini
2014-08-15 22:32 Devan Franchini
2014-08-15 22:32 Devan Franchini
2014-06-16 3:40 [gentoo-commits] proj/layman:master " Brian Dolbec
2014-06-16 3:37 ` [gentoo-commits] proj/layman:gsoc2014 " Brian Dolbec
2014-06-16 3:40 [gentoo-commits] proj/layman:master " Brian Dolbec
2014-06-16 3:37 ` [gentoo-commits] proj/layman:gsoc2014 " Brian Dolbec
2014-06-16 3:37 Brian Dolbec
2014-05-16 1:07 Devan Franchini
2014-05-16 1:07 Devan Franchini
2014-05-16 0:58 Devan Franchini
2014-05-15 20:46 Devan Franchini
2014-05-15 20:37 Devan Franchini
2014-05-14 23:49 Devan Franchini
2014-05-14 23:49 Devan Franchini
2014-05-14 23:49 Devan Franchini
2014-05-14 23:36 Devan Franchini
2014-05-14 23:10 Devan Franchini
2014-05-14 17:32 Devan Franchini
2014-05-14 17:32 Devan Franchini
2014-05-14 4:01 Devan Franchini
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox