* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-12 7:46 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-12 7:46 UTC (permalink / raw
To: gentoo-commits
commit: 2497e6df850f23e7ae17f01c436a3ee1a2883c31
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 12 02:01:23 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Nov 12 07:23:08 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=2497e6df
move output classes to thier own file.
---
mirrorselect/main.py | 110 +++-----------------------------------
mirrorselect/output.py | 138 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 145 insertions(+), 103 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 1d01e6d..0f624b7 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -37,108 +37,13 @@ import time
import urllib
import urlparse
import hashlib
-from optparse import IndentedHelpFormatter, OptionParser
-from mirrorselect.mirrorparser3 import MirrorParser3, MIRRORS_3_XML, MIRRORS_RSYNC_DATA
-import codecs
-
-class Output(object):
- """Handles text output. Only prints messages with level <= verbosity.
- Therefore, verbosity=2 is everything (debug), and verbosity=0 is urgent
- messages only (quiet)."""
-
- def __init__(self, verbosity=1, out=sys.stderr):
- esc_seq = "\x1b["
- codes = {}
-
- codes["reset"] = esc_seq + "39;49;00m"
- codes["bold"] = esc_seq + "01m"
- codes["blue"] = esc_seq + "34;01m"
- codes["green"] = esc_seq + "32;01m"
- codes["yellow"] = esc_seq + "33;01m"
- codes["red"] = esc_seq + "31;01m"
-
- self.codes = codes
- del codes
-
- self.verbosity = verbosity
- self.file = out
-
- def red(self, text):
- return self.codes["red"]+text+self.codes["reset"]
-
- def green(self, text):
- return self.codes["green"]+text+self.codes["reset"]
-
- def white(self, text):
- return self.codes["bold"]+text+self.codes["reset"]
-
- def blue(self, text):
- return self.codes["blue"]+text+self.codes["reset"]
-
- def yellow(self, text):
- return self.codes["yellow"]+text+self.codes["reset"]
+from optparse import OptionParser
+from mirrorselect.mirrorparser3 import (MirrorParser3, MIRRORS_3_XML,
+ MIRRORS_RSYNC_DATA)
+from mirrorselect.output import output, ColoredFormatter
- def print_info(self, message, level=1):
- """Prints an info message with a green star, like einfo."""
- if level <= self.verbosity:
- self.file.write('\r' + self.green('* ') + message)
- self.file.flush()
-
- def print_warn(self, message, level=1):
- """Prints a warning."""
- if level <= self.verbosity:
- self.file.write(self.yellow('Warning: ') + message)
- self.file.flush()
-
- def print_err(self, message, level=0):
- """prints an error message with a big red ERROR."""
- if level <= self.verbosity:
- self.file.write(self.red('\nERROR: ') + message + '\n')
- self.file.flush()
- sys.exit(1)
-
- def write(self, message, level=1):
- """A wrapper arounf stderr.write, to enforce verbosity settings."""
- if level <= self.verbosity:
- self.file.write(message)
- self.file.flush()
-
-
-class ColoredFormatter(IndentedHelpFormatter):
-
- """HelpFormatter with colorful output.
-
- Extends format_option.
- Overrides format_heading.
- """
+import codecs
- def format_heading(self, heading):
- """Return a colorful heading."""
- return "%*s%s:\n" % (self.current_indent, "", output.white(heading))
-
- def format_option(self, option):
- """Return colorful formatted help for an option."""
- option = IndentedHelpFormatter.format_option(self, option)
- # long options with args
- option = re.sub(
- r"--([a-zA-Z]*)=([a-zA-Z]*)",
- lambda m: "-%s %s" % (output.green(m.group(1)),
- output.blue(m.group(2))),
- option)
- # short options with args
- option = re.sub(
- r"-([a-zA-Z]) ?([0-9A-Z]+)",
- lambda m: " -" + output.green(m.group(1)) + ' ' + output.blue(m.group(2)),
- option)
- # options without args
- option = re.sub(
- r"-([a-zA-Z\d]+)", lambda m: "-" + output.green(m.group(1)),
- option)
- return option
-
- def format_description(self, description):
- """Do not wrap."""
- return description + '\n'
class Extractor(object):
@@ -696,7 +601,7 @@ def get_filesystem_mirrors(out, path, sync=False):
try:
f = open(path,'r')
- except IOError, e:
+ except IOError:
return fsmirrors
""" Search for 'var' in make.conf and extract value """
@@ -727,7 +632,7 @@ def get_filesystem_mirrors(out, path, sync=False):
break
elif key is None:
break
- except Exception, e:
+ except Exception:
fsmirrors = []
return fsmirrors
@@ -858,7 +763,6 @@ def parse_args(argv, config_path):
return options
-output = Output() #the only FUCKING global. Damnit.
def main(argv):
"""Lets Rock!"""
# start with the new location
diff --git a/mirrorselect/output.py b/mirrorselect/output.py
new file mode 100644
index 0000000..b00226a
--- /dev/null
+++ b/mirrorselect/output.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+#-*- coding:utf-8 -*-
+
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2005-2012 Gentoo Foundation
+
+ Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
+ Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
+ Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
+
+from __future__ import print_function
+
+import sys
+import re
+
+from optparse import IndentedHelpFormatter
+
+
+class Output(object):
+ """Handles text output. Only prints messages with level <= verbosity.
+ Therefore, verbosity=2 is everything (debug), and verbosity=0 is urgent
+ messages only (quiet)."""
+
+ def __init__(self, verbosity=1, out=sys.stderr):
+ esc_seq = "\x1b["
+ codes = {}
+
+ codes["reset"] = esc_seq + "39;49;00m"
+ codes["bold"] = esc_seq + "01m"
+ codes["blue"] = esc_seq + "34;01m"
+ codes["green"] = esc_seq + "32;01m"
+ codes["yellow"] = esc_seq + "33;01m"
+ codes["red"] = esc_seq + "31;01m"
+
+ self.codes = codes
+ del codes
+
+ self.verbosity = verbosity
+ self.file = out
+
+ def red(self, text):
+ return self.codes["red"]+text+self.codes["reset"]
+
+ def green(self, text):
+ return self.codes["green"]+text+self.codes["reset"]
+
+ def white(self, text):
+ return self.codes["bold"]+text+self.codes["reset"]
+
+ def blue(self, text):
+ return self.codes["blue"]+text+self.codes["reset"]
+
+ def yellow(self, text):
+ return self.codes["yellow"]+text+self.codes["reset"]
+
+ def print_info(self, message, level=1):
+ """Prints an info message with a green star, like einfo."""
+ if level <= self.verbosity:
+ self.file.write('\r' + self.green('* ') + message)
+ self.file.flush()
+
+ def print_warn(self, message, level=1):
+ """Prints a warning."""
+ if level <= self.verbosity:
+ self.file.write(self.yellow('Warning: ') + message)
+ self.file.flush()
+
+ def print_err(self, message, level=0):
+ """prints an error message with a big red ERROR."""
+ if level <= self.verbosity:
+ self.file.write(self.red('\nERROR: ') + message + '\n')
+ self.file.flush()
+ sys.exit(1)
+
+ def write(self, message, level=1):
+ """A wrapper arounf stderr.write, to enforce verbosity settings."""
+ if level <= self.verbosity:
+ self.file.write(message)
+ self.file.flush()
+
+
+class ColoredFormatter(IndentedHelpFormatter):
+
+ """HelpFormatter with colorful output.
+
+ Extends format_option.
+ Overrides format_heading.
+ """
+
+ def format_heading(self, heading):
+ """Return a colorful heading."""
+ return "%*s%s:\n" % (self.current_indent, "", output.white(heading))
+
+ def format_option(self, option):
+ """Return colorful formatted help for an option."""
+ option = IndentedHelpFormatter.format_option(self, option)
+ # long options with args
+ option = re.sub(
+ r"--([a-zA-Z]*)=([a-zA-Z]*)",
+ lambda m: "-%s %s" % (output.green(m.group(1)),
+ output.blue(m.group(2))),
+ option)
+ # short options with args
+ option = re.sub(
+ r"-([a-zA-Z]) ?([0-9A-Z]+)",
+ lambda m: " -" + output.green(m.group(1)) + ' ' + output.blue(m.group(2)),
+ option)
+ # options without args
+ option = re.sub(
+ r"-([a-zA-Z\d]+)", lambda m: "-" + output.green(m.group(1)),
+ option)
+ return option
+
+ def format_description(self, description):
+ """Do not wrap."""
+ return description + '\n'
+
+
+output = Output() #the only FUCKING global. Damnit.
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-12 7:46 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-12 7:46 UTC (permalink / raw
To: gentoo-commits
commit: 44374d85bc357d91d69d1ca4c296947f76b46990
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 12 02:22:13 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Nov 12 07:23:08 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=44374d85
move the selector classes to their own file
---
mirrorselect/main.py | 498 +-------------------------------
mirrorselect/{main.py => selectors.py} | 335 ++-------------------
2 files changed, 32 insertions(+), 801 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 0f624b7..25c73ae 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -23,507 +23,17 @@
__revision__ = '2.1.0'
-import math
import os
import re
import shlex
import shutil
-import signal
import socket
import string
-import subprocess
import sys
-import time
-import urllib
-import urlparse
-import hashlib
from optparse import OptionParser
-from mirrorselect.mirrorparser3 import (MirrorParser3, MIRRORS_3_XML,
- MIRRORS_RSYNC_DATA)
+from mirrorselect.mirrorparser3 import MIRRORS_3_XML, MIRRORS_RSYNC_DATA
from mirrorselect.output import output, ColoredFormatter
-
-import codecs
-
-
-
-class Extractor(object):
- """The Extractor employs a MirrorParser3 object to get a list of valid
- mirrors, and then filters them. Only the mirrors that should be tested, based on
- user input are saved. They will be in the hosts attribute."""
-
- def __init__(self, list_url, options):
- parser = MirrorParser3()
- self.hosts = []
-
- hosts = self.getlist(parser, list_url)
- output.write('Extractor(): fetched mirrors.xml,'
- ' %s hosts before filtering\n' % len(hosts), 2)
-
- if not options.rsync:
- if options.ftp:
- hosts = self.restrict_protocall('ftp', hosts)
- if options.http:
- hosts = self.restrict_protocall('http', hosts)
-
- self.hosts = hosts
-
- def restrict_protocall(self, prot, hosts):
- """
- Removes hosts that are not of the specified type.
- "prot" must always be exactly 'http' or 'ftp'.
- """
- myhosts = []
-
- output.print_info('Limiting test to %s hosts. ' % prot )
-
- for host in hosts:
- if host[0].startswith(prot):
- myhosts.append(host)
-
- output.write('%s of %s removed.\n' % (len(hosts) - len(myhosts),
- len(hosts)) )
-
- return myhosts
-
-
- def getlist(self, parser, url):
- """
- Uses the supplied parser to get a list of urls.
- Takes a parser object, url, and filering options.
- """
-
- output.write('getlist(): fetching ' + url + '\n', 2)
-
- output.print_info('Downloading a list of mirrors...')
-
- try:
- parser.parse(urllib.urlopen(url).read())
- except EnvironmentError:
- pass
-
- if len(parser.tuples()) == 0:
- output.print_err('Could not get mirror list. Check your internet'
- ' connection.')
-
- output.write(' Got %d mirrors.\n' % len(parser.tuples()))
-
- return parser.tuples()
-
-
-class Shallow(object):
- """handles rapid server selection via netselect"""
-
- def __init__(self, hosts, options):
- self.urls = []
-
- if options.blocksize is not None:
- self.netselect_split(hosts, options.servers,
- options.blocksize)
- else:
- self.netselect(hosts, options.servers)
-
- if len(self.urls) == 0:
- output.print_err('Netselect failed to return any mirrors.'
- ' Try again using block mode.')
-
-
- def netselect(self, hosts, number, quiet=False):
- """
- Uses Netselect to choose the closest hosts, _very_ quickly
- """
- if not quiet:
- hosts = [host[0] for host in hosts]
- top_host_dict = {}
- top_hosts = []
-
- if not quiet:
- output.print_info('Using netselect to choose the top %d mirrors...' \
- % number)
-
- host_string = ' '.join(hosts)
-
- output.write('\nnetselect(): running "netselect -s%d %s"\n' % (int(number),
- host_string), 2)
-
- proc = subprocess.Popen( ['netselect', '-s%d' % (number,)] + hosts,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
- out, err = proc.communicate()
-
- if err:
- output.write('netselect(): netselect stderr: %s\n' % err, 2)
-
- for line in out.splitlines():
- line = line.split()
- if len(line) < 2:
- continue
- top_hosts.append(line[1])
- top_host_dict[line[0]] = line[1]
-
- if not quiet:
- output.write('Done.\n')
-
- output.write('\nnetselect(): returning %s and %s\n' % (top_hosts,
- top_host_dict), 2)
-
- if quiet:
- return top_hosts, top_host_dict
- else:
- self.urls = top_hosts
-
-
- def netselect_split(self, hosts, number, block_size):
- """
- This uses netselect to test mirrors in chunks, each at most block_size in length.
- This is done in a tournament style.
- """
- hosts = [host[0] for host in hosts]
-
- output.write('netselect_split() got %s hosts.\n' % len(hosts), 2)
-
- host_blocks = self.host_blocks(hosts, block_size)
-
- output.write(' split into %s blocks\n' % len(host_blocks), 2)
-
- top_hosts = []
- ret_hosts = {}
-
- block_index = 0
- for block in host_blocks:
- output.print_info('Using netselect to choose the top '
- '%d hosts, in blocks of %s. %s of %s blocks complete.'
- % (number, block_size, block_index, len(host_blocks)))
-
- host_dict = self.netselect(block, len(block), quiet=True)[1]
-
- output.write('ran netselect(%s, %s), and got %s\n' % (block, len(block),
- host_dict), 2)
-
- for key in host_dict.keys():
- ret_hosts[key] = host_dict[key]
- block_index += 1
-
- sys.stderr.write('\rUsing netselect to choose the top'
- '%d hosts, in blocks of %s. %s of %s blocks complete.\n'
- % (number, block_size, block_index, len(host_blocks)))
-
- host_ranking_keys = ret_hosts.keys()
- host_ranking_keys.sort()
-
- for rank in host_ranking_keys[:number]:
- top_hosts.append(ret_hosts[rank])
-
- output.write('netselect_split(): returns %s\n' % top_hosts, 2)
-
- self.urls = top_hosts
-
-
- def host_blocks(self, hosts, block_size):
- """
- Takes a list of hosts and a block size, and returns an list of lists of URLs.
- Each of the sublists is at most block_size in length.
- """
- host_array = []
- mylist = []
-
- while len(hosts) > block_size:
- while (len(mylist) < block_size):
- mylist.append(hosts.pop())
- host_array.append(mylist)
- mylist = []
- host_array.append(hosts)
-
- output.write('\n_host_blocks(): returns %s blocks, each about %s in size\n'
- % (len(host_array), len(host_array[0])), 2)
-
- return host_array
-
-
-class Deep(object):
- """handles deep mode mirror selection."""
-
- def __init__(self, hosts, options):
- self.urls = []
- self._hosts = hosts
- self._number = options.servers
- self._dns_timeout = options.timeout
- self._connect_timeout = options.timeout
- self._download_timeout = options.timeout
-
- addr_families = []
- if options.ipv4:
- addr_families.append(socket.AF_INET)
- elif options.ipv6:
- addr_families.append(socket.AF_INET6)
- else:
- addr_families.append(socket.AF_INET)
- if socket.has_ipv6:
- addr_families.append(socket.AF_INET6)
-
- self._addr_families = addr_families
-
- self.deeptest()
-
- def deeptest(self):
- """
- Takes a list of hosts and returns the fastest, using _deeptime()
- Doesn't waste time finnishing a test that has already taken longer than
- the slowest mirror weve already got.
- """
- top_hosts = {}
- prog = 0
- maxtime = self._download_timeout
- hosts = [host[0] for host in self._hosts]
- num_hosts = len(hosts)
-
- for host in hosts:
-
- prog += 1
- output.print_info('Downloading 100k files from each mirror... [%s of %s]'\
- % (prog, num_hosts) )
-
- mytime, ignore = self.deeptime(host, maxtime)
-
- if not ignore and mytime < maxtime:
- maxtime, top_hosts = self._list_add((mytime, host), \
- maxtime, top_hosts, self._number)
- else:
- continue
-
- output.write('deeptest(): got %s hosts, and returned %s\n' % (num_hosts, \
- str(top_hosts.values())), 2)
-
- output.write('\n') #this just makes output nicer
-
- #can't just return the dict.valuse, because we want the fastest mirror first...
- keys = top_hosts.keys()
- keys.sort()
-
- rethosts = []
- for key in keys:
- rethosts.append(top_hosts[key])
-
- self.urls = rethosts
-
-
- def deeptime(self, url, maxtime):
- """
- Takes a single url and fetch command, and downloads the test file.
- Can be given an optional timeout, for use with a clever algorithm.
- Like mine.
- """
- output.write('\n_deeptime(): maxtime is %s\n' % maxtime, 2)
-
- if url.endswith('/'): #append the path to the testfile to the URL
- url = url + 'distfiles/mirrorselect-test'
- else:
- url = url + '/distfiles/mirrorselect-test'
-
- url_parts = urlparse.urlparse(url)
-
- class TimeoutException(Exception):
- pass
-
- def timeout_handler(signum, frame):
- raise TimeoutException()
-
- signal.signal(signal.SIGALRM, timeout_handler)
-
- ips = []
- for family in self._addr_families:
- ipv6 = family == socket.AF_INET6
- try:
- try:
- signal.alarm(self._dns_timeout)
- for family, socktype, proto, canonname, sockaddr in \
- socket.getaddrinfo(url_parts.hostname, None,
- family, socket.SOCK_STREAM):
- ip = sockaddr[0]
- if ipv6:
- ip = "[%s]" % ip
- ips.append(ip)
- finally:
- signal.alarm(0)
- except socket.error, e:
- output.write('deeptime(): dns error for host %s: %s\n' % \
- (url_parts.hostname, e), 2)
- except TimeoutException:
- output.write('deeptime(): dns timeout for host %s\n' % \
- (url_parts.hostname,), 2)
-
- if not ips:
- output.write('deeptime(): unable to resolve ip for host %s\n' % \
- (url_parts.hostname,), 2)
- return (None, True)
-
- delta = 0
- f = None
-
- for ip in ips:
- try:
- try:
- signal.alarm(self._connect_timeout)
- f = urllib.urlopen(url)
- break
- finally:
- signal.alarm(0)
- except EnvironmentError, e:
- output.write(('deeptime(): connection to host %s ' + \
- 'failed for ip %s: %s\n') % \
- (url_parts.hostname, ip, e), 2)
- except TimeoutException:
- output.write(('deeptime(): connection to host %s ' + \
- 'timed out for ip %s\n') % \
- (url_parts.hostname, ip), 2)
-
- if f is None:
- output.write('deeptime(): unable to ' + \
- 'connect to host %s\n' % \
- (url_parts.hostname,), 2)
- return (None, True)
-
- try:
- # Close the initial "wake up" connection.
- try:
- signal.alarm(self._connect_timeout)
- f.close()
- finally:
- signal.alarm(0)
- except EnvironmentError, e:
- output.write(('deeptime(): close connection to host %s ' + \
- 'failed for ip %s: %s\n') % \
- (url_parts.hostname, ip, e), 2)
- except TimeoutException:
- output.write(('deeptime(): close connection to host %s ' + \
- 'timed out for ip %s\n') % \
- (url_parts.hostname, ip), 2)
-
- try:
- # The first connection serves to "wake up" the route between
- # the local and remote machines. A second connection is used
- # for the timed run.
- try:
- signal.alarm(int(math.ceil(maxtime)))
- stime = time.time()
- f = urllib.urlopen(url)
-
- if hashlib.md5(f.read()).hexdigest() != "bdf077b2e683c506bf9e8f2494eeb044":
- return (None, True)
-
- delta = time.time() - stime
- f.close()
- finally:
- signal.alarm(0)
-
- except EnvironmentError, e:
- output.write(('deeptime(): download from host %s ' + \
- 'failed for ip %s: %s\n') % \
- (url_parts.hostname, ip, e), 2)
- return (None, True)
- except TimeoutException:
- output.write(('deeptime(): download from host %s ' + \
- 'timed out for ip %s\n') % \
- (url_parts.hostname, ip), 2)
- return (None, True)
-
- signal.signal(signal.SIGALRM, signal.SIG_DFL)
-
- output.write('deeptime(): download completed.\n', 2)
- output.write('deeptime(): %s seconds for host %s\n' % (delta, url), 2)
- return (delta, False)
-
- def _list_add(self, time_host, maxtime, host_dict, maxlen):
- """
- Takes argumets ((time, host), maxtime, host_dict, maxlen)
- Adds a new time:host pair to the dictionary of top hosts.
- If the dictionary is full, the slowest host is removed to make space.
- Returns the new maxtime, be it the specified timeout, or the slowest host.
- """
- if len(host_dict) < maxlen: #still have room, and host is fast. add it.
-
- output.write('_list_add(): added host %s. with a time of %s\n' %
- (time_host[1], time_host[0]), 2)
-
- host_dict.update(dict([time_host]))
- times = host_dict.keys()
- times.sort()
-
- else: #We need to make room in the dict before we add. Kill the slowest.
- output.write('_list_add(): Adding host %s with a time of %s\n' %
- (time_host[1], time_host[0]), 2)
- times = host_dict.keys()
- times.sort()
- output.write('_list_add(): removing %s\n' % host_dict[times[-1]],
- 2)
- del host_dict[times[-1]]
- host_dict.update(dict([time_host]))
- #done adding. now return the appropriate time
- times = host_dict.keys()
- times.sort()
-
- if len(host_dict) < maxlen: #check again to choose new timeout
- output.write('_list_add(): host_dict is not full yet.'
- ' reusing timeout of %s sec.\n' % maxtime, 2)
- retval = maxtime
- else:
- output.write('_list_add(): host_dict is full. Selecting the best'
- ' timeout\n', 2)
- if times[-1] < maxtime:
- retval = times[-1]
- else:
- retval = maxtime
-
- output.write('_list_add(): new max time is %s seconds,'
- ' and now len(host_dict)= %s\n' % (retval, len(host_dict)), 2)
-
- return retval, host_dict
-
-
-class Interactive(object):
- """Handles interactive host selection."""
-
- def __init__(self, hosts, options):
- self.urls = []
-
- self.interactive(hosts, options)
- output.write('Interactive.interactive(): self.urls = %s\n' % self.urls, 2)
-
- if len(self.urls[0]) == 0:
- sys.exit(1)
-
-
- def interactive(self, hosts, options):
- """
- Some sort of interactive menu thingy.
- """
- if options.rsync:
- dialog = 'dialog --stdout --title "Gentoo RSYNC Mirrors"'\
- ' --radiolist "Please select your desired mirror:" 20 110 14'
- else:
- dialog = 'dialog --separate-output --stdout --title'\
- ' "Gentoo Download Mirrors" --checklist "Please'\
- ' select your desired mirrors:'
- if not options.ipv4 and not options.ipv6:
- dialog += '\n* = supports ipv6'
-
- dialog += '" 20 110 14'
-
- for (url, args) in sorted(hosts, key = lambda x: (x[1]['country'].lower(), x[1]['name'].lower()) ):
- marker = ""
- if options.rsync and not url.endswith("/gentoo-portage"):
- url+="/gentoo-portage"
- if (not options.ipv6 and not options.ipv4) and args['ipv6'] == 'y':
- marker = "* "
- if options.ipv6 and ( args['ipv6'] == 'n' ): continue
- if options.ipv4 and ( args['ipv4'] == 'n' ): continue
-
- dialog += ' ' + '"%s" "%s%s: %s" "OFF"' % ( url, marker, args['country'], args['name'] )
-
- mirror_fd = os.popen('%s' % codecs.encode(dialog, 'utf8'))
- mirrors = mirror_fd.read()
- mirror_fd.close()
-
- self.urls = mirrors.rstrip().split('\n')
+from mirrorselect.selectors import Deep, Shallow, Extractor, Interactive
def _have_bin(name):
@@ -539,9 +49,6 @@ def _have_bin(name):
return None
-def handler(signum, frame):
- output.print_err('Caught signal %s. Exiting' % signum)
-
def write_config(hosts, out, path, sync=False):
"""
Writes the make.conf style string to the given file, or to stdout.
@@ -773,7 +280,6 @@ def main(argv):
config_path = '/etc/make.conf'
#output.print_info("config_path set to :", config_path)
- signal.signal(signal.SIGINT, handler)
options = parse_args(argv, config_path)
output.verbosity = options.verbosity
diff --git a/mirrorselect/main.py b/mirrorselect/selectors.py
old mode 100755
new mode 100644
similarity index 60%
copy from mirrorselect/main.py
copy to mirrorselect/selectors.py
index 0f624b7..d2e7e63
--- a/mirrorselect/main.py
+++ b/mirrorselect/selectors.py
@@ -1,51 +1,48 @@
-#!/usr/bin/python
-
-# Mirrorselect 2.x
-# Tool for selecting Gentoo source and rsync mirrors.
-#
-# Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
-# Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
-# Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
-# Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
-
-__revision__ = '2.1.0'
+#!/usr/bin/env python
+#-*- coding:utf-8 -*-
+
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2005-2012 Gentoo Foundation
+
+ Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
+ Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
+ Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
import math
import os
-import re
-import shlex
-import shutil
import signal
import socket
-import string
import subprocess
import sys
import time
import urllib
import urlparse
import hashlib
-from optparse import OptionParser
-from mirrorselect.mirrorparser3 import (MirrorParser3, MIRRORS_3_XML,
- MIRRORS_RSYNC_DATA)
-from mirrorselect.output import output, ColoredFormatter
+from mirrorselect.mirrorparser3 import MirrorParser3
+from mirrorselect.output import output
import codecs
-
class Extractor(object):
"""The Extractor employs a MirrorParser3 object to get a list of valid
mirrors, and then filters them. Only the mirrors that should be tested, based on
@@ -524,275 +521,3 @@ class Interactive(object):
mirror_fd.close()
self.urls = mirrors.rstrip().split('\n')
-
-
-def _have_bin(name):
- """
- Determines whether a particular binary is available on the host system.
- """
- for path_dir in os.environ.get("PATH", "").split(":"):
- if not path_dir:
- continue
- file_path = os.path.join(path_dir, name)
- if os.path.isfile(file_path) and os.access(file_path, os.X_OK):
- return file_path
- return None
-
-
-def handler(signum, frame):
- output.print_err('Caught signal %s. Exiting' % signum)
-
-def write_config(hosts, out, path, sync=False):
- """
- Writes the make.conf style string to the given file, or to stdout.
- """
- if sync:
- var = 'SYNC'
- else:
- var = 'GENTOO_MIRRORS'
-
- mirror_string = '%s="%s"' % (var, ' '.join(hosts))
-
- if out:
- print
- print mirror_string
- sys.exit(0)
-
-
- output.write('\n')
- output.print_info('Modifying %s with new mirrors...\n' % path)
- try:
- config = open(path, 'r')
- output.write('\tReading make.conf\n')
- lines = config.readlines()
- config.close()
- output.write('\tMoving to %s.backup\n' % path)
- shutil.move(path, path + '.backup')
- except IOError:
- lines = []
-
- regex = re.compile('^%s=.*' % var)
- for line in lines:
- if regex.match(line):
- lines.remove(line)
-
- lines.append(mirror_string)
-
- output.write('\tWriting new %s\n' % path)
- config = open(path, 'w')
- for line in lines:
- config.write(line)
- config.write('\n')
- config.close()
-
- output.print_info('Done.\n')
- sys.exit(0)
-
-def get_filesystem_mirrors(out, path, sync=False):
- """
- Read the current mirrors and retain mounted filesystems mirrors
- """
- fsmirrors = []
-
- if sync:
- var = 'SYNC'
- else:
- var = 'GENTOO_MIRRORS'
-
- try:
- f = open(path,'r')
- except IOError:
- return fsmirrors
-
- """ Search for 'var' in make.conf and extract value """
- try:
- lex = shlex.shlex(f, posix=True)
- lex.wordchars = string.digits+string.letters+"~!@#$%*_\:;?,./-+{}"
- lex.quotes = "\"'"
- while 1:
- key = lex.get_token()
- if key == var:
- equ = lex.get_token()
-
- if (equ == ''):
- break
- elif (equ != '='):
- break
-
- val = lex.get_token()
- if val is None:
- break
-
- """ Look for mounted filesystem in value """
- mirrorlist = val.rsplit()
- p = re.compile('rsync://|http://|ftp://', re.IGNORECASE)
- for mirror in mirrorlist:
- if (p.match(mirror) == None):
- fsmirrors.append(mirror)
- break
- elif key is None:
- break
- except Exception:
- fsmirrors = []
-
- return fsmirrors
-
-def parse_args(argv, config_path):
- """
- Does argument parsing and some sanity checks.
- Returns an optparse Options object.
-
- The descriptions, grouping, and possibly the amount sanity checking
- need some finishing touches.
- """
- desc = "\n".join((
- output.white("examples:"),
- "",
- output.white(" automatic:"),
- " # mirrorselect -s5",
- " # mirrorselect -s3 -b10 -o >> /mnt/gentoo/etc/portage/make.conf",
- " # mirrorselect -D -s4",
- "",
- output.white(" interactive:"),
- " # mirrorselect -i -r",
- ))
- parser = OptionParser(
- formatter=ColoredFormatter(), description=desc,
- version='Mirrorselect version: %s' % __revision__)
-
- group = parser.add_option_group("Main modes")
- group.add_option(
- "-i", "--interactive", action="store_true", default=False,
- help="Interactive Mode, this will present a list "
- "to make it possible to select mirrors you wish to use.")
- group.add_option(
- "-D", "--deep", action="store_true", default=False,
- help="Deep mode. This is used to give a more accurate "
- "speed test. It will download a 100k file from "
- "each server. Because of this you should only use "
- "this option if you have a good connection.")
-
- group = parser.add_option_group(
- "Server type selection (choose at most one)")
- group.add_option(
- "-F", "--ftp", action="store_true", default=False,
- help="ftp only mode. Will not consider hosts of other "
- "types.")
- group.add_option(
- "-H", "--http", action="store_true", default=False,
- help="http only mode. Will not consider hosts of other types")
- group.add_option(
- "-r", "--rsync", action="store_true", default=False,
- help="rsync mode. Allows you to interactively select your"
- " rsync mirror. Requires -i to be used.")
- group.add_option(
- "-4", "--ipv4", action="store_true", default=False,
- help="only use IPv4")
- group.add_option(
- "-6", "--ipv6", action="store_true", default=False,
- help="only use IPv6")
-
- group = parser.add_option_group("Other options")
- group.add_option(
- "-o", "--output", action="store_true", default=False,
- help="Output Only Mode, this is especially useful "
- "when being used during installation, to redirect "
- "output to a file other than %s" % config_path)
- group.add_option(
- "-b", "--blocksize", action="store", type="int",
- help="This is to be used in automatic mode "
- "and will split the hosts into blocks of BLOCKSIZE for "
- "use with netselect. This is required for certain "
- "routers which block 40+ requests at any given time. "
- "Recommended parameters to pass are: -s3 -b10")
- group.add_option(
- "-t", "--timeout", action="store", type="int",
- default="10", help="Timeout for deep mode. Defaults to 10 seconds.")
- group.add_option(
- "-s", "--servers", action="store", type="int", default=1,
- help="Specify Number of servers for Automatic Mode "
- "to select. this is only valid for download mirrors. "
- "If this is not specified, a default of 1 is used.")
- group.add_option(
- "-d", "--debug", action="store_const", const=2, dest="verbosity",
- default=1, help="debug mode")
- group.add_option(
- "-q", "--quiet", action="store_const", const=0, dest="verbosity",
- help="Quiet mode")
-
- if len(argv) == 1:
- parser.print_help()
- sys.exit(1)
-
- options, args = parser.parse_args(argv[1:])
-
- # sanity checks
-
- # hack: check if more than one of these is set
- if options.http + options.ftp + options.rsync > 1:
- output.print_err('Choose at most one of -H, -f and -r')
-
- if options.ipv4 and options.ipv6:
- output.print_err('Choose at most one of --ipv4 and --ipv6')
-
- if (options.ipv6 and not socket.has_ipv6) and not options.interactive:
- options.ipv6 = False
- output.print_err('The --ipv6 option requires python ipv6 support')
-
- if options.rsync and not options.interactive:
- output.print_err('rsync servers can only be selected with -i')
-
- if options.interactive and (
- options.deep or
- options.blocksize or
- options.servers > 1):
- output.print_err('Invalid option combination with -i')
-
- if (not options.deep) and (not _have_bin('netselect') ):
- output.print_err(
- 'You do not appear to have netselect on your system. '
- 'You must use the -D flag')
-
- if (os.getuid() != 0) and not options.output:
- output.print_err('Must be root to write to %s!\n' % config_path)
-
- if args:
- output.print_err('Unexpected arguments passed.')
-
- # return results
- return options
-
-
-def main(argv):
- """Lets Rock!"""
- # start with the new location
- config_path = '/etc/portage/make.conf'
- if not os.access(config_path, os.F_OK):
- # check if the old location is what is used
- if os.access('/etc/make.conf', os.F_OK):
- config_path = '/etc/make.conf'
-
- #output.print_info("config_path set to :", config_path)
- signal.signal(signal.SIGINT, handler)
-
- options = parse_args(argv, config_path)
- output.verbosity = options.verbosity
-
- fsmirrors = get_filesystem_mirrors(options.output, config_path, options.rsync)
- if options.rsync:
- hosts = Extractor(MIRRORS_RSYNC_DATA, options).hosts
- else:
- hosts = Extractor(MIRRORS_3_XML, options).hosts
-
- if options.interactive:
- selector = Interactive(hosts, options)
- elif options.deep:
- selector = Deep(hosts, options)
- else:
- selector = Shallow(hosts, options)
-
- write_config(fsmirrors + selector.urls, options.output, config_path, options.rsync)
-
-
-if __name__ == '__main__':
- main(sys.argv)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-12 7:46 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-12 7:46 UTC (permalink / raw
To: gentoo-commits
commit: 92a938df33eeb5e7da5069cf6b2029e68949b5e5
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 12 04:21:10 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Nov 12 07:23:09 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=92a938df
move version to it's own file. split out more logic from main() to their own functions.
---
mirrorselect/main.py | 49 +++++++++++++++++++++++++++++-----------------
mirrorselect/version.py | 32 ++++++++++++++++++++++++++++++
2 files changed, 63 insertions(+), 18 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 2f257a6..7052f57 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -21,7 +21,6 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
-__revision__ = '2.1.0'
import os
import re
@@ -34,6 +33,7 @@ from optparse import OptionParser
from mirrorselect.mirrorparser3 import MIRRORS_3_XML, MIRRORS_RSYNC_DATA
from mirrorselect.output import Output, ColoredFormatter
from mirrorselect.selectors import Deep, Shallow, Extractor, Interactive
+from mirrorselect.version import version
class MirrorSelect(object):
@@ -152,7 +152,7 @@ class MirrorSelect(object):
return fsmirrors
- def parse_args(self, argv, config_path):
+ def _parse_args(self, argv, config_path):
"""
Does argument parsing and some sanity checks.
Returns an optparse Options object.
@@ -173,7 +173,7 @@ class MirrorSelect(object):
))
parser = OptionParser(
formatter=ColoredFormatter(self.output), description=desc,
- version='Mirrorselect version: %s' % __revision__)
+ version='Mirrorselect version: %s' % version)
group = parser.add_option_group("Main modes")
group.add_option(
@@ -278,6 +278,26 @@ class MirrorSelect(object):
return options
+ def get_available_hosts(self, options):
+ ''''''
+ if options.rsync:
+ hosts = Extractor(MIRRORS_RSYNC_DATA, options, self.output).hosts
+ else:
+ hosts = Extractor(MIRRORS_3_XML, options, self.output).hosts
+ return hosts
+
+
+ def select_urls(self, hosts, options):
+ ''''''
+ if options.interactive:
+ selector = Interactive(hosts, options, self.output)
+ elif options.deep:
+ selector = Deep(hosts, options, self.output)
+ else:
+ selector = Shallow(hosts, options, self.output)
+ return selector.urls
+
+
def main(self, argv):
"""Lets Rock!"""
# start with the new location
@@ -287,23 +307,16 @@ class MirrorSelect(object):
if os.access('/etc/make.conf', os.F_OK):
config_path = '/etc/make.conf'
- #self.output.print_info("config_path set to :", config_path)
-
- options = self.parse_args(argv, config_path)
+ options = self._parse_args(argv, config_path)
self.output.verbosity = options.verbosity
- fsmirrors = self.get_filesystem_mirrors(options.output, config_path, options.rsync)
- if options.rsync:
- hosts = Extractor(MIRRORS_RSYNC_DATA, options, self.output).hosts
- else:
- hosts = Extractor(MIRRORS_3_XML, options, self.output).hosts
+ fsmirrors = self.get_filesystem_mirrors(options.output,
+ config_path, options.rsync)
- if options.interactive:
- selector = Interactive(hosts, options, self.output)
- elif options.deep:
- selector = Deep(hosts, options, self.output)
- else:
- selector = Shallow(hosts, options, self.output)
+ hosts = self.get_available_hosts(options)
+
+ urls = self.select_urls(hosts, options)
- self.write_config(fsmirrors + selector.urls, options.output, config_path, options.rsync)
+ self.write_config(fsmirrors + urls, options.output,
+ config_path, options.rsync)
diff --git a/mirrorselect/version.py b/mirrorselect/version.py
new file mode 100644
index 0000000..53c66cb
--- /dev/null
+++ b/mirrorselect/version.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#-*- coding:utf-8 -*-
+
+
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2005-2012 Gentoo Foundation
+
+ Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
+ Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
+ Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
+
+version = "git"
+
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-12 7:46 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-12 7:46 UTC (permalink / raw
To: gentoo-commits
commit: c4ddff3ed75c939dc6ef179b69635fdf214183c1
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 12 07:10:26 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Nov 12 07:23:09 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=c4ddff3e
Move config_path determination to it's own function.
Docstring updates, remove unused parameters.
Rename path parameter to config_path for clarity.
---
mirrorselect/main.py | 152 ++++++++++++++++++++++++++++++++++---------------
1 files changed, 105 insertions(+), 47 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 7052f57..9db4342 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -1,25 +1,35 @@
-#!/usr/bin/python
-
-# Mirrorselect 2.x
-# Tool for selecting Gentoo source and rsync mirrors.
-#
-# Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
-# Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
-# Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
-# Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+#!/usr/bin/env python
+#-*- coding:utf-8 -*-
+
+
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2005-2012 Gentoo Foundation
+
+ Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
+ Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
+ Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
+
+
+from __future__ import print_function
import os
@@ -40,13 +50,22 @@ class MirrorSelect(object):
'''Main operational class'''
def __init__(self, output=None):
- '''Main class init'''
+ '''MirrorSelect class init
+
+ @param output: mirrorselect.output.Ouptut() class instance
+ or None for the default instance
+ '''
self.output = output or Output()
+
@staticmethod
def _have_bin(name):
- """
- Determines whether a particular binary is available on the host system.
+ """Determines whether a particular binary is available
+ on the host system. It searches in the PATH environment
+ variable paths.
+
+ @param name: string, binary name to search for
+ @rtype: string or None
"""
for path_dir in os.environ.get("PATH", "").split(":"):
if not path_dir:
@@ -57,9 +76,14 @@ class MirrorSelect(object):
return None
- def write_config(self, hosts, out, path, sync=False):
- """
- Writes the make.conf style string to the given file, or to stdout.
+ def write_config(self, hosts, out, config_path, sync=False):
+ """Writes the make.conf style string to the given file, or to stdout.
+
+ @param hosts: list of host urls to write
+ @param out: boolean, used to redirect output to stdout
+ @param config_path; string
+ @param sync: boolean, used to switch between SYNC and GENTOO_MIRRORS
+ make.conf variable target
"""
if sync:
var = 'SYNC'
@@ -69,20 +93,19 @@ class MirrorSelect(object):
mirror_string = '%s="%s"' % (var, ' '.join(hosts))
if out:
- print
- print mirror_string
+ print()
+ print(mirror_string)
sys.exit(0)
-
self.output.write('\n')
- self.output.print_info('Modifying %s with new mirrors...\n' % path)
+ self.output.print_info('Modifying %s with new mirrors...\n' % config_path)
try:
- config = open(path, 'r')
+ config = open(config_path, 'r')
self.output.write('\tReading make.conf\n')
lines = config.readlines()
config.close()
- self.output.write('\tMoving to %s.backup\n' % path)
- shutil.move(path, path + '.backup')
+ self.output.write('\tMoving to %s.backup\n' % config_path)
+ shutil.move(config_path, config_path + '.backup')
except IOError:
lines = []
@@ -93,8 +116,8 @@ class MirrorSelect(object):
lines.append(mirror_string)
- self.output.write('\tWriting new %s\n' % path)
- config = open(path, 'w')
+ self.output.write('\tWriting new %s\n' % config_path)
+ config = open(config_path, 'w')
for line in lines:
config.write(line)
config.write('\n')
@@ -103,9 +126,15 @@ class MirrorSelect(object):
self.output.print_info('Done.\n')
sys.exit(0)
- def get_filesystem_mirrors(self, out, path, sync=False):
- """
- Read the current mirrors and retain mounted filesystems mirrors
+
+ @staticmethod
+ def get_filesystem_mirrors(config_path, sync=False):
+ """Read the current mirrors and retain mounted filesystems mirrors
+
+ @param config_path: string
+ @param sync: boolean, used to switch between SYNC and GENTOO_MIRRORS
+ make.conf variable target
+ @rtype list
"""
fsmirrors = []
@@ -115,7 +144,7 @@ class MirrorSelect(object):
var = 'GENTOO_MIRRORS'
try:
- f = open(path,'r')
+ f = open(config_path,'r')
except IOError:
return fsmirrors
@@ -152,6 +181,7 @@ class MirrorSelect(object):
return fsmirrors
+
def _parse_args(self, argv, config_path):
"""
Does argument parsing and some sanity checks.
@@ -279,7 +309,12 @@ class MirrorSelect(object):
def get_available_hosts(self, options):
- ''''''
+ '''Returns a list of hosts suitable for consideration by a user
+ based on user input
+
+ @param options: parser.parse_args() options instance
+ @rtype: list
+ '''
if options.rsync:
hosts = Extractor(MIRRORS_RSYNC_DATA, options, self.output).hosts
else:
@@ -288,7 +323,16 @@ class MirrorSelect(object):
def select_urls(self, hosts, options):
- ''''''
+ '''Returns the list of selected host urls using
+ the options passed in to run one of the three selector types.
+ 1) Interactive ncurses dialog
+ 2) Deep mode mirror selection.
+ 3) (Shallow) Rapid server selection via netselect
+
+ @param hosts: list of hosts to choose from
+ @param options: parser.parse_args() options instance
+ @rtype: list
+ '''
if options.interactive:
selector = Interactive(hosts, options, self.output)
elif options.deep:
@@ -298,20 +342,34 @@ class MirrorSelect(object):
return selector.urls
- def main(self, argv):
- """Lets Rock!"""
+ @staticmethod
+ def get_make_conf_path():
+ '''Checks for the existance of make.conf in /etc/portage/
+ Failing that it checks for it in /etc/
+ Failing in /etc/ it defaults to /etc/portage/make.conf
+
+ @rtype: string
+ '''
# start with the new location
config_path = '/etc/portage/make.conf'
if not os.access(config_path, os.F_OK):
# check if the old location is what is used
if os.access('/etc/make.conf', os.F_OK):
config_path = '/etc/make.conf'
+ return config_path
+
+
+ def main(self, argv):
+ """Lets Rock!
+
+ @param argv: list of command line arguments to parse
+ """
+ config_path = self.get_make_conf_path()
options = self._parse_args(argv, config_path)
self.output.verbosity = options.verbosity
- fsmirrors = self.get_filesystem_mirrors(options.output,
- config_path, options.rsync)
+ fsmirrors = self.get_filesystem_mirrors(config_path, options.rsync)
hosts = self.get_available_hosts(options)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-12 15:56 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-12 15:56 UTC (permalink / raw
To: gentoo-commits
commit: 834e89227be486c6458aeb8c28423bbbb6fa3bd5
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 12 15:52:36 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Nov 12 15:52:36 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=834e8922
initial EPREFIX enabling. not fully tested.
---
mirrorselect/main.py | 15 ++++++++++++---
1 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 13c0a9f..528f383 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -47,6 +47,15 @@ from mirrorselect.selectors import Deep, Shallow, Extractor, Interactive
from mirrorselect.version import version
+# establish the eprefix, initially set so eprefixify can
+# set it on install
+EPREFIX = "@GENTOO_PORTAGE_EPREFIX@"
+
+# check and set it if it wasn't
+if "GENTOO_PORTAGE_EPREFIX" in EPREFIX:
+ EPREFIX = ''
+
+
class MirrorSelect(object):
'''Main operational class'''
@@ -352,11 +361,11 @@ class MirrorSelect(object):
@rtype: string
'''
# start with the new location
- config_path = '/etc/portage/make.conf'
+ config_path = EPREFIX + '/etc/portage/make.conf'
if not os.access(config_path, os.F_OK):
# check if the old location is what is used
- if os.access('/etc/make.conf', os.F_OK):
- config_path = '/etc/make.conf'
+ if os.access(EPREFIX + '/etc/make.conf', os.F_OK):
+ config_path = EPREFIX + '/etc/make.conf'
return config_path
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-12 20:37 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-12 20:37 UTC (permalink / raw
To: gentoo-commits
commit: eacec616dd653bb01a14a09b4bac832aa287f84b
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 12 19:25:56 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Nov 12 20:35:40 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=eacec616
add py3 compatability fixes, update mirrorparser3.py header to a docstring
---
mirrorselect/main.py | 11 +++++
mirrorselect/mirrorparser3.py | 70 ++++++++++++++++++++-------------
mirrorselect/selectors.py | 85 ++++++++++++++++++++++++++---------------
3 files changed, 107 insertions(+), 59 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 528f383..55c1cb0 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -56,6 +56,12 @@ if "GENTOO_PORTAGE_EPREFIX" in EPREFIX:
EPREFIX = ''
+if sys.hexversion >= 0x3000000:
+ _unicode = str
+else:
+ _unicode = unicode
+
+
class MirrorSelect(object):
'''Main operational class'''
@@ -100,6 +106,9 @@ class MirrorSelect(object):
else:
var = 'GENTOO_MIRRORS'
+ if hasattr(hosts[0], 'decode'):
+ hosts = [x.decode('utf-8') for x in hosts]
+
mirror_string = '%s="%s"' % (var, ' '.join(hosts))
if out:
@@ -127,7 +136,9 @@ class MirrorSelect(object):
lines.append(mirror_string)
self.output.write('\tWriting new %s\n' % config_path)
+
config = open(config_path, 'w')
+
for line in lines:
config.write(line)
config.write('\n')
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index e5df3fc..e34f9c2 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -1,24 +1,33 @@
#!/usr/bin/python
+# coding: utf-8
-# Mirrorselect 1.x
-# Tool for selecting Gentoo source and rsync mirrors.
-#
-# Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
-# Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2009-2012 Gentoo Foundation
+
+ Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+ Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
+
+
+from __future__ import print_function
from xml.etree import ElementTree as ET
@@ -52,16 +61,21 @@ class MirrorParser3:
}
def tuples(self):
- return [(url, args) for url, args in self._dict.items()]
+ return [(url, args) for url, args in list(self._dict.items())]
def uris(self):
- return [url for url, args in self._dict.items()]
+ return [url for url, args in list(self._dict.items())]
if __name__ == '__main__':
- import urllib
- parser = MirrorParser3()
- parser.parse(urllib.urlopen(MIRRORS_3_XML).read())
- print '===== tuples'
- print parser.tuples()
- print '===== uris'
- print parser.uris()
+ if int(sys.version[0]) == 3:
+ import urllib.request, urllib.parse, urllib.error
+ parser = MirrorParser3()
+ parser.parse(urllib.request.urlopen(MIRRORS_3_XML).read())
+ else:
+ import urllib
+ parser = MirrorParser3()
+ parser.parse(urllib.urlopen(MIRRORS_3_XML).read())
+ print('===== tuples')
+ print(parser.tuples())
+ print('===== uris')
+ print(parser.uris())
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index a8f8e24..98bd698 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -29,18 +29,30 @@ Distributed under the terms of the GNU General Public License v2
"""
import math
-import os
import signal
import socket
import subprocess
import sys
import time
-import urllib
-import urlparse
import hashlib
+
+if int(sys.version[0]) == 3:
+ import urllib.request, urllib.parse, urllib.error
+ url_parse = urllib.parse
+ url_open = urllib.request.urlopen
+else:
+ import urllib
+ import urlparse
+ url_parse = urlparse.urlparse
+ url_open = urllib.urlopen
+
+
from mirrorselect.mirrorparser3 import MirrorParser3
-import codecs
+if sys.hexversion >= 0x3000000:
+ _unicode = str
+else:
+ _unicode = unicode
class Extractor(object):
@@ -95,7 +107,7 @@ class Extractor(object):
self.output.print_info('Downloading a list of mirrors...')
try:
- parser.parse(urllib.urlopen(url).read())
+ parser.parse(url_open(url).read())
except EnvironmentError:
pass
@@ -198,7 +210,7 @@ class Shallow(object):
self.output.write('ran netselect(%s, %s), and got %s\n' % (block, len(block),
host_dict), 2)
- for key in host_dict.keys():
+ for key in list(host_dict.keys()):
ret_hosts[key] = host_dict[key]
block_index += 1
@@ -206,7 +218,7 @@ class Shallow(object):
'%d hosts, in blocks of %s. %s of %s blocks complete.\n'
% (number, block_size, block_index, len(host_blocks)))
- host_ranking_keys = ret_hosts.keys()
+ host_ranking_keys = list(ret_hosts.keys())
host_ranking_keys.sort()
for rank in host_ranking_keys[:number]:
@@ -291,12 +303,12 @@ class Deep(object):
continue
self.output.write('deeptest(): got %s hosts, and returned %s\n' % (num_hosts, \
- str(top_hosts.values())), 2)
+ str(list(top_hosts.values()))), 2)
self.output.write('\n') #this just makes output nicer
#can't just return the dict.valuse, because we want the fastest mirror first...
- keys = top_hosts.keys()
+ keys = list(top_hosts.keys())
keys.sort()
rethosts = []
@@ -319,7 +331,7 @@ class Deep(object):
else:
url = url + '/distfiles/mirrorselect-test'
- url_parts = urlparse.urlparse(url)
+ url_parts = url_parse(url)
class TimeoutException(Exception):
pass
@@ -344,7 +356,7 @@ class Deep(object):
ips.append(ip)
finally:
signal.alarm(0)
- except socket.error, e:
+ except socket.error as e:
self.output.write('deeptime(): dns error for host %s: %s\n' % \
(url_parts.hostname, e), 2)
except TimeoutException:
@@ -363,11 +375,11 @@ class Deep(object):
try:
try:
signal.alarm(self._connect_timeout)
- f = urllib.urlopen(url)
+ f = url_open(url)
break
finally:
signal.alarm(0)
- except EnvironmentError, e:
+ except EnvironmentError as e:
self.output.write(('deeptime(): connection to host %s ' + \
'failed for ip %s: %s\n') % \
(url_parts.hostname, ip, e), 2)
@@ -389,7 +401,7 @@ class Deep(object):
f.close()
finally:
signal.alarm(0)
- except EnvironmentError, e:
+ except EnvironmentError as e:
self.output.write(('deeptime(): close connection to host %s ' + \
'failed for ip %s: %s\n') % \
(url_parts.hostname, ip, e), 2)
@@ -405,7 +417,7 @@ class Deep(object):
try:
signal.alarm(int(math.ceil(maxtime)))
stime = time.time()
- f = urllib.urlopen(url)
+ f = url_open(url)
if hashlib.md5(f.read()).hexdigest() != "bdf077b2e683c506bf9e8f2494eeb044":
return (None, True)
@@ -415,7 +427,7 @@ class Deep(object):
finally:
signal.alarm(0)
- except EnvironmentError, e:
+ except EnvironmentError as e:
self.output.write(('deeptime(): download from host %s ' + \
'failed for ip %s: %s\n') % \
(url_parts.hostname, ip, e), 2)
@@ -445,20 +457,20 @@ class Deep(object):
(time_host[1], time_host[0]), 2)
host_dict.update(dict([time_host]))
- times = host_dict.keys()
+ times = list(host_dict.keys())
times.sort()
else: #We need to make room in the dict before we add. Kill the slowest.
self.output.write('_list_add(): Adding host %s with a time of %s\n' %
(time_host[1], time_host[0]), 2)
- times = host_dict.keys()
+ times = list(host_dict.keys())
times.sort()
self.output.write('_list_add(): removing %s\n' % host_dict[times[-1]],
2)
del host_dict[times[-1]]
host_dict.update(dict([time_host]))
#done adding. now return the appropriate time
- times = host_dict.keys()
+ times = list(host_dict.keys())
times.sort()
if len(host_dict) < maxlen: #check again to choose new timeout
@@ -498,16 +510,17 @@ class Interactive(object):
Some sort of interactive menu thingy.
"""
if options.rsync:
- dialog = 'dialog --stdout --title "Gentoo RSYNC Mirrors"'\
- ' --radiolist "Please select your desired mirror:" 20 110 14'
+ dialog = ['dialog', '--stdout', '--title', '"Gentoo RSYNC Mirrors"',
+ '--radiolist', '"Please select your desired mirror:"',
+ '20', '110', '14']
else:
- dialog = 'dialog --separate-output --stdout --title'\
- ' "Gentoo Download Mirrors" --checklist "Please'\
- ' select your desired mirrors:'
+ dialog = ['dialog', '--separate-output', '--stdout', '--title',
+ '"Gentoo Download Mirrors"', '--checklist',
+ '"Please select your desired mirrors:']
if not options.ipv4 and not options.ipv6:
- dialog += '\n* = supports ipv6'
+ dialog[-1] += '\n* = supports ipv6'
- dialog += '" 20 110 14'
+ dialog.extend(['20', '110', '14'])
for (url, args) in sorted(hosts, key = lambda x: (x[1]['country'].lower(), x[1]['name'].lower()) ):
marker = ""
@@ -518,10 +531,20 @@ class Interactive(object):
if options.ipv6 and ( args['ipv6'] == 'n' ): continue
if options.ipv4 and ( args['ipv4'] == 'n' ): continue
- dialog += ' ' + '"%s" "%s%s: %s" "OFF"' % ( url, marker, args['country'], args['name'] )
+ #dialog.append('"%s" "%s%s: %s" "OFF"' % ( url, marker, args['country'], args['name']))
+ dialog.extend(["%s" %url,
+ "%s%s: %s" %(marker, args['country'], args['name']),
+ "OFF"])
+ dialog = [_unicode(x) for x in dialog]
+ proc = subprocess.Popen( dialog,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ out, err = proc.communicate()
+
+ self.urls = out.splitlines()
- mirror_fd = os.popen('%s' % codecs.encode(dialog, 'utf8'))
- mirrors = mirror_fd.read()
- mirror_fd.close()
+ if hasattr(self.urls[0], 'decode'):
+ self.urls = [x.decode('utf-8').rstrip() for x in self.urls]
+ else:
+ self.urls = [x.rstrip() for x in self.urls]
- self.urls = mirrors.rstrip().split('\n')
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-12 21:41 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-12 21:41 UTC (permalink / raw
To: gentoo-commits
commit: cd29d33d2fc969844e6a2d13fbdd9de52be1d365
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 12 21:39:44 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon Nov 12 21:39:44 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=cd29d33d
fix dupe'd py3 check, set
---
mirrorselect/selectors.py | 7 ++-----
1 files changed, 2 insertions(+), 5 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 98bd698..e3ad7d0 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -40,20 +40,17 @@ if int(sys.version[0]) == 3:
import urllib.request, urllib.parse, urllib.error
url_parse = urllib.parse
url_open = urllib.request.urlopen
+ _unicode = str
else:
import urllib
import urlparse
url_parse = urlparse.urlparse
url_open = urllib.urlopen
+ _unicode = unicode
from mirrorselect.mirrorparser3 import MirrorParser3
-if sys.hexversion >= 0x3000000:
- _unicode = str
-else:
- _unicode = unicode
-
class Extractor(object):
"""The Extractor employs a MirrorParser3 object to get a list of valid
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-14 20:28 Paul Varner
0 siblings, 0 replies; 86+ messages in thread
From: Paul Varner @ 2012-11-14 20:28 UTC (permalink / raw
To: gentoo-commits
commit: 576d19d461e97581a6daacc344847bfb565363d6
Author: Paul Varner <fuzzyray <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 14 20:25:01 2012 +0000
Commit: Paul Varner <fuzzyray <AT> gentoo <DOT> org>
CommitDate: Wed Nov 14 20:25:01 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=576d19d4
Call the base class constructor for ColoredFormatter so the class gets
properly constructed.
---
mirrorselect/output.py | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/mirrorselect/output.py b/mirrorselect/output.py
index 4137cad..632791a 100644
--- a/mirrorselect/output.py
+++ b/mirrorselect/output.py
@@ -107,6 +107,7 @@ class ColoredFormatter(IndentedHelpFormatter):
"""
def __init__(self, output):
+ IndentedHelpFormatter.__init__(self)
self.output = output
def format_heading(self, heading):
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-15 3:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-15 3:44 UTC (permalink / raw
To: gentoo-commits
commit: 37135bfdcf9cae9ed3273d3ef446dc886e809e60
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 15 03:43:40 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Nov 15 03:43:40 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=37135bfd
fix a null selection in the interactive dialog and hopefully a POSIX locale issue
---
mirrorselect/output.py | 40 ++++++++++++++++++++++++++++++++++++++++
mirrorselect/selectors.py | 16 ++++++++--------
2 files changed, 48 insertions(+), 8 deletions(-)
diff --git a/mirrorselect/output.py b/mirrorselect/output.py
index 632791a..fab3d9a 100644
--- a/mirrorselect/output.py
+++ b/mirrorselect/output.py
@@ -31,10 +31,50 @@ Distributed under the terms of the GNU General Public License v2
import sys
import re
+import codecs
from optparse import IndentedHelpFormatter
+if sys.hexversion >= 0x3000000:
+ _unicode = str
+else:
+ _unicode = unicode
+
+
+def encoder(text, _encoding_):
+ return codecs.encode(text, _encoding_, 'replace')
+
+
+def decode_selection(selection):
+ '''utility function to decode a list of strings
+ accoring to the filesystem encoding
+ '''
+ # fix None passed in, return an empty list
+ selection = selection or []
+ enc = sys.getfilesystemencoding()
+ if enc is not None:
+ return [encoder(i, enc) for i in selection]
+ return selection
+
+
+def get_encoding(output):
+ if hasattr(output, 'encoding') \
+ and output.encoding != None:
+ return output.encoding
+ else:
+ encoding = locale.getpreferredencoding()
+ # Make sure that python knows the encoding. Bug 350156
+ try:
+ # We don't care about what is returned, we just want to
+ # verify that we can find a codec.
+ codecs.lookup(encoding)
+ except LookupError:
+ # Python does not know the encoding, so use utf-8.
+ encoding = 'utf_8'
+ return encoding
+
+
class Output(object):
"""Handles text output. Only prints messages with level <= verbosity.
Therefore, verbosity=2 is everything (debug), and verbosity=0 is urgent
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index e3ad7d0..b2a5fc7 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -40,16 +40,15 @@ if int(sys.version[0]) == 3:
import urllib.request, urllib.parse, urllib.error
url_parse = urllib.parse
url_open = urllib.request.urlopen
- _unicode = str
else:
import urllib
import urlparse
url_parse = urlparse.urlparse
url_open = urllib.urlopen
- _unicode = unicode
from mirrorselect.mirrorparser3 import MirrorParser3
+from mirrorselect.output import encoder, get_encoding, decode_selection
class Extractor(object):
@@ -498,7 +497,7 @@ class Interactive(object):
self.interactive(hosts, options)
self.output.write('Interactive.interactive(): self.urls = %s\n' % self.urls, 2)
- if len(self.urls[0]) == 0:
+ if not self.urls or len(self.urls[0]) == 0:
sys.exit(1)
@@ -532,7 +531,7 @@ class Interactive(object):
dialog.extend(["%s" %url,
"%s%s: %s" %(marker, args['country'], args['name']),
"OFF"])
- dialog = [_unicode(x) for x in dialog]
+ dialog = [encoder(x, get_encoding(sys.stdout)) for x in dialog]
proc = subprocess.Popen( dialog,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -540,8 +539,9 @@ class Interactive(object):
self.urls = out.splitlines()
- if hasattr(self.urls[0], 'decode'):
- self.urls = [x.decode('utf-8').rstrip() for x in self.urls]
- else:
- self.urls = [x.rstrip() for x in self.urls]
+ if self.urls:
+ if hasattr(self.urls[0], 'decode'):
+ self.urls = decode_selection([x.decode('utf-8').rstrip() for x in self.urls])
+ else:
+ self.urls = decode_selection([x.rstrip() for x in self.urls])
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-11-15 3:53 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-11-15 3:53 UTC (permalink / raw
To: gentoo-commits
commit: 9d2d6c6d0af675586655c784155b24acf3c81fce
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 15 03:53:44 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Nov 15 03:53:44 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=9d2d6c6d
add missing import from last change
---
mirrorselect/output.py | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/mirrorselect/output.py b/mirrorselect/output.py
index fab3d9a..d2ccc0a 100644
--- a/mirrorselect/output.py
+++ b/mirrorselect/output.py
@@ -32,6 +32,7 @@ Distributed under the terms of the GNU General Public License v2
import sys
import re
import codecs
+import locale
from optparse import IndentedHelpFormatter
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-12-15 21:25 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-12-15 21:25 UTC (permalink / raw
To: gentoo-commits
commit: 0bf5677196fcca080b2ad61dbf2d08da537912e9
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 15 21:24:53 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Dec 15 21:24:53 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=0bf56771
add eprefix comapatibility code.
---
mirrorselect/main.py | 8 +++++++-
1 files changed, 7 insertions(+), 1 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 55c1cb0..cbff890 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -46,6 +46,12 @@ from mirrorselect.output import Output, ColoredFormatter
from mirrorselect.selectors import Deep, Shallow, Extractor, Interactive
from mirrorselect.version import version
+# eprefix compatibility
+try:
+ from portage.const import rootuid
+except ImportError:
+ rootuid = 0
+
# establish the eprefix, initially set so eprefixify can
# set it on install
@@ -319,7 +325,7 @@ class MirrorSelect(object):
'You do not appear to have netselect on your system. '
'You must use the -D flag')
- if (os.getuid() != 0) and not options.output:
+ if (os.getuid() != rootuid) and not options.output:
self.output.print_err('Must be root to write to %s!\n' % config_path)
if args:
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2012-12-16 2:38 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2012-12-16 2:38 UTC (permalink / raw
To: gentoo-commits
commit: 362ba4585a48273289f382c16947e858ff3818cf
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Dec 16 02:37:50 2012 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Dec 16 02:37:50 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=362ba458
Add aditional debug info, fix bug 373195, removing garbage or otherwise inaccessible file system mirrors from existing SYNC or GENTOO_MIRRORS make.conf values.
---
mirrorselect/main.py | 14 +++++++++++---
mirrorselect/selectors.py | 4 +++-
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index cbff890..48d74a7 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -154,8 +154,7 @@ class MirrorSelect(object):
sys.exit(0)
- @staticmethod
- def get_filesystem_mirrors(config_path, sync=False):
+ def get_filesystem_mirrors(self, config_path, sync=False):
"""Read the current mirrors and retain mounted filesystems mirrors
@param config_path: string
@@ -170,6 +169,7 @@ class MirrorSelect(object):
else:
var = 'GENTOO_MIRRORS'
+ self.output.write('get_filesystem_mirrors(): config_path = %s\n' % config_path)
try:
f = open(config_path,'r')
except IOError:
@@ -182,6 +182,8 @@ class MirrorSelect(object):
lex.quotes = "\"'"
while 1:
key = lex.get_token()
+ #self.output.write('get_filesystem_mirrors(): processing key = %s\n' % key, 2)
+
if key == var:
equ = lex.get_token()
@@ -196,16 +198,22 @@ class MirrorSelect(object):
""" Look for mounted filesystem in value """
mirrorlist = val.rsplit()
+ self.output.write('get_filesystem_mirrors(): mirrorlist = %s\n' % mirrorlist, 2)
p = re.compile('rsync://|http://|ftp://', re.IGNORECASE)
for mirror in mirrorlist:
if (p.match(mirror) == None):
- fsmirrors.append(mirror)
+ if os.access(mirror, os.F_OK):
+ self.output.write('get_filesystem_mirrors(): found file system mirror = %s\n' % mirror, 2)
+ fsmirrors.append(mirror)
+ else:
+ self.output.write('get_filesystem_mirrors(): ignoring non-accessible mirror = %s\n' % mirror, 2)
break
elif key is None:
break
except Exception:
fsmirrors = []
+ self.output.write('get_filesystem_mirrors(): fsmirrors = %s\n' % fsmirrors, 2)
return fsmirrors
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index b2a5fc7..38ca07a 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -303,14 +303,16 @@ class Deep(object):
self.output.write('\n') #this just makes output nicer
- #can't just return the dict.valuse, because we want the fastest mirror first...
+ #can't just return the dict.values, because we want the fastest mirror first...
keys = list(top_hosts.keys())
keys.sort()
rethosts = []
for key in keys:
+ #self.output.write('deeptest(): adding rethost %s, %s' % (key, top_hosts[key]), 2)
rethosts.append(top_hosts[key])
+ self.output.write('deeptest(): final rethost %s' % (rethosts), 2)
self.urls = rethosts
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-03-10 13:07 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-03-10 13:07 UTC (permalink / raw
To: gentoo-commits
commit: d7c287c09566926a441680597ccf060806b5ba33
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 10 13:01:09 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Mar 10 13:01:09 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=d7c287c0
change the default noise level on an ouptut message.
---
mirrorselect/main.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 48d74a7..3f90694 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -169,7 +169,7 @@ class MirrorSelect(object):
else:
var = 'GENTOO_MIRRORS'
- self.output.write('get_filesystem_mirrors(): config_path = %s\n' % config_path)
+ self.output.write('get_filesystem_mirrors(): config_path = %s\n' % config_path, 2)
try:
f = open(config_path,'r')
except IOError:
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-15 3:51 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-15 3:51 UTC (permalink / raw
To: gentoo-commits
commit: 15514cb04216b9fc259e905f453799bd8efa8483
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 15 03:50:09 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue Oct 15 03:50:09 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=15514cb0
fix missed import.
---
mirrorselect/mirrorparser3.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 976ff17..6a11855 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -84,6 +84,7 @@ class MirrorParser3:
return [url for url, args in list(self._dict.items())]
if __name__ == '__main__':
+ import sys
if int(sys.version[0]) == 3:
import urllib.request, urllib.parse, urllib.error
parser = MirrorParser3()
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-15 3:51 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-15 3:51 UTC (permalink / raw
To: gentoo-commits
commit: e5e82eb4ec1517144bf25d7ffc48b86109f60150
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 15 03:48:52 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue Oct 15 03:48:52 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=e5e82eb4
Add country and region filtering in the xml parsing.
Move protocol filtering to the new xml filtering system.
---
mirrorselect/main.py | 10 ++++++++++
mirrorselect/mirrorparser3.py | 21 +++++++++++++++++++--
mirrorselect/selectors.py | 8 +-------
3 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 3f90694..5ba9116 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -271,6 +271,16 @@ class MirrorSelect(object):
group.add_option(
"-6", "--ipv6", action="store_true", default=False,
help="only use IPv6")
+ group.add_option(
+ "-c", "--country", action="store", default=None,
+ help="only use mirrors from the specified country "
+ "NOTE: Names with a space must be quoted "
+ "eg.: -c 'South Korea'")
+ group.add_option(
+ "-R", "--region", action="store", default=None,
+ help="only use mirrors from the specified region"
+ "NOTE: Names with a space must be quoted"
+ "eg.: -r 'North America'")
group = parser.add_option_group("Other options")
group.add_option(
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index e34f9c2..976ff17 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -35,7 +35,15 @@ MIRRORS_3_XML = 'http://www.gentoo.org/main/en/mirrors3.xml'
MIRRORS_RSYNC_DATA = 'http://www.gentoo.org/main/en/mirrors-rsync-data.xml'
class MirrorParser3:
- def __init__(self):
+ def __init__(self, options=None):
+ self.filters = {}
+ for opt in ["country", "region"]:
+ value = getattr(options, opt)
+ if value is not None:
+ self.filters[opt] = value
+ for opt in ["ftp", "http"]:
+ if getattr(options, opt):
+ self.filters["proto"] = opt
self._reset()
def _reset(self):
@@ -51,7 +59,7 @@ class MirrorParser3:
name = e.text
if e.tag == 'uri':
uri = e.text
- self._dict[uri] = {
+ data = {
"name": name,
"country": mirrorgroup.get("countryname"),
"region": mirrorgroup.get("region"),
@@ -59,6 +67,15 @@ class MirrorParser3:
"ipv6": e.get("ipv6"),
"proto": e.get("protocol"),
}
+ if len(self.filters):
+ good = True
+ for f in self.filters:
+ if data[f] != self.filters[f]:
+ good = False
+ if good:
+ self._dict[uri] = data
+ else:
+ self._dict[uri] = data
def tuples(self):
return [(url, args) for url, args in list(self._dict.items())]
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 38ca07a..a603a47 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -58,19 +58,13 @@ class Extractor(object):
def __init__(self, list_url, options, output):
self.output = output
- parser = MirrorParser3()
+ parser = MirrorParser3(options)
self.hosts = []
hosts = self.getlist(parser, list_url)
self.output.write('Extractor(): fetched mirrors.xml,'
' %s hosts before filtering\n' % len(hosts), 2)
- if not options.rsync:
- if options.ftp:
- hosts = self.restrict_protocall('ftp', hosts)
- if options.http:
- hosts = self.restrict_protocall('http', hosts)
-
self.hosts = hosts
def restrict_protocall(self, prot, hosts):
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-15 7:27 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-15 7:27 UTC (permalink / raw
To: gentoo-commits
commit: 38b89be6d91c21123f986bb1aad96aaf42734dc3
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 15 07:23:11 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue Oct 15 07:23:11 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=38b89be6
Move the filtering code back into Extractor class
---
mirrorselect/mirrorparser3.py | 19 +--------------
mirrorselect/selectors.py | 56 ++++++++++++++++++++++++++-----------------
2 files changed, 35 insertions(+), 40 deletions(-)
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 6a11855..0db0edb 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -36,14 +36,6 @@ MIRRORS_RSYNC_DATA = 'http://www.gentoo.org/main/en/mirrors-rsync-data.xml'
class MirrorParser3:
def __init__(self, options=None):
- self.filters = {}
- for opt in ["country", "region"]:
- value = getattr(options, opt)
- if value is not None:
- self.filters[opt] = value
- for opt in ["ftp", "http"]:
- if getattr(options, opt):
- self.filters["proto"] = opt
self._reset()
def _reset(self):
@@ -59,7 +51,7 @@ class MirrorParser3:
name = e.text
if e.tag == 'uri':
uri = e.text
- data = {
+ self._dict[uri] = {
"name": name,
"country": mirrorgroup.get("countryname"),
"region": mirrorgroup.get("region"),
@@ -67,15 +59,6 @@ class MirrorParser3:
"ipv6": e.get("ipv6"),
"proto": e.get("protocol"),
}
- if len(self.filters):
- good = True
- for f in self.filters:
- if data[f] != self.filters[f]:
- good = False
- if good:
- self._dict[uri] = data
- else:
- self._dict[uri] = data
def tuples(self):
return [(url, args) for url, args in list(self._dict.items())]
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index a603a47..df9dcd7 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -58,32 +58,44 @@ class Extractor(object):
def __init__(self, list_url, options, output):
self.output = output
- parser = MirrorParser3(options)
+ filters = {}
+ for opt in ["country", "region"]:
+ value = getattr(options, opt)
+ if value is not None:
+ filters[opt] = value
+ self.output.print_info('Limiting test to %s = %s hosts. \n'
+ %(opt, value))
+ for opt in ["ftp", "http"]:
+ if getattr(options, opt):
+ filters["proto"] = opt
+ self.output.print_info('Limiting test to %s hosts. \n' % opt )
+ parser = MirrorParser3()
self.hosts = []
- hosts = self.getlist(parser, list_url)
- self.output.write('Extractor(): fetched mirrors.xml,'
- ' %s hosts before filtering\n' % len(hosts), 2)
-
- self.hosts = hosts
-
- def restrict_protocall(self, prot, hosts):
- """
- Removes hosts that are not of the specified type.
- "prot" must always be exactly 'http' or 'ftp'.
- """
- myhosts = []
+ self.unfiltered_hosts = self.getlist(parser, list_url)
- self.output.print_info('Limiting test to %s hosts. ' % prot )
+ self.hosts = self.filter_hosts(filters, self.unfiltered_hosts)
- for host in hosts:
- if host[0].startswith(prot):
- myhosts.append(host)
+ self.output.write('Extractor(): fetched mirrors.xml,'
+ ' %s hosts after filtering\n' % len(self.hosts), 2)
- self.output.write('%s of %s removed.\n' % (len(hosts) - len(myhosts),
- len(hosts)) )
- return myhosts
+ @staticmethod
+ def filter_hosts(filters, hosts):
+ """Filter the hosts to the criteria passed in
+ Return the filtered list
+ """
+ if not len(filters):
+ return hosts
+ filtered = []
+ for uri, data in hosts:
+ good = True
+ for f in filters:
+ if data[f] != filters[f]:
+ good = False
+ if good:
+ filtered.append((uri, data))
+ return filtered
def getlist(self, parser, url):
@@ -102,8 +114,8 @@ class Extractor(object):
pass
if len(parser.tuples()) == 0:
- self.output.print_err('Could not get mirror list. Check your internet'
- ' connection.')
+ self.output.print_err('Could not get mirror list. '
+ 'Check your internet connection.')
self.output.write(' Got %d mirrors.\n' % len(parser.tuples()))
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-15 14:39 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-15 14:39 UTC (permalink / raw
To: gentoo-commits
commit: 43f851f0e50b7bc0a97165b6f4b953cd9ae19085
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 15 14:36:49 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue Oct 15 14:36:49 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=43f851f0
add a missed continue to optimize the loop
---
mirrorselect/selectors.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index df9dcd7..1c2deb2 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -93,6 +93,7 @@ class Extractor(object):
for f in filters:
if data[f] != filters[f]:
good = False
+ continue
if good:
filtered.append((uri, data))
return filtered
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-15 14:39 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-15 14:39 UTC (permalink / raw
To: gentoo-commits
commit: 31dbe7baaa693dc5180359c34549a267072599d4
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 15 14:38:26 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue Oct 15 14:38:26 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=31dbe7ba
add an -a, --all_mirrors option to dump all search results
---
mirrorselect/main.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 034a8de..cfdcd55 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -242,6 +242,10 @@ class MirrorSelect(object):
group = parser.add_option_group("Main modes")
group.add_option(
+ "-a", "--all_mirrors", action="store_true", default=False,
+ help="This will present a list of all results"
+ "to make it possible to select mirrors you wish to use.")
+ group.add_option(
"-i", "--interactive", action="store_true", default=False,
help="Interactive Mode, this will present a list "
"to make it possible to select mirrors you wish to use.")
@@ -418,7 +422,10 @@ class MirrorSelect(object):
hosts = self.get_available_hosts(options)
- urls = self.select_urls(hosts, options)
+ if options.all_mirrors:
+ urls = [url for url, args in list(hosts)]
+ else:
+ urls = self.select_urls(hosts, options)
self.write_config(fsmirrors + urls, options.output,
config_path, options.rsync)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-15 14:39 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-15 14:39 UTC (permalink / raw
To: gentoo-commits
commit: 38500517d0396bd1cc94bf31293ba95ad2982dad
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 15 14:37:35 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue Oct 15 14:37:35 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=38500517
fix some missed spaces in the help message
---
mirrorselect/main.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 5ba9116..034a8de 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -278,8 +278,8 @@ class MirrorSelect(object):
"eg.: -c 'South Korea'")
group.add_option(
"-R", "--region", action="store", default=None,
- help="only use mirrors from the specified region"
- "NOTE: Names with a space must be quoted"
+ help="only use mirrors from the specified region "
+ "NOTE: Names with a space must be quoted "
"eg.: -r 'North America'")
group = parser.add_option_group("Other options")
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-15 22:43 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-15 22:43 UTC (permalink / raw
To: gentoo-commits
commit: 820ef8d8ba8bfe20eec5bc929381312f72e1c65f
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 15 22:42:37 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Tue Oct 15 22:42:37 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=820ef8d8
Add -a to rsync sanity checks.
---
mirrorselect/main.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index cfdcd55..45cc30b 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -268,7 +268,7 @@ class MirrorSelect(object):
group.add_option(
"-r", "--rsync", action="store_true", default=False,
help="rsync mode. Allows you to interactively select your"
- " rsync mirror. Requires -i to be used.")
+ " rsync mirror. Requires -i or -a to be used.")
group.add_option(
"-4", "--ipv4", action="store_true", default=False,
help="only use IPv4")
@@ -333,8 +333,8 @@ class MirrorSelect(object):
options.ipv6 = False
self.output.print_err('The --ipv6 option requires python ipv6 support')
- if options.rsync and not options.interactive:
- self.output.print_err('rsync servers can only be selected with -i')
+ if options.rsync and not (options.interactive or options.all_mirrors):
+ self.output.print_err('rsync servers can only be selected with -i or -a')
if options.interactive and (
options.deep or
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-16 8:36 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-16 8:36 UTC (permalink / raw
To: gentoo-commits
commit: 4e1962dd26d6729c4dbbb08674769cab1e16da28
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 16 08:34:06 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Oct 16 08:34:06 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=4e1962dd
fix some typos
---
mirrorselect/output.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/output.py b/mirrorselect/output.py
index d2ccc0a..31aa984 100644
--- a/mirrorselect/output.py
+++ b/mirrorselect/output.py
@@ -126,14 +126,14 @@ class Output(object):
self.file.flush()
def print_err(self, message, level=0):
- """prints an error message with a big red ERROR."""
+ """Prints an error message with a big red ERROR."""
if level <= self.verbosity:
self.file.write(self.red('\nERROR: ') + message + '\n')
self.file.flush()
sys.exit(1)
def write(self, message, level=1):
- """A wrapper arounf stderr.write, to enforce verbosity settings."""
+ """A wrapper around stderr.write, to enforce verbosity settings."""
if level <= self.verbosity:
self.file.write(message)
self.file.flush()
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-16 8:36 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-16 8:36 UTC (permalink / raw
To: gentoo-commits
commit: a56a352e92b8a5fc06e83e91aa97994071f1742b
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 16 08:33:40 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Oct 16 08:33:40 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=a56a352e
Fix IndexError for zero length search results.
---
mirrorselect/main.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 45cc30b..6a9c43d 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -427,6 +427,9 @@ class MirrorSelect(object):
else:
urls = self.select_urls(hosts, options)
- self.write_config(fsmirrors + urls, options.output,
- config_path, options.rsync)
-
+ if len(urls):
+ self.write_config(fsmirrors + urls, options.output,
+ config_path, options.rsync)
+ else:
+ self.output.write("No search results found. "
+ "Check your filter settings and re-run mirrorselect\n")
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-16 8:36 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-16 8:36 UTC (permalink / raw
To: gentoo-commits
commit: ac18bff94973434cfa4a2d09bf8f96b314588500
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 16 08:34:29 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Oct 16 08:34:29 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=ac18bff9
tweak message formatting
---
mirrorselect/selectors.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 1c2deb2..3e77221 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -63,7 +63,7 @@ class Extractor(object):
value = getattr(options, opt)
if value is not None:
filters[opt] = value
- self.output.print_info('Limiting test to %s = %s hosts. \n'
+ self.output.print_info('Limiting test to "%s=%s" hosts. \n'
%(opt, value))
for opt in ["ftp", "http"]:
if getattr(options, opt):
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-16 8:36 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-16 8:36 UTC (permalink / raw
To: gentoo-commits
commit: b14ff1ed1877ccd244d76ead7fd9b66fd8c6f9f1
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 16 08:35:46 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Oct 16 08:35:46 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=b14ff1ed
Force -o for -a + -r options to prevent multiple results being saved to the config.
---
mirrorselect/main.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 6a9c43d..5f7dc64 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -243,7 +243,7 @@ class MirrorSelect(object):
group = parser.add_option_group("Main modes")
group.add_option(
"-a", "--all_mirrors", action="store_true", default=False,
- help="This will present a list of all results"
+ help="This will present a list of all filtered search results "
"to make it possible to select mirrors you wish to use.")
group.add_option(
"-i", "--interactive", action="store_true", default=False,
@@ -335,6 +335,12 @@ class MirrorSelect(object):
if options.rsync and not (options.interactive or options.all_mirrors):
self.output.print_err('rsync servers can only be selected with -i or -a')
+ elif options.rsync and options.all_mirrors and not options.output:
+ # force output to screen.
+ # multiple uri's break normal sync operation
+ options.output = True
+ self.output.print_info("Forcing output to screen, as "
+ "multiple rsync uris are not supported\n")
if options.interactive and (
options.deep or
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-16 9:17 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-16 9:17 UTC (permalink / raw
To: gentoo-commits
commit: 3b46a1e2cc4179b47ce95b790431e22d81f209b3
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 16 09:16:53 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed Oct 16 09:16:53 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=3b46a1e2
Make the -r -a options select only the rotation server rather than force -o for screen output only.
---
mirrorselect/main.py | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 5f7dc64..61fd5c2 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -244,7 +244,9 @@ class MirrorSelect(object):
group.add_option(
"-a", "--all_mirrors", action="store_true", default=False,
help="This will present a list of all filtered search results "
- "to make it possible to select mirrors you wish to use.")
+ "to make it possible to select mirrors you wish to use. "
+ " For the -r, --rsync option, it will select the rotation server "
+ "only. As multiple rsync URL's are not supported.")
group.add_option(
"-i", "--interactive", action="store_true", default=False,
help="Interactive Mode, this will present a list "
@@ -335,12 +337,6 @@ class MirrorSelect(object):
if options.rsync and not (options.interactive or options.all_mirrors):
self.output.print_err('rsync servers can only be selected with -i or -a')
- elif options.rsync and options.all_mirrors and not options.output:
- # force output to screen.
- # multiple uri's break normal sync operation
- options.output = True
- self.output.print_info("Forcing output to screen, as "
- "multiple rsync uris are not supported\n")
if options.interactive and (
options.deep or
@@ -429,7 +425,9 @@ class MirrorSelect(object):
hosts = self.get_available_hosts(options)
if options.all_mirrors:
- urls = [url for url, args in list(hosts)]
+ urls = sorted([url for url, args in list(hosts)])
+ if options.rsync:
+ urls = [urls[0]]
else:
urls = self.select_urls(hosts, options)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-17 3:16 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-17 3:16 UTC (permalink / raw
To: gentoo-commits
commit: 2451fb7d7013e3dc18de9bd0cd9314168b75e05a
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 17 03:07:31 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Oct 17 03:07:31 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=2451fb7d
Move Extractor class to it's own file.
---
| 116 ++++++++++++++++++++++++++++++++++++++++++++++
mirrorselect/main.py | 3 +-
mirrorselect/selectors.py | 73 -----------------------------
3 files changed, 118 insertions(+), 74 deletions(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
new file mode 100644
index 0000000..8a3132c
--- /dev/null
+++ b/mirrorselect/extractor.py
@@ -0,0 +1,116 @@
+#-*- coding:utf-8 -*-
+
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2005-2012 Gentoo Foundation
+
+ Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
+ Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
+ Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+ Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
+
+import sys
+
+if int(sys.version[0]) == 3:
+ import urllib.request, urllib.parse, urllib.error
+ url_parse = urllib.parse
+ url_open = urllib.request.urlopen
+else:
+ import urllib
+ import urlparse
+ url_parse = urlparse.urlparse
+ url_open = urllib.urlopen
+
+
+from mirrorselect.mirrorparser3 import MirrorParser3
+
+
+class Extractor(object):
+ """The Extractor employs a MirrorParser3 object to get a list of valid
+ mirrors, and then filters them. Only the mirrors that should be tested, based on
+ user input are saved. They will be in the hosts attribute."""
+
+ def __init__(self, list_url, options, output):
+ self.output = output
+ filters = {}
+ for opt in ["country", "region"]:
+ value = getattr(options, opt)
+ if value is not None:
+ filters[opt] = value
+ self.output.print_info('Limiting test to "%s=%s" hosts. \n'
+ %(opt, value))
+ for opt in ["ftp", "http"]:
+ if getattr(options, opt):
+ filters["proto"] = opt
+ self.output.print_info('Limiting test to %s hosts. \n' % opt )
+ parser = MirrorParser3()
+ self.hosts = []
+
+ self.unfiltered_hosts = self.getlist(parser, list_url)
+
+ self.hosts = self.filter_hosts(filters, self.unfiltered_hosts)
+
+ self.output.write('Extractor(): fetched mirrors.xml,'
+ ' %s hosts after filtering\n' % len(self.hosts), 2)
+
+
+ @staticmethod
+ def filter_hosts(filters, hosts):
+ """Filter the hosts to the criteria passed in
+ Return the filtered list
+ """
+ if not len(filters):
+ return hosts
+ filtered = []
+ for uri, data in hosts:
+ good = True
+ for f in filters:
+ if data[f] != filters[f]:
+ good = False
+ continue
+ if good:
+ filtered.append((uri, data))
+ return filtered
+
+
+ def getlist(self, parser, url):
+ """
+ Uses the supplied parser to get a list of urls.
+ Takes a parser object, url, and filering options.
+ """
+
+ self.output.write('getlist(): fetching ' + url + '\n', 2)
+
+ self.output.print_info('Downloading a list of mirrors...')
+
+ try:
+ parser.parse(url_open(url).read())
+ except EnvironmentError:
+ pass
+
+ if len(parser.tuples()) == 0:
+ self.output.print_err('Could not get mirror list. '
+ 'Check your internet connection.')
+
+ self.output.write(' Got %d mirrors.\n' % len(parser.tuples()))
+
+ return parser.tuples()
+
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index ca7d794..d309624 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -43,7 +43,8 @@ import sys
from optparse import OptionParser
from mirrorselect.mirrorparser3 import MIRRORS_3_XML, MIRRORS_RSYNC_DATA
from mirrorselect.output import Output, ColoredFormatter
-from mirrorselect.selectors import Deep, Shallow, Extractor, Interactive
+from mirrorselect.selectors import Deep, Shallow, Interactive
+from mirrorselect.extractor import Extractor
from mirrorselect.version import version
# eprefix compatibility
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 3e77221..9d718fd 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -47,82 +47,9 @@ else:
url_open = urllib.urlopen
-from mirrorselect.mirrorparser3 import MirrorParser3
from mirrorselect.output import encoder, get_encoding, decode_selection
-class Extractor(object):
- """The Extractor employs a MirrorParser3 object to get a list of valid
- mirrors, and then filters them. Only the mirrors that should be tested, based on
- user input are saved. They will be in the hosts attribute."""
-
- def __init__(self, list_url, options, output):
- self.output = output
- filters = {}
- for opt in ["country", "region"]:
- value = getattr(options, opt)
- if value is not None:
- filters[opt] = value
- self.output.print_info('Limiting test to "%s=%s" hosts. \n'
- %(opt, value))
- for opt in ["ftp", "http"]:
- if getattr(options, opt):
- filters["proto"] = opt
- self.output.print_info('Limiting test to %s hosts. \n' % opt )
- parser = MirrorParser3()
- self.hosts = []
-
- self.unfiltered_hosts = self.getlist(parser, list_url)
-
- self.hosts = self.filter_hosts(filters, self.unfiltered_hosts)
-
- self.output.write('Extractor(): fetched mirrors.xml,'
- ' %s hosts after filtering\n' % len(self.hosts), 2)
-
-
- @staticmethod
- def filter_hosts(filters, hosts):
- """Filter the hosts to the criteria passed in
- Return the filtered list
- """
- if not len(filters):
- return hosts
- filtered = []
- for uri, data in hosts:
- good = True
- for f in filters:
- if data[f] != filters[f]:
- good = False
- continue
- if good:
- filtered.append((uri, data))
- return filtered
-
-
- def getlist(self, parser, url):
- """
- Uses the supplied parser to get a list of urls.
- Takes a parser object, url, and filering options.
- """
-
- self.output.write('getlist(): fetching ' + url + '\n', 2)
-
- self.output.print_info('Downloading a list of mirrors...')
-
- try:
- parser.parse(url_open(url).read())
- except EnvironmentError:
- pass
-
- if len(parser.tuples()) == 0:
- self.output.print_err('Could not get mirror list. '
- 'Check your internet connection.')
-
- self.output.write(' Got %d mirrors.\n' % len(parser.tuples()))
-
- return parser.tuples()
-
-
class Shallow(object):
"""handles rapid server selection via netselect"""
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-17 3:16 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-17 3:16 UTC (permalink / raw
To: gentoo-commits
commit: cad72683e6b2d347e0c45e98b9bd5a1807773838
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 17 03:04:55 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Oct 17 03:04:55 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=cad72683
remove unused unicode variable
---
mirrorselect/main.py | 6 ------
mirrorselect/output.py | 6 ------
2 files changed, 12 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 61fd5c2..ca7d794 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -62,12 +62,6 @@ if "GENTOO_PORTAGE_EPREFIX" in EPREFIX:
EPREFIX = ''
-if sys.hexversion >= 0x3000000:
- _unicode = str
-else:
- _unicode = unicode
-
-
class MirrorSelect(object):
'''Main operational class'''
diff --git a/mirrorselect/output.py b/mirrorselect/output.py
index 31aa984..5854000 100644
--- a/mirrorselect/output.py
+++ b/mirrorselect/output.py
@@ -37,12 +37,6 @@ import locale
from optparse import IndentedHelpFormatter
-if sys.hexversion >= 0x3000000:
- _unicode = str
-else:
- _unicode = unicode
-
-
def encoder(text, _encoding_):
return codecs.encode(text, _encoding_, 'replace')
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-17 3:16 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-17 3:16 UTC (permalink / raw
To: gentoo-commits
commit: ae600d2d704209985cc50b914ab47982a842d87b
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 17 03:11:21 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Oct 17 03:11:21 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=ae600d2d
Add repos.conf support.
Move config file actions to their own file.
---
mirrorselect/configs.py | 177 ++++++++++++++++++++++++++++++++++++++++++++++++
mirrorselect/main.py | 168 ++++++++++++++-------------------------------
2 files changed, 226 insertions(+), 119 deletions(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
new file mode 100644
index 0000000..3534b72
--- /dev/null
+++ b/mirrorselect/configs.py
@@ -0,0 +1,177 @@
+#-*- coding:utf-8 -*-
+
+
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2005-2012 Gentoo Foundation
+
+ Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
+ Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
+ Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+ Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
+
+
+from __future__ import print_function
+
+
+import os
+import re
+import shlex
+import shutil
+import string
+import sys
+
+
+def get_make_conf_path(EPREFIX):
+ # try the newer make.conf location
+ config_path = EPREFIX + '/etc/portage/make.conf'
+ if not os.access(config_path, os.F_OK):
+ # check if the old location is what is used
+ if os.access(EPREFIX + '/etc/make.conf', os.F_OK):
+ config_path = EPREFIX + '/etc/make.conf'
+ return config_path
+
+
+def write_make_conf(output, config_path, var, mirror_string):
+ """Write the make.conf target changes
+
+ @param output: file, or output to print messages to
+ @param mirror_string: "var='hosts'" string to write
+ @param config_path; string
+ """
+ output.write('\n')
+ output.print_info('Modifying %s with new mirrors...\n' % config_path)
+ try:
+ config = open(config_path, 'r')
+ output.write('\tReading make.conf\n')
+ lines = config.readlines()
+ config.close()
+ output.write('\tMoving to %s.backup\n' % config_path)
+ shutil.move(config_path, config_path + '.backup')
+ except IOError:
+ lines = []
+
+ regex = re.compile('^%s=.*' % var)
+ for line in lines:
+ if regex.match(line):
+ lines.remove(line)
+
+ lines.append(mirror_string)
+
+ output.write('\tWriting new %s\n' % config_path)
+
+ config = open(config_path, 'w')
+
+ for line in lines:
+ config.write(line)
+ config.write('\n')
+ config.close()
+
+ output.print_info('Done.\n')
+ sys.exit(0)
+
+
+def write_repos_conf(output, config_path, var, value):
+ """Saves the new var value to a ConfigParser style file
+
+ @param output: file, or output to print messages to
+ @param config_path; string
+ @param var: string; the variable to save teh value to.
+ @param value: string, the value to set var to
+ """
+ try:
+ from configparser import ConfigParser
+ except ImportError:
+ from ConfigParser import ConfigParser
+ config = ConfigParser()
+ config.read(config_path)
+ if config.has_option('gentoo', var):
+ config.set('gentoo', var, value)
+ with open(config_path, 'wb') as configfile:
+ config.write(configfile)
+ else:
+ output.print_err("write_repos_conf(): Failed to find section 'gentoo',"
+ " variable: %s\nChanges NOT SAVED" %var)
+
+
+def get_filesystem_mirrors(output, config_path, sync=False):
+ """Read the current mirrors and retain mounted filesystems mirrors
+
+ @param config_path: string
+ @param sync: boolean, used to switch between SYNC and GENTOO_MIRRORS
+ make.conf variable target
+ @rtype list
+ """
+ fsmirrors = []
+
+ if sync:
+ var = 'SYNC'
+ else:
+ var = 'GENTOO_MIRRORS'
+
+ output.write('get_filesystem_mirrors(): config_path = %s\n' % config_path, 2)
+ try:
+ f = open(config_path,'r')
+ except IOError:
+ return fsmirrors
+
+ """ Search for 'var' in make.conf and extract value """
+ try:
+ lex = shlex.shlex(f, posix=True)
+ lex.wordchars = string.digits+string.letters+"~!@#$%*_\:;?,./-+{}"
+ lex.quotes = "\"'"
+ while 1:
+ key = lex.get_token()
+ #output.write('get_filesystem_mirrors(): processing key = %s\n' % key, 2)
+
+ if key == var:
+ equ = lex.get_token()
+
+ if (equ == ''):
+ break
+ elif (equ != '='):
+ break
+
+ val = lex.get_token()
+ if val is None:
+ break
+
+ """ Look for mounted filesystem in value """
+ mirrorlist = val.rsplit()
+ output.write('get_filesystem_mirrors(): mirrorlist = %s\n' % mirrorlist, 2)
+ p = re.compile('rsync://|http://|ftp://', re.IGNORECASE)
+ for mirror in mirrorlist:
+ if (p.match(mirror) == None):
+ if os.access(mirror, os.F_OK):
+ output.write('get_filesystem_mirrors(): found file system mirror = %s\n' % mirror, 2)
+ fsmirrors.append(mirror)
+ else:
+ output.write('get_filesystem_mirrors(): ignoring non-accessible mirror = %s\n' % mirror, 2)
+ break
+ elif key is None:
+ break
+ except Exception:
+ fsmirrors = []
+
+ output.write('get_filesystem_mirrors(): fsmirrors = %s\n' % fsmirrors, 2)
+ return fsmirrors
+
+
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index d309624..8a1bfca 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -34,17 +34,15 @@ from __future__ import print_function
import os
-import re
-import shlex
-import shutil
import socket
-import string
import sys
from optparse import OptionParser
from mirrorselect.mirrorparser3 import MIRRORS_3_XML, MIRRORS_RSYNC_DATA
from mirrorselect.output import Output, ColoredFormatter
from mirrorselect.selectors import Deep, Shallow, Interactive
from mirrorselect.extractor import Extractor
+from mirrorselect.configs import (get_make_conf_path, write_make_conf,
+ write_repos_conf, get_filesystem_mirrors)
from mirrorselect.version import version
# eprefix compatibility
@@ -93,123 +91,44 @@ class MirrorSelect(object):
return None
- def write_config(self, hosts, out, config_path, sync=False):
- """Writes the make.conf style string to the given file, or to stdout.
+ def change_config(self, hosts, out, config_path, sync=False):
+ """Writes the config changes to the given file, or to stdout.
@param hosts: list of host urls to write
@param out: boolean, used to redirect output to stdout
@param config_path; string
- @param sync: boolean, used to switch between SYNC and GENTOO_MIRRORS
- make.conf variable target
+ @param sync: boolean, used to switch between sync-uri repos.conf target,
+ SYNC and GENTOO_MIRRORS make.conf variable target
"""
if sync:
- var = 'SYNC'
+ if 'repos.conf' in config_path:
+ var = "sync-uri"
+ else:
+ var = 'SYNC'
else:
var = 'GENTOO_MIRRORS'
if hasattr(hosts[0], 'decode'):
hosts = [x.decode('utf-8') for x in hosts]
- mirror_string = '%s="%s"' % (var, ' '.join(hosts))
+ if var == "sync-uri" and out:
+ mirror_string = '%s = %s' % (var, ' '.join(hosts))
+ else:
+ mirror_string = '%s="%s"' % (var, ' '.join(hosts))
if out:
- print()
- print(mirror_string)
- sys.exit(0)
-
- self.output.write('\n')
- self.output.print_info('Modifying %s with new mirrors...\n' % config_path)
- try:
- config = open(config_path, 'r')
- self.output.write('\tReading make.conf\n')
- lines = config.readlines()
- config.close()
- self.output.write('\tMoving to %s.backup\n' % config_path)
- shutil.move(config_path, config_path + '.backup')
- except IOError:
- lines = []
-
- regex = re.compile('^%s=.*' % var)
- for line in lines:
- if regex.match(line):
- lines.remove(line)
-
- lines.append(mirror_string)
-
- self.output.write('\tWriting new %s\n' % config_path)
-
- config = open(config_path, 'w')
-
- for line in lines:
- config.write(line)
- config.write('\n')
- config.close()
-
- self.output.print_info('Done.\n')
- sys.exit(0)
-
-
- def get_filesystem_mirrors(self, config_path, sync=False):
- """Read the current mirrors and retain mounted filesystems mirrors
-
- @param config_path: string
- @param sync: boolean, used to switch between SYNC and GENTOO_MIRRORS
- make.conf variable target
- @rtype list
- """
- fsmirrors = []
-
- if sync:
- var = 'SYNC'
+ self.write_to_output(mirror_string)
+ elif var == "sync-uri":
+ write_repos_conf(self.output, config_path, var, ' '.join(hosts))
else:
- var = 'GENTOO_MIRRORS'
+ write_make_conf(self.output, config_path, var, mirror_string)
- self.output.write('get_filesystem_mirrors(): config_path = %s\n' % config_path, 2)
- try:
- f = open(config_path,'r')
- except IOError:
- return fsmirrors
-
- """ Search for 'var' in make.conf and extract value """
- try:
- lex = shlex.shlex(f, posix=True)
- lex.wordchars = string.digits+string.letters+"~!@#$%*_\:;?,./-+{}"
- lex.quotes = "\"'"
- while 1:
- key = lex.get_token()
- #self.output.write('get_filesystem_mirrors(): processing key = %s\n' % key, 2)
-
- if key == var:
- equ = lex.get_token()
-
- if (equ == ''):
- break
- elif (equ != '='):
- break
-
- val = lex.get_token()
- if val is None:
- break
-
- """ Look for mounted filesystem in value """
- mirrorlist = val.rsplit()
- self.output.write('get_filesystem_mirrors(): mirrorlist = %s\n' % mirrorlist, 2)
- p = re.compile('rsync://|http://|ftp://', re.IGNORECASE)
- for mirror in mirrorlist:
- if (p.match(mirror) == None):
- if os.access(mirror, os.F_OK):
- self.output.write('get_filesystem_mirrors(): found file system mirror = %s\n' % mirror, 2)
- fsmirrors.append(mirror)
- else:
- self.output.write('get_filesystem_mirrors(): ignoring non-accessible mirror = %s\n' % mirror, 2)
- break
- elif key is None:
- break
- except Exception:
- fsmirrors = []
-
- self.output.write('get_filesystem_mirrors(): fsmirrors = %s\n' % fsmirrors, 2)
- return fsmirrors
+
+ @staticmethod
+ def write_to_output(mirror_string):
+ print()
+ print(mirror_string)
+ sys.exit(0)
def _parse_args(self, argv, config_path):
@@ -362,8 +281,10 @@ class MirrorSelect(object):
@rtype: list
'''
if options.rsync:
+ self.output.write("using url: %s" % MIRRORS_RSYNC_DATA, 2)
hosts = Extractor(MIRRORS_RSYNC_DATA, options, self.output).hosts
else:
+ self.output.write("using url: %s" % MIRRORS_3_XML, 2)
hosts = Extractor(MIRRORS_3_XML, options, self.output).hosts
return hosts
@@ -388,21 +309,22 @@ class MirrorSelect(object):
return selector.urls
- @staticmethod
- def get_make_conf_path():
- '''Checks for the existance of make.conf in /etc/portage/
+ def get_conf_path(self, rsync=False):
+ '''Checks for the existance of repos.conf or make.conf in /etc/portage/
Failing that it checks for it in /etc/
Failing in /etc/ it defaults to /etc/portage/make.conf
@rtype: string
'''
- # start with the new location
- config_path = EPREFIX + '/etc/portage/make.conf'
- if not os.access(config_path, os.F_OK):
- # check if the old location is what is used
- if os.access(EPREFIX + '/etc/make.conf', os.F_OK):
- config_path = EPREFIX + '/etc/make.conf'
- return config_path
+ if rsync:
+ # startwith repos.conf
+ config_path = EPREFIX + '/etc/portage/repos.conf/gentoo.conf'
+ if not os.access(config_path, os.F_OK):
+ self.output.write("Failed access to gentoo.conf: "
+ "%s\n" % os.access(config_path, os.F_OK), 2)
+ return get_make_conf_path(EPREFIX)
+ return config_path
+ return get_make_conf_path(EPREFIX)
def main(self, argv):
@@ -410,12 +332,20 @@ class MirrorSelect(object):
@param argv: list of command line arguments to parse
"""
- config_path = self.get_make_conf_path()
-
+ config_path = self.get_conf_path()
options = self._parse_args(argv, config_path)
self.output.verbosity = options.verbosity
+ self.output.write("main(); config_path = %s\n" % config_path, 2)
+
+ # reset config_path to find repos.conf/gentoo.conf if it exists
+ if options.rsync:
+ config_path = self.get_conf_path(options.rsync)
+ self.output.write("main(); reset config_path = %s\n" % config_path, 2)
+ else:
+ self.output.write("main(); rsync = %s" % str(options.rsync),2)
- fsmirrors = self.get_filesystem_mirrors(config_path, options.rsync)
+ fsmirrors = get_filesystem_mirrors(self.output,
+ config_path, options.rsync)
hosts = self.get_available_hosts(options)
@@ -427,7 +357,7 @@ class MirrorSelect(object):
urls = self.select_urls(hosts, options)
if len(urls):
- self.write_config(fsmirrors + urls, options.output,
+ self.change_config(fsmirrors + urls, options.output,
config_path, options.rsync)
else:
self.output.write("No search results found. "
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-17 6:57 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-17 6:57 UTC (permalink / raw
To: gentoo-commits
commit: 168e7bf0f5ed0adf5acbf09bdd6c4181f5bcdcb8
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 17 06:57:04 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Oct 17 06:57:04 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=168e7bf0
Fix py3 compatibilty.
---
mirrorselect/configs.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
index 3534b72..8a83287 100644
--- a/mirrorselect/configs.py
+++ b/mirrorselect/configs.py
@@ -105,7 +105,7 @@ def write_repos_conf(output, config_path, var, value):
config.read(config_path)
if config.has_option('gentoo', var):
config.set('gentoo', var, value)
- with open(config_path, 'wb') as configfile:
+ with open(config_path, 'w') as configfile:
config.write(configfile)
else:
output.print_err("write_repos_conf(): Failed to find section 'gentoo',"
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-17 14:26 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-17 14:26 UTC (permalink / raw
To: gentoo-commits
commit: e8bd46886261aca42917e0d627a0a49b337b02f5
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 17 14:25:38 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Oct 17 14:25:38 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=e8bd4688
Fix bug 483232, incorrect urlparse import for python3.
---
mirrorselect/selectors.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 9d718fd..230ca15 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -38,7 +38,7 @@ import hashlib
if int(sys.version[0]) == 3:
import urllib.request, urllib.parse, urllib.error
- url_parse = urllib.parse
+ url_parse = urllib.parse.urlparse
url_open = urllib.request.urlopen
else:
import urllib
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-18 6:46 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-18 6:46 UTC (permalink / raw
To: gentoo-commits
commit: f727908fd506c0602be6bc8b51d844b978469555
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 18 02:09:49 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Oct 18 02:09:49 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=f727908f
Improve python version testing for imports.
Use Arfrever's suggested code.
---
| 2 +-
mirrorselect/mirrorparser3.py | 2 +-
mirrorselect/selectors.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 8a3132c..2b7d049 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -29,7 +29,7 @@ Distributed under the terms of the GNU General Public License v2
import sys
-if int(sys.version[0]) == 3:
+if sys.version_info[0] >= 3:
import urllib.request, urllib.parse, urllib.error
url_parse = urllib.parse
url_open = urllib.request.urlopen
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 0db0edb..6b06a9c 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -68,7 +68,7 @@ class MirrorParser3:
if __name__ == '__main__':
import sys
- if int(sys.version[0]) == 3:
+ if sys.version_info[0] >= 3:
import urllib.request, urllib.parse, urllib.error
parser = MirrorParser3()
parser.parse(urllib.request.urlopen(MIRRORS_3_XML).read())
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 230ca15..d72fd23 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -36,7 +36,7 @@ import sys
import time
import hashlib
-if int(sys.version[0]) == 3:
+if sys.version_info[0] >= 3:
import urllib.request, urllib.parse, urllib.error
url_parse = urllib.parse.urlparse
url_open = urllib.request.urlopen
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-18 6:46 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-18 6:46 UTC (permalink / raw
To: gentoo-commits
commit: db2455365da647e2e03961724667d9fe4f95850c
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 18 02:06:27 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Oct 18 02:06:27 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=db245536
get_filesystem_mirrors() cleanup & fixes
Fix py3 string.letters error
Unwrap function in try: except:, narrow it to only lex.get_token()
Cleanup sloppy/redundant code.
Thanks to Arfrever's help.
---
mirrorselect/configs.py | 78 ++++++++++++++++++++++++++++---------------------
1 file changed, 44 insertions(+), 34 deletions(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
index 8a83287..9fa4f2c 100644
--- a/mirrorselect/configs.py
+++ b/mirrorselect/configs.py
@@ -40,6 +40,12 @@ import string
import sys
+try: # py2
+ letters = string.letters
+except AttributeError: # py3
+ letters = string.ascii_letters
+
+
def get_make_conf_path(EPREFIX):
# try the newer make.conf location
config_path = EPREFIX + '/etc/portage/make.conf'
@@ -120,6 +126,16 @@ def get_filesystem_mirrors(output, config_path, sync=False):
make.conf variable target
@rtype list
"""
+
+ def get_token(lex):
+ '''internal function for getting shlex tokens
+ '''
+ try:
+ val = lex.get_token()
+ except ValueError:
+ val = None
+ return val
+
fsmirrors = []
if sync:
@@ -134,42 +150,36 @@ def get_filesystem_mirrors(output, config_path, sync=False):
return fsmirrors
""" Search for 'var' in make.conf and extract value """
- try:
- lex = shlex.shlex(f, posix=True)
- lex.wordchars = string.digits+string.letters+"~!@#$%*_\:;?,./-+{}"
- lex.quotes = "\"'"
- while 1:
- key = lex.get_token()
- #output.write('get_filesystem_mirrors(): processing key = %s\n' % key, 2)
-
- if key == var:
- equ = lex.get_token()
-
- if (equ == ''):
- break
- elif (equ != '='):
- break
-
- val = lex.get_token()
- if val is None:
- break
-
- """ Look for mounted filesystem in value """
- mirrorlist = val.rsplit()
- output.write('get_filesystem_mirrors(): mirrorlist = %s\n' % mirrorlist, 2)
- p = re.compile('rsync://|http://|ftp://', re.IGNORECASE)
- for mirror in mirrorlist:
- if (p.match(mirror) == None):
- if os.access(mirror, os.F_OK):
- output.write('get_filesystem_mirrors(): found file system mirror = %s\n' % mirror, 2)
- fsmirrors.append(mirror)
- else:
- output.write('get_filesystem_mirrors(): ignoring non-accessible mirror = %s\n' % mirror, 2)
+ lex = shlex.shlex(f, posix=True)
+ lex.wordchars = string.digits+letters+"~!@#$%*_\:;?,./-+{}"
+ lex.quotes = "\"'"
+ p = re.compile('rsync://|http://|ftp://', re.IGNORECASE)
+ while 1:
+ key = get_token(lex)
+ #output.write('get_filesystem_mirrors(): processing key = %s\n' % key, 2)
+
+ if key == var:
+ equ = get_token(lex)
+ if (equ != '='):
break
- elif key is None:
+
+ val = get_token(lex)
+ if val is None:
break
- except Exception:
- fsmirrors = []
+
+ """ Look for mounted filesystem in value """
+ mirrorlist = val.rsplit()
+ output.write('get_filesystem_mirrors(): mirrorlist = %s\n' % mirrorlist, 2)
+ for mirror in mirrorlist:
+ if (p.match(mirror) == None):
+ if os.access(mirror, os.F_OK):
+ output.write('get_filesystem_mirrors(): found file system mirror = %s\n' % mirror, 2)
+ fsmirrors.append(mirror)
+ else:
+ output.write('get_filesystem_mirrors(): ignoring non-accessible mirror = %s\n' % mirror, 2)
+ break
+ elif key is None:
+ break
output.write('get_filesystem_mirrors(): fsmirrors = %s\n' % fsmirrors, 2)
return fsmirrors
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-18 6:46 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-18 6:46 UTC (permalink / raw
To: gentoo-commits
commit: 1be270280f8ca26ff76dfc3b86a8ddabda09d4f6
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 18 06:42:20 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Oct 18 06:42:20 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=1be27028
Add -f, --file & -m, --md5 options to override mirrorselect-test file used.
---
mirrorselect/main.py | 11 +++++++++++
mirrorselect/selectors.py | 14 ++++++++++----
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 8a1bfca..5844a65 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -229,6 +229,17 @@ class MirrorSelect(object):
group.add_option(
"-q", "--quiet", action="store_const", const=0, dest="verbosity",
help="Quiet mode")
+ group.add_option(
+ "-f", "--file", action="store", default='mirrorselect-test',
+ help="An alternate file to download for deep testing. "
+ "Please choose the file carefully as to not abuse the system "
+ "by selecting an overly large size file. You must also "
+ " use the -m, --md5 option.")
+ group.add_option(
+ "-m", "--md5", action="store", default='bdf077b2e683c506bf9e8f2494eeb044',
+ help="An alternate file md5sum value used to compare the downloaded "
+ "file against for deep testing.")
+
if len(argv) == 1:
parser.print_help()
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index d72fd23..2c7bac8 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -191,6 +191,8 @@ class Deep(object):
self._dns_timeout = options.timeout
self._connect_timeout = options.timeout
self._download_timeout = options.timeout
+ self.test_file = options.file
+ self.test_md5 = options.md5
addr_families = []
if options.ipv4:
@@ -221,8 +223,12 @@ class Deep(object):
for host in hosts:
prog += 1
- self.output.print_info('Downloading 100k files from each mirror... [%s of %s]'\
- % (prog, num_hosts) )
+ if self.test_file is not 'mirrorselect-test':
+ self.output.print_info('Downloading %s files from each mirror... [%s of %s]'\
+ % (self.test_file, prog, num_hosts) )
+ else:
+ self.output.print_info('Downloading 100k files from each mirror... [%s of %s]'\
+ % (prog, num_hosts) )
mytime, ignore = self.deeptime(host, maxtime)
@@ -259,9 +265,9 @@ class Deep(object):
self.output.write('\n_deeptime(): maxtime is %s\n' % maxtime, 2)
if url.endswith('/'): #append the path to the testfile to the URL
- url = url + 'distfiles/mirrorselect-test'
+ url = url + 'distfiles/' + self.test_file
else:
- url = url + '/distfiles/mirrorselect-test'
+ url = url + '/distfiles/' + self.test_file
url_parts = url_parse(url)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-18 6:46 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-18 6:46 UTC (permalink / raw
To: gentoo-commits
commit: 17023eb1d83d99cc92d2d9e09bf8e2c47227d88c
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 18 06:45:23 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Oct 18 06:45:23 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=17023eb1
refactor Deep classes deeptime()
fixes many errors, including dns errors on some systems,
Adds improved debug reporting.
Adds total download failures output to the results.
---
mirrorselect/selectors.py | 65 +++++++++++++++++++++++++++--------------------
1 file changed, 37 insertions(+), 28 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 2c7bac8..4da6156 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -39,11 +39,13 @@ import hashlib
if sys.version_info[0] >= 3:
import urllib.request, urllib.parse, urllib.error
url_parse = urllib.parse.urlparse
+ url_unparse = urllib.parse.urlunparse
url_open = urllib.request.urlopen
else:
import urllib
import urlparse
url_parse = urlparse.urlparse
+ url_unparse = urlparse.urlunparse
url_open = urllib.urlopen
@@ -200,9 +202,7 @@ class Deep(object):
elif options.ipv6:
addr_families.append(socket.AF_INET6)
else:
- addr_families.append(socket.AF_INET)
- if socket.has_ipv6:
- addr_families.append(socket.AF_INET6)
+ addr_families.append(socket.AF_UNSPEC)
self._addr_families = addr_families
@@ -219,6 +219,7 @@ class Deep(object):
maxtime = self._download_timeout
hosts = [host[0] for host in self._hosts]
num_hosts = len(hosts)
+ self.dl_failures = 0
for host in hosts:
@@ -252,7 +253,9 @@ class Deep(object):
#self.output.write('deeptest(): adding rethost %s, %s' % (key, top_hosts[key]), 2)
rethosts.append(top_hosts[key])
- self.output.write('deeptest(): final rethost %s' % (rethosts), 2)
+ self.output.write('deeptest(): final rethost %s\n' % (rethosts), 2)
+ self.output.write('deeptest(): final md5 failures %s of %s\n'
+ % (self.dl_failures, num_hosts), 2)
self.urls = rethosts
@@ -280,40 +283,38 @@ class Deep(object):
signal.signal(signal.SIGALRM, timeout_handler)
ips = []
- for family in self._addr_families:
- ipv6 = family == socket.AF_INET6
+ for addr_family in self._addr_families:
try:
- try:
- signal.alarm(self._dns_timeout)
- for family, socktype, proto, canonname, sockaddr in \
- socket.getaddrinfo(url_parts.hostname, None,
- family, socket.SOCK_STREAM):
- ip = sockaddr[0]
- if ipv6:
- ip = "[%s]" % ip
- ips.append(ip)
- finally:
- signal.alarm(0)
+ signal.alarm(self._dns_timeout)
+ for result in socket.getaddrinfo(url_parts.hostname, None,
+ addr_family, socket.SOCK_STREAM, 0, socket.AI_ADDRCONFIG):
+ family, _, __, ___, sockaddr = result
+ ip = sockaddr[0]
+ if family == socket.AF_INET6:
+ ip = "[%s]" % ip
+ ips.append(ip)
except socket.error as e:
- self.output.write('deeptime(): dns error for host %s: %s\n' % \
- (url_parts.hostname, e), 2)
+ self.output.write('deeptime(): dns error for host %s: %s\n' % (url_parts.hostname, e), 2)
except TimeoutException:
- self.output.write('deeptime(): dns timeout for host %s\n' % \
- (url_parts.hostname,), 2)
+ self.output.write('deeptime(): dns timeout for host %s\n' % url_parts.hostname, 2)
+ finally:
+ signal.alarm(0)
if not ips:
- self.output.write('deeptime(): unable to resolve ip for host %s\n' % \
- (url_parts.hostname,), 2)
+ self.output.write('deeptime(): unable to resolve ip for host %s\n' % url_parts.hostname, 2)
return (None, True)
delta = 0
f = None
for ip in ips:
+ test_parts = url_parts._replace(netloc=ip)
+ test_url = url_unparse(test_parts)
+ self.output.write('deeptime(): testing url: %s\n' % test_url, 2)
try:
try:
signal.alarm(self._connect_timeout)
- f = url_open(url)
+ f = url_open(test_url)
break
finally:
signal.alarm(0)
@@ -324,7 +325,7 @@ class Deep(object):
except TimeoutException:
self.output.write(('deeptime(): connection to host %s ' + \
'timed out for ip %s\n') % \
- (url_parts.hostname, ip), 2)
+ (test_parts.hostname, ip), 2)
if f is None:
self.output.write('deeptime(): unable to ' + \
@@ -355,13 +356,21 @@ class Deep(object):
try:
signal.alarm(int(math.ceil(maxtime)))
stime = time.time()
- f = url_open(url)
+ f = url_open(test_url)
- if hashlib.md5(f.read()).hexdigest() != "bdf077b2e683c506bf9e8f2494eeb044":
- return (None, True)
+ md5 = hashlib.md5(f.read()).hexdigest()
delta = time.time() - stime
f.close()
+ if md5 != self.test_md5:
+ self.output.write(
+ "deeptime(): md5sum error for file: %s\n" % self.test_file +
+ " expected: %s\n" % self.test_md5 +
+ " got.....: %s\n" % md5 +
+ " host....: %s, %s\n" % (url_parts.hostname, ip))
+ self.dl_failures += 1
+ return (None, True)
+
finally:
signal.alarm(0)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-18 14:26 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-18 14:26 UTC (permalink / raw
To: gentoo-commits
commit: 6a703ce2d162d96df2a08124e3d3ca3d2f657f07
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 18 06:45:23 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Oct 18 14:14:42 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=6a703ce2
refactor Deep classes deeptime()
Thanks to Douglas Freed, fixed many errors,
including dns errors on my system.
Adds improved debug reporting.
Adds total download failures output to the results.
---
mirrorselect/selectors.py | 65 +++++++++++++++++++++++++++--------------------
1 file changed, 37 insertions(+), 28 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 2c7bac8..4da6156 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -39,11 +39,13 @@ import hashlib
if sys.version_info[0] >= 3:
import urllib.request, urllib.parse, urllib.error
url_parse = urllib.parse.urlparse
+ url_unparse = urllib.parse.urlunparse
url_open = urllib.request.urlopen
else:
import urllib
import urlparse
url_parse = urlparse.urlparse
+ url_unparse = urlparse.urlunparse
url_open = urllib.urlopen
@@ -200,9 +202,7 @@ class Deep(object):
elif options.ipv6:
addr_families.append(socket.AF_INET6)
else:
- addr_families.append(socket.AF_INET)
- if socket.has_ipv6:
- addr_families.append(socket.AF_INET6)
+ addr_families.append(socket.AF_UNSPEC)
self._addr_families = addr_families
@@ -219,6 +219,7 @@ class Deep(object):
maxtime = self._download_timeout
hosts = [host[0] for host in self._hosts]
num_hosts = len(hosts)
+ self.dl_failures = 0
for host in hosts:
@@ -252,7 +253,9 @@ class Deep(object):
#self.output.write('deeptest(): adding rethost %s, %s' % (key, top_hosts[key]), 2)
rethosts.append(top_hosts[key])
- self.output.write('deeptest(): final rethost %s' % (rethosts), 2)
+ self.output.write('deeptest(): final rethost %s\n' % (rethosts), 2)
+ self.output.write('deeptest(): final md5 failures %s of %s\n'
+ % (self.dl_failures, num_hosts), 2)
self.urls = rethosts
@@ -280,40 +283,38 @@ class Deep(object):
signal.signal(signal.SIGALRM, timeout_handler)
ips = []
- for family in self._addr_families:
- ipv6 = family == socket.AF_INET6
+ for addr_family in self._addr_families:
try:
- try:
- signal.alarm(self._dns_timeout)
- for family, socktype, proto, canonname, sockaddr in \
- socket.getaddrinfo(url_parts.hostname, None,
- family, socket.SOCK_STREAM):
- ip = sockaddr[0]
- if ipv6:
- ip = "[%s]" % ip
- ips.append(ip)
- finally:
- signal.alarm(0)
+ signal.alarm(self._dns_timeout)
+ for result in socket.getaddrinfo(url_parts.hostname, None,
+ addr_family, socket.SOCK_STREAM, 0, socket.AI_ADDRCONFIG):
+ family, _, __, ___, sockaddr = result
+ ip = sockaddr[0]
+ if family == socket.AF_INET6:
+ ip = "[%s]" % ip
+ ips.append(ip)
except socket.error as e:
- self.output.write('deeptime(): dns error for host %s: %s\n' % \
- (url_parts.hostname, e), 2)
+ self.output.write('deeptime(): dns error for host %s: %s\n' % (url_parts.hostname, e), 2)
except TimeoutException:
- self.output.write('deeptime(): dns timeout for host %s\n' % \
- (url_parts.hostname,), 2)
+ self.output.write('deeptime(): dns timeout for host %s\n' % url_parts.hostname, 2)
+ finally:
+ signal.alarm(0)
if not ips:
- self.output.write('deeptime(): unable to resolve ip for host %s\n' % \
- (url_parts.hostname,), 2)
+ self.output.write('deeptime(): unable to resolve ip for host %s\n' % url_parts.hostname, 2)
return (None, True)
delta = 0
f = None
for ip in ips:
+ test_parts = url_parts._replace(netloc=ip)
+ test_url = url_unparse(test_parts)
+ self.output.write('deeptime(): testing url: %s\n' % test_url, 2)
try:
try:
signal.alarm(self._connect_timeout)
- f = url_open(url)
+ f = url_open(test_url)
break
finally:
signal.alarm(0)
@@ -324,7 +325,7 @@ class Deep(object):
except TimeoutException:
self.output.write(('deeptime(): connection to host %s ' + \
'timed out for ip %s\n') % \
- (url_parts.hostname, ip), 2)
+ (test_parts.hostname, ip), 2)
if f is None:
self.output.write('deeptime(): unable to ' + \
@@ -355,13 +356,21 @@ class Deep(object):
try:
signal.alarm(int(math.ceil(maxtime)))
stime = time.time()
- f = url_open(url)
+ f = url_open(test_url)
- if hashlib.md5(f.read()).hexdigest() != "bdf077b2e683c506bf9e8f2494eeb044":
- return (None, True)
+ md5 = hashlib.md5(f.read()).hexdigest()
delta = time.time() - stime
f.close()
+ if md5 != self.test_md5:
+ self.output.write(
+ "deeptime(): md5sum error for file: %s\n" % self.test_file +
+ " expected: %s\n" % self.test_md5 +
+ " got.....: %s\n" % md5 +
+ " host....: %s, %s\n" % (url_parts.hostname, ip))
+ self.dl_failures += 1
+ return (None, True)
+
finally:
signal.alarm(0)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-19 9:06 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-19 9:06 UTC (permalink / raw
To: gentoo-commits
commit: ea6eac96004ede150a3a1acdb7a83d67b34c8390
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 19 08:59:34 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Oct 19 09:04:49 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=ea6eac96
Improve debug print statements.
---
| 2 +-
mirrorselect/main.py | 6 +++---
mirrorselect/selectors.py | 6 ++++--
3 files changed, 8 insertions(+), 6 deletions(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 2b7d049..3a113fb 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -68,7 +68,7 @@ class Extractor(object):
self.hosts = self.filter_hosts(filters, self.unfiltered_hosts)
- self.output.write('Extractor(): fetched mirrors.xml,'
+ self.output.write('Extractor(): fetched mirrors,'
' %s hosts after filtering\n' % len(self.hosts), 2)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 5844a65..0e41f30 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -292,10 +292,10 @@ class MirrorSelect(object):
@rtype: list
'''
if options.rsync:
- self.output.write("using url: %s" % MIRRORS_RSYNC_DATA, 2)
+ self.output.write("using url: %s\n" % MIRRORS_RSYNC_DATA, 2)
hosts = Extractor(MIRRORS_RSYNC_DATA, options, self.output).hosts
else:
- self.output.write("using url: %s" % MIRRORS_3_XML, 2)
+ self.output.write("using url: %s\n" % MIRRORS_3_XML, 2)
hosts = Extractor(MIRRORS_3_XML, options, self.output).hosts
return hosts
@@ -353,7 +353,7 @@ class MirrorSelect(object):
config_path = self.get_conf_path(options.rsync)
self.output.write("main(); reset config_path = %s\n" % config_path, 2)
else:
- self.output.write("main(); rsync = %s" % str(options.rsync),2)
+ self.output.write("main(); rsync = %s\n" % str(options.rsync),2)
fsmirrors = get_filesystem_mirrors(self.output,
config_path, options.rsync)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 581cbd5..a3d0baf 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -306,6 +306,7 @@ class Deep(object):
self.output.write('deeptime(): unable to resolve ip for host %s\n' % url_parts.hostname, 2)
return (None, True)
+ self.output.write("deeptime(): ip's for host %s: %s\n" % (url_parts.hostname, str(ips)), 2)
delta = 0
f = None
@@ -343,14 +344,15 @@ class Deep(object):
finally:
signal.alarm(0)
except EnvironmentError as e:
- self.output.write(('deeptime(): close connection to host %s ' + \
+ self.output.write(('deeptime(): closing connection to host %s ' + \
'failed for ip %s: %s\n') % \
(url_parts.hostname, ip, e), 2)
except TimeoutException:
- self.output.write(('deeptime(): close connection to host %s ' + \
+ self.output.write(('deeptime(): closing connection to host %s ' + \
'timed out for ip %s\n') % \
(url_parts.hostname, ip), 2)
+ self.output.write('deeptime(): timing url: %s\n' % test_url, 2)
try:
# The first connection serves to "wake up" the route between
# the local and remote machines. A second connection is used
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-19 9:06 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-19 9:06 UTC (permalink / raw
To: gentoo-commits
commit: 3cbd4574c9d47f5d75bcc69667ce59a9de93cda1
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 19 08:58:23 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Oct 19 08:58:23 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=3cbd4574
Move the TimeoutException and handler out of the Deep class's deeptime().
---
mirrorselect/selectors.py | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 4da6156..581cbd5 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -182,6 +182,14 @@ class Shallow(object):
return host_array
+class TimeoutException(Exception):
+ pass
+
+
+def timeout_handler(signum, frame):
+ raise TimeoutException()
+
+
class Deep(object):
"""handles deep mode mirror selection."""
@@ -274,12 +282,6 @@ class Deep(object):
url_parts = url_parse(url)
- class TimeoutException(Exception):
- pass
-
- def timeout_handler(signum, frame):
- raise TimeoutException()
-
signal.signal(signal.SIGALRM, timeout_handler)
ips = []
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2013-10-19 9:06 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2013-10-19 9:06 UTC (permalink / raw
To: gentoo-commits
commit: 6a2e7f6c9d72eb4b8f7fafe5239c8c8f18387043
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 19 09:04:33 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Oct 19 09:04:58 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=6a2e7f6c
fix url testing for vhosts that do not handle ip requests.
Migrate to urllib2
Add HTTPError exception handling
Move the code to it's own function in order to handle recursion to test the connection using the hostname url since the ip url failed.
---
mirrorselect/selectors.py | 60 +++++++++++++++++++++++++++++++++--------------
1 file changed, 43 insertions(+), 17 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index a3d0baf..d59d601 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -41,12 +41,14 @@ if sys.version_info[0] >= 3:
url_parse = urllib.parse.urlparse
url_unparse = urllib.parse.urlunparse
url_open = urllib.request.urlopen
+ HTTPError = urllib.HTTPError
else:
- import urllib
+ import urllib2
import urlparse
url_parse = urlparse.urlparse
url_unparse = urlparse.urlunparse
- url_open = urllib.urlopen
+ url_open = urllib2.urlopen
+ HTTPError = urllib2.HTTPError
from mirrorselect.output import encoder, get_encoding, decode_selection
@@ -314,21 +316,11 @@ class Deep(object):
test_parts = url_parts._replace(netloc=ip)
test_url = url_unparse(test_parts)
self.output.write('deeptime(): testing url: %s\n' % test_url, 2)
- try:
- try:
- signal.alarm(self._connect_timeout)
- f = url_open(test_url)
- break
- finally:
- signal.alarm(0)
- except EnvironmentError as e:
- self.output.write(('deeptime(): connection to host %s ' + \
- 'failed for ip %s: %s\n') % \
- (url_parts.hostname, ip, e), 2)
- except TimeoutException:
- self.output.write(('deeptime(): connection to host %s ' + \
- 'timed out for ip %s\n') % \
- (test_parts.hostname, ip), 2)
+
+ f, test_url, early_out = self._test_connection(test_url, url_parts,
+ ip, ips[ips.index(ip):])
+ if early_out:
+ break
if f is None:
self.output.write('deeptime(): unable to ' + \
@@ -395,6 +387,40 @@ class Deep(object):
self.output.write('deeptime(): %s seconds for host %s\n' % (delta, url), 2)
return (delta, False)
+
+ def _test_connection(self, test_url, url_parts, ip, ips):
+ """Tests the url for a connection, will recurse using
+ the original url instead of the ip if an HTTPError occurs
+ Returns f, test_url, early_out
+ """
+ early_out = False
+ f = None
+ try:
+ try:
+ signal.alarm(self._connect_timeout)
+ f = url_open(test_url)
+ early_out = True
+ finally:
+ signal.alarm(0)
+ except HTTPError, e:
+ self.output.write(('deeptime(): connection to host %s\n' + \
+ ' returned HTTPError: %s for ip %s\n' \
+ ' Switching back to original url\n') % \
+ (url_parts.hostname, e, ip), 2)
+ if len(ips) == 1:
+ test_url = url_unparse(url_parts)
+ return self._test_connection(test_url, url_parts, ip, [])
+ except EnvironmentError as e:
+ self.output.write(('deeptime(): connection to host %s ' + \
+ 'failed for ip %s:\n %s\n') % \
+ (url_parts.hostname, ip, e), 2)
+ except TimeoutException:
+ self.output.write(('deeptime(): connection to host %s ' + \
+ 'timed out for ip %s\n') % \
+ (url_parts.hostname, ip), 2)
+ return f, test_url, early_out
+
+
def _list_add(self, time_host, maxtime, host_dict, maxlen):
"""
Takes argumets ((time, host), maxtime, host_dict, maxlen)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02 7:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: 1f4d0ba04d038cf1374975af217db646bfbc8f31
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 05:15:41 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Oct 20 05:15:41 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=1f4d0ba0
fix HTTPError import and exception
---
mirrorselect/selectors.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index d59d601..1544937 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -41,7 +41,7 @@ if sys.version_info[0] >= 3:
url_parse = urllib.parse.urlparse
url_unparse = urllib.parse.urlunparse
url_open = urllib.request.urlopen
- HTTPError = urllib.HTTPError
+ HTTPError = urllib.error.HTTPError
else:
import urllib2
import urlparse
@@ -402,7 +402,7 @@ class Deep(object):
early_out = True
finally:
signal.alarm(0)
- except HTTPError, e:
+ except HTTPError as e:
self.output.write(('deeptime(): connection to host %s\n' + \
' returned HTTPError: %s for ip %s\n' \
' Switching back to original url\n') % \
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
2013-10-20 18:19 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
@ 2014-03-02 7:44 ` Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: eabccfc8eeef5d0c76a444545fa390f400a9d4ed
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 05:20:39 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Oct 20 18:14:21 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=eabccfc8
Work in progress for adding ssl support for downloading the mirrors lists.
add the request code in it's own file and class.
---
mirrorselect/connections.py | 182 ++++++++++++++++++++++++++++++++++++++++++++
| 36 ++++-----
2 files changed, 200 insertions(+), 18 deletions(-)
diff --git a/mirrorselect/connections.py b/mirrorselect/connections.py
new file mode 100644
index 0000000..ca4aa88
--- /dev/null
+++ b/mirrorselect/connections.py
@@ -0,0 +1,182 @@
+#-*- coding:utf-8 -*-
+
+"""Mirrorselect 2.x
+ Tool for selecting Gentoo source and rsync mirrors.
+
+Copyright 2005-2012 Gentoo Foundation
+
+ Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
+ Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
+ Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
+ Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
+ Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org>
+
+Distributed under the terms of the GNU General Public License v2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+"""
+
+import sys
+import os
+
+VERIFY_SSL = False
+VERIFY_MSGS = []
+
+import requests
+from requests.exceptions import SSLError
+
+# py3.2
+if sys.hexversion >= 0x30200f0:
+ VERIFY_SSL = True
+else:
+ try: # import and enable SNI support for py2
+ from requests.packages.urllib3.contrib import pyopenssl
+ pyopenssl.inject_into_urllib3()
+ VERIFY_SSL = True
+ VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
+ except ImportError as e:
+ VERIFY_MSGS = [
+ "Failed to import and inject pyopenssl/SNI support into urllib3",
+ "Disabling certificate verification",
+ "Error was:" + e
+ ]
+ VERIFY_SSL = False
+
+
+from mirrorselect.version import version
+
+
+class Connector(object):
+ """Primary connection interface using the dev-python/requests package
+ """
+
+ def __init__(self, output, proxies):
+ self.output = output
+ self.proxies = proxies
+ self.headers = {'Accept-Charset': 'utf-8',
+ 'User-Agent': 'Mirrorselect-' + version}
+
+ if VERIFY_MSGS:
+ for msg in VERIFY_MSGS:
+ self.output.write(msg + '\n', 2)
+
+
+ def add_timestamp(self, headers, tpath=None, timestamp=None):
+ """for possilble future caching of the list"""
+ if tpath and os.path.exists(tpath):
+ # fileopen is a layman comaptibility function not yet implemented here
+ with fileopen(tpath,'r') as previous:
+ timestamp = previous.read()
+ if timestamp:
+ headers['If-Modified-Since'] = timestamp
+ self.output.write('Current-modified: %s\n' % timestamp, 2)
+ return headers
+
+
+ def fetch_url(self, url, headers=None, timestamp=None):
+ """Fetches the url
+
+ @param url: string
+ @param headers: dictionary, optional headers to use
+ @param tpath: string, optional filepath to a timestamp file
+ to use in the headers
+ @param timestamp: string, optional timestamp to use in the headers
+
+ """
+
+ if not headers:
+ headers = self.headers
+
+ if timestamp:
+ self.add_timestamp(headers, timestamp=timestamp)
+
+ verify = 'https' in url and VERIFY_SSL
+ self.output.write("Enabled ssl certificate verification: %s, for: %s\n"
+ %(str(verify), url), 3)
+
+ self.output.write('Connector.fetch_url(); headers = %s\n' %str(headers), 4)
+ self.output.write('Connector.fetch_url(); connecting to opener\n', 2)
+
+ try:
+ connection = requests.get(
+ url,
+ headers=headers,
+ verify=verify,
+ proxies=self.proxies,
+ )
+ except SSLError as error:
+ self.output.print_err('Connector.fetch_url(); Failed to update the '
+ 'mirror list from: %s\nSSLError was:%s\n'
+ % (url, str(error)))
+ except Exception as error:
+ self.output.print_err('Connector.fetch_url(); Failed to retrieve '
+ 'the content from: %s\nError was: %s\n'
+ % (url, str(error)))
+
+ self.output.write('Connector.fetch_url() HEADERS = %s\n' %str(connection.headers), 4)
+ self.output.write('Connector.fetch_url() Status_code = %i\n' % connection.status_code, 2)
+ return connection
+
+
+ @staticmethod
+ def normalize_headers(headers, to_lower=True):
+ """ py2, py3 compatibility function, since only py2 returns keys as lower()
+ """
+ if to_lower:
+ return dict((x.lower(), x) for x in list(headers))
+ return dict((x.upper(), x) for x in list(headers))
+
+
+ def fetch_content(self, url, tpath=None):
+ """Fetch the mirror list
+
+ @param url: string of the content to fetch
+ @param headers: dictionary, optional headers to use
+ @param tpath: string, optional filepath to a timestamp file
+ to use in the headers
+ @returns (success bool, content fetched , timestamp of fetched content,
+ content headers returned)
+ """
+
+ fheaders = self.headers
+
+ if tpath:
+ fheaders = self.add_timestamp(fheaders, tpath)
+
+ connection = self.fetch_url(url, fheaders)
+
+ headers = self.normalize_headers(connection.headers)
+
+ if 'last-modified' in headers:
+ timestamp = headers['last-modified']
+ elif 'date' in headers:
+ timestamp = headers['date']
+ else:
+ timestamp = None
+
+ if connection.status_code in [304]:
+ self.output.write('Content already up to date: %s\n'
+ % url, 4)
+ self.output.write('Last-modified: %s\n' % timestamp, 4)
+ elif connection.status_code not in [200]:
+ self.output.print_err('Connector.fetch_content(); HTTP Status-Code was:\n'
+ 'url: %s\n%s'
+ % (url, str(connection.status_code)))
+
+ if connection.status_code in [200]:
+ self.output.write('New content downloaded for: %s\n'
+ % url, 4)
+ return (True, connection.content, timestamp)
+ return (False, '', '')
+
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 3a113fb..c8d5bd5 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -27,20 +27,10 @@ Distributed under the terms of the GNU General Public License v2
"""
-import sys
-
-if sys.version_info[0] >= 3:
- import urllib.request, urllib.parse, urllib.error
- url_parse = urllib.parse
- url_open = urllib.request.urlopen
-else:
- import urllib
- import urlparse
- url_parse = urlparse.urlparse
- url_open = urllib.urlopen
-
+import os
from mirrorselect.mirrorparser3 import MirrorParser3
+from mirrorselect.connections import Connector
class Extractor(object):
@@ -50,6 +40,7 @@ class Extractor(object):
def __init__(self, list_url, options, output):
self.output = output
+ self.output.print_info('Using url: %s\n' % list_url)
filters = {}
for opt in ["country", "region"]:
value = getattr(options, opt)
@@ -61,6 +52,15 @@ class Extractor(object):
if getattr(options, opt):
filters["proto"] = opt
self.output.print_info('Limiting test to %s hosts. \n' % opt )
+
+ self.proxies = {}
+
+ for proxy in ['http_proxy', 'https_proxy']:
+ if options.proxy:
+ self.proxies[proxy.split('_')[0]] = options.proxy
+ elif os.getenv(proxy):
+ self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
+
parser = MirrorParser3()
self.hosts = []
@@ -99,14 +99,13 @@ class Extractor(object):
self.output.write('getlist(): fetching ' + url + '\n', 2)
- self.output.print_info('Downloading a list of mirrors...')
+ self.output.print_info('Downloading a list of mirrors...\n')
- try:
- parser.parse(url_open(url).read())
- except EnvironmentError:
- pass
+ fetcher = Connector(self.output, self.proxies)
+ success, mirrorlist, timestamp = fetcher.fetch_content(url)
+ parser.parse(mirrorlist)
- if len(parser.tuples()) == 0:
+ if (not mirrorlist) or len(parser.tuples()) == 0:
self.output.print_err('Could not get mirror list. '
'Check your internet connection.')
@@ -114,3 +113,4 @@ class Extractor(object):
return parser.tuples()
+
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02 7:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: ecf8787be751a013cc15ab1b77d58ed7655c1f02
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 05:19:47 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Oct 20 05:19:47 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=ecf8787b
Add a proxy option to the cli. Preliminary only, not tested.
---
mirrorselect/main.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 5838b79..4c26c02 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -240,6 +240,12 @@ class MirrorSelect(object):
default='bdf077b2e683c506bf9e8f2494eeb044',
help="An alternate file md5sum value used to compare the downloaded "
"file against for deep testing.")
+ group.add_option(
+ "-P", "--proxy", action="store",
+ default=None,
+ help="Proxy server to use if not the default proxy "
+ "in the environment")
+
if len(argv) == 1:
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02 7:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: 73a02c287f30f7378bd59c15bea802c52cd8b0fe
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 05:18:57 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Oct 20 05:18:57 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=73a02c28
wrap a long line.
---
mirrorselect/main.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 85b0345..5838b79 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -236,7 +236,8 @@ class MirrorSelect(object):
"by selecting an overly large size file. You must also "
" use the -m, --md5 option.")
group.add_option(
- "-m", "--md5", action="store", default='bdf077b2e683c506bf9e8f2494eeb044',
+ "-m", "--md5", action="store",
+ default='bdf077b2e683c506bf9e8f2494eeb044',
help="An alternate file md5sum value used to compare the downloaded "
"file against for deep testing.")
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02 7:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: 9c79ab5df791ca60f42bf939102ccca3845daa5f
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 05:18:16 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Oct 20 05:18:16 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=9c79ab5d
Make --debug level settable on the cli
---
mirrorselect/main.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 0e41f30..85b0345 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -224,8 +224,8 @@ class MirrorSelect(object):
"to select. this is only valid for download mirrors. "
"If this is not specified, a default of 1 is used.")
group.add_option(
- "-d", "--debug", action="store_const", const=2, dest="verbosity",
- default=1, help="debug mode")
+ "-d", "--debug", action="store", type="int", dest="verbosity",
+ default=1, help="debug mode, pass in the debug level [1-9]")
group.add_option(
"-q", "--quiet", action="store_const", const=0, dest="verbosity",
help="Quiet mode")
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02 7:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: 9adaf7f83ff26b484b60917002dd2814e81ce7d0
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 05:16:14 2013 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Oct 20 05:16:14 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=9adaf7f8
Add new urls for api.gentoo.org
---
mirrorselect/mirrorparser3.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 6b06a9c..16cb1c6 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -31,8 +31,13 @@ from __future__ import print_function
from xml.etree import ElementTree as ET
-MIRRORS_3_XML = 'http://www.gentoo.org/main/en/mirrors3.xml'
-MIRRORS_RSYNC_DATA = 'http://www.gentoo.org/main/en/mirrors-rsync-data.xml'
+# old url's
+#MIRRORS_3_XML = 'http://www.gentoo.org/main/en/mirrors3.xml'
+#MIRRORS_RSYNC_DATA = 'http://www.gentoo.org/main/en/mirrors-rsync-data.xml'
+
+# new urls
+MIRRORS_3_XML = 'https://api.gentoo.org/mirrors/distfiles.xml'
+MIRRORS_RSYNC_DATA = 'https://api.gentoo.org/mirrors/rsync.xml'
class MirrorParser3:
def __init__(self, options=None):
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02 7:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: 42ad8fd9f720e52fa578dccde9591bac2ea97c7d
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 31 15:36:00 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Fri Jan 31 15:39:18 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=42ad8fd9
Indent adjustment
---
mirrorselect/main.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 4c26c02..15b8ead 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -140,16 +140,16 @@ class MirrorSelect(object):
need some finishing touches.
"""
desc = "\n".join((
- self.output.white("examples:"),
- "",
- self.output.white(" automatic:"),
- " # mirrorselect -s5",
- " # mirrorselect -s3 -b10 -o >> /mnt/gentoo/etc/portage/make.conf",
- " # mirrorselect -D -s4",
- "",
- self.output.white(" interactive:"),
- " # mirrorselect -i -r",
- ))
+ self.output.white("examples:"),
+ "",
+ self.output.white(" automatic:"),
+ " # mirrorselect -s5",
+ " # mirrorselect -s3 -b10 -o >> /mnt/gentoo/etc/portage/make.conf",
+ " # mirrorselect -D -s4",
+ "",
+ self.output.white(" interactive:"),
+ " # mirrorselect -i -r",
+ ))
parser = OptionParser(
formatter=ColoredFormatter(self.output), description=desc,
version='Mirrorselect version: %s' % version)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
2014-01-31 15:44 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
@ 2014-03-02 7:44 ` Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: e1d0a9d4610eaafb9297e896ba4ed264e5fe939f
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 23 22:56:44 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Jan 23 22:56:44 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=e1d0a9d4
Use the new sslfetch pkg for the Connector class.
---
| 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index c8d5bd5..a949c75 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -30,8 +30,10 @@ Distributed under the terms of the GNU General Public License v2
import os
from mirrorselect.mirrorparser3 import MirrorParser3
-from mirrorselect.connections import Connector
+from sslfetch.connections import Connector
+from mirrorselect.version import version
+USERAGENT = "Mirrorselect-" + version
class Extractor(object):
"""The Extractor employs a MirrorParser3 object to get a list of valid
@@ -101,7 +103,7 @@ class Extractor(object):
self.output.print_info('Downloading a list of mirrors...\n')
- fetcher = Connector(self.output, self.proxies)
+ fetcher = Connector(self.output, self.proxies, USERAGENT)
success, mirrorlist, timestamp = fetcher.fetch_content(url)
parser.parse(mirrorlist)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
2014-03-02 7:44 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
@ 2014-03-02 7:44 ` Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: dc98292e2aeca8ec13aceb2efce802d9375e11aa
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 1 03:49:11 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Feb 1 03:53:59 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=dc98292e
Tweak the proxy code to correctly handle the cli option.
---
| 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 217d1e1..eb26faf 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -58,10 +58,11 @@ class Extractor(object):
self.proxies = {}
for proxy in ['http_proxy', 'https_proxy']:
- if options.proxy:
- self.proxies[proxy.split('_')[0]] = options.proxy
+ prox = proxy.split('_')[0]
+ if options.proxy and prox + ":" in options.proxy:
+ self.proxies[prox] = options.proxy
elif os.getenv(proxy):
- self.proxies[proxy.split('_')[0]] = os.getenv(proxy)
+ self.proxies[prox] = os.getenv(proxy)
parser = MirrorParser3()
self.hosts = []
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-03-02 7:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-03-02 7:44 UTC (permalink / raw
To: gentoo-commits
commit: a3b025b7f3d5e67b1735bb773b72b5766537225f
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 23 22:53:08 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Jan 23 22:53:08 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=a3b025b7
Remove connections.py. Instead moved it to it's own ssl-fetch pkg.
---
mirrorselect/connections.py | 182 --------------------------------------------
1 file changed, 182 deletions(-)
diff --git a/mirrorselect/connections.py b/mirrorselect/connections.py
deleted file mode 100644
index ca4aa88..0000000
--- a/mirrorselect/connections.py
+++ /dev/null
@@ -1,182 +0,0 @@
-#-*- coding:utf-8 -*-
-
-"""Mirrorselect 2.x
- Tool for selecting Gentoo source and rsync mirrors.
-
-Copyright 2005-2012 Gentoo Foundation
-
- Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org>
- Copyright (C) 2008 Zac Medico <zmedico@gentoo.org>
- Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
- Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
- Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org>
-
-Distributed under the terms of the GNU General Public License v2
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 2 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
-
-"""
-
-import sys
-import os
-
-VERIFY_SSL = False
-VERIFY_MSGS = []
-
-import requests
-from requests.exceptions import SSLError
-
-# py3.2
-if sys.hexversion >= 0x30200f0:
- VERIFY_SSL = True
-else:
- try: # import and enable SNI support for py2
- from requests.packages.urllib3.contrib import pyopenssl
- pyopenssl.inject_into_urllib3()
- VERIFY_SSL = True
- VERIFY_MSGS = ["Successfully enabled ssl certificate verification."]
- except ImportError as e:
- VERIFY_MSGS = [
- "Failed to import and inject pyopenssl/SNI support into urllib3",
- "Disabling certificate verification",
- "Error was:" + e
- ]
- VERIFY_SSL = False
-
-
-from mirrorselect.version import version
-
-
-class Connector(object):
- """Primary connection interface using the dev-python/requests package
- """
-
- def __init__(self, output, proxies):
- self.output = output
- self.proxies = proxies
- self.headers = {'Accept-Charset': 'utf-8',
- 'User-Agent': 'Mirrorselect-' + version}
-
- if VERIFY_MSGS:
- for msg in VERIFY_MSGS:
- self.output.write(msg + '\n', 2)
-
-
- def add_timestamp(self, headers, tpath=None, timestamp=None):
- """for possilble future caching of the list"""
- if tpath and os.path.exists(tpath):
- # fileopen is a layman comaptibility function not yet implemented here
- with fileopen(tpath,'r') as previous:
- timestamp = previous.read()
- if timestamp:
- headers['If-Modified-Since'] = timestamp
- self.output.write('Current-modified: %s\n' % timestamp, 2)
- return headers
-
-
- def fetch_url(self, url, headers=None, timestamp=None):
- """Fetches the url
-
- @param url: string
- @param headers: dictionary, optional headers to use
- @param tpath: string, optional filepath to a timestamp file
- to use in the headers
- @param timestamp: string, optional timestamp to use in the headers
-
- """
-
- if not headers:
- headers = self.headers
-
- if timestamp:
- self.add_timestamp(headers, timestamp=timestamp)
-
- verify = 'https' in url and VERIFY_SSL
- self.output.write("Enabled ssl certificate verification: %s, for: %s\n"
- %(str(verify), url), 3)
-
- self.output.write('Connector.fetch_url(); headers = %s\n' %str(headers), 4)
- self.output.write('Connector.fetch_url(); connecting to opener\n', 2)
-
- try:
- connection = requests.get(
- url,
- headers=headers,
- verify=verify,
- proxies=self.proxies,
- )
- except SSLError as error:
- self.output.print_err('Connector.fetch_url(); Failed to update the '
- 'mirror list from: %s\nSSLError was:%s\n'
- % (url, str(error)))
- except Exception as error:
- self.output.print_err('Connector.fetch_url(); Failed to retrieve '
- 'the content from: %s\nError was: %s\n'
- % (url, str(error)))
-
- self.output.write('Connector.fetch_url() HEADERS = %s\n' %str(connection.headers), 4)
- self.output.write('Connector.fetch_url() Status_code = %i\n' % connection.status_code, 2)
- return connection
-
-
- @staticmethod
- def normalize_headers(headers, to_lower=True):
- """ py2, py3 compatibility function, since only py2 returns keys as lower()
- """
- if to_lower:
- return dict((x.lower(), x) for x in list(headers))
- return dict((x.upper(), x) for x in list(headers))
-
-
- def fetch_content(self, url, tpath=None):
- """Fetch the mirror list
-
- @param url: string of the content to fetch
- @param headers: dictionary, optional headers to use
- @param tpath: string, optional filepath to a timestamp file
- to use in the headers
- @returns (success bool, content fetched , timestamp of fetched content,
- content headers returned)
- """
-
- fheaders = self.headers
-
- if tpath:
- fheaders = self.add_timestamp(fheaders, tpath)
-
- connection = self.fetch_url(url, fheaders)
-
- headers = self.normalize_headers(connection.headers)
-
- if 'last-modified' in headers:
- timestamp = headers['last-modified']
- elif 'date' in headers:
- timestamp = headers['date']
- else:
- timestamp = None
-
- if connection.status_code in [304]:
- self.output.write('Content already up to date: %s\n'
- % url, 4)
- self.output.write('Last-modified: %s\n' % timestamp, 4)
- elif connection.status_code not in [200]:
- self.output.print_err('Connector.fetch_content(); HTTP Status-Code was:\n'
- 'url: %s\n%s'
- % (url, str(connection.status_code)))
-
- if connection.status_code in [200]:
- self.output.write('New content downloaded for: %s\n'
- % url, 4)
- return (True, connection.content, timestamp)
- return (False, '', '')
-
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-05-05 2:04 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-05-05 2:04 UTC (permalink / raw
To: gentoo-commits
commit: a54f7eab35470bf34a3955cda94d372af3cd030d
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 6 16:35:22 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Mar 6 16:35:22 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=a54f7eab
Fix typo in help message.
---
mirrorselect/main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 15b8ead..3014c3f 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -200,7 +200,7 @@ class MirrorSelect(object):
"-R", "--region", action="store", default=None,
help="only use mirrors from the specified region "
"NOTE: Names with a space must be quoted "
- "eg.: -r 'North America'")
+ "eg.: -R 'North America'")
group = parser.add_option_group("Other options")
group.add_option(
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-05-05 2:04 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-05-05 2:04 UTC (permalink / raw
To: gentoo-commits
commit: 3445069b3da6a33356f2d16a9ff27a722873512d
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon May 5 02:03:01 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon May 5 02:03:01 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=3445069b
selectors.py: Add IncompleteRead exception
Improve output formatting.
---
mirrorselect/selectors.py | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 5d30695..1aa5f11 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -42,6 +42,8 @@ if sys.version_info[0] >= 3:
url_unparse = urllib.parse.urlunparse
url_open = urllib.request.urlopen
HTTPError = urllib.error.HTTPError
+ import http.client
+ IncompleteRead = http.client.IncompleteRead
else:
import urllib2
import urlparse
@@ -49,6 +51,8 @@ else:
url_unparse = urlparse.urlunparse
url_open = urllib2.urlopen
HTTPError = urllib2.HTTPError
+ import httplib
+ IncompleteRead = httplib.IncompleteRead
from mirrorselect.output import encoder, get_encoding, decode_selection
@@ -369,7 +373,7 @@ class Deep(object):
f.close()
if md5 != self.test_md5:
self.output.write(
- "deeptime(): md5sum error for file: %s\n"
+ "\ndeeptime(): md5sum error for file: %s\n"
% self.test_file +
" expected: %s\n" % self.test_md5 +
" got.....: %s\n" % md5 +
@@ -382,13 +386,17 @@ class Deep(object):
signal.alarm(0)
except EnvironmentError as e:
- self.output.write(('deeptime(): download from host %s '
+ self.output.write(('\ndeeptime(): download from host %s '
'failed for ip %s: %s\n') % (url_parts.hostname, ip, e), 2)
return (None, True)
except TimeoutException:
- self.output.write(('deeptime(): download from host %s '
+ self.output.write(('\ndeeptime(): download from host %s '
'timed out for ip %s\n') % (url_parts.hostname, ip), 2)
return (None, True)
+ except IncompleteRead as e:
+ self.output.write(('\ndeeptime(): download from host %s '
+ 'failed for ip %s: %s\n') % (url_parts.hostname, ip, e), 2)
+ return (None, True)
signal.signal(signal.SIGALRM, signal.SIG_DFL)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-05-05 2:04 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-05-05 2:04 UTC (permalink / raw
To: gentoo-commits
commit: 9f6c6d2fd749df7528fcb2b7a85790362fb4d232
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon May 5 02:00:35 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Mon May 5 02:00:35 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=9f6c6d2f
main.py: Sort the cli options
---
mirrorselect/main.py | 60 ++++++++++++++++++++++++++--------------------------
1 file changed, 30 insertions(+), 30 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 3014c3f..d17a4f1 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -162,19 +162,24 @@ class MirrorSelect(object):
" For the -r, --rsync option, it will select the rotation server "
"only. As multiple rsync URL's are not supported.")
group.add_option(
- "-i", "--interactive", action="store_true", default=False,
- help="Interactive Mode, this will present a list "
- "to make it possible to select mirrors you wish to use.")
- group.add_option(
"-D", "--deep", action="store_true", default=False,
help="Deep mode. This is used to give a more accurate "
"speed test. It will download a 100k file from "
"each server. Because of this you should only use "
"this option if you have a good connection.")
+ group.add_option(
+ "-i", "--interactive", action="store_true", default=False,
+ help="Interactive Mode, this will present a list "
+ "to make it possible to select mirrors you wish to use.")
group = parser.add_option_group(
"Server type selection (choose at most one)")
group.add_option(
+ "-c", "--country", action="store", default=None,
+ help="only use mirrors from the specified country "
+ "NOTE: Names with a space must be quoted "
+ "eg.: -c 'South Korea'")
+ group.add_option(
"-F", "--ftp", action="store_true", default=False,
help="ftp only mode. Will not consider hosts of other "
"types.")
@@ -186,29 +191,19 @@ class MirrorSelect(object):
help="rsync mode. Allows you to interactively select your"
" rsync mirror. Requires -i or -a to be used.")
group.add_option(
+ "-R", "--region", action="store", default=None,
+ help="only use mirrors from the specified region "
+ "NOTE: Names with a space must be quoted "
+ "eg.: -R 'North America'")
+ group.add_option(
"-4", "--ipv4", action="store_true", default=False,
help="only use IPv4")
group.add_option(
"-6", "--ipv6", action="store_true", default=False,
help="only use IPv6")
- group.add_option(
- "-c", "--country", action="store", default=None,
- help="only use mirrors from the specified country "
- "NOTE: Names with a space must be quoted "
- "eg.: -c 'South Korea'")
- group.add_option(
- "-R", "--region", action="store", default=None,
- help="only use mirrors from the specified region "
- "NOTE: Names with a space must be quoted "
- "eg.: -R 'North America'")
group = parser.add_option_group("Other options")
group.add_option(
- "-o", "--output", action="store_true", default=False,
- help="Output Only Mode, this is especially useful "
- "when being used during installation, to redirect "
- "output to a file other than %s" % config_path)
- group.add_option(
"-b", "--blocksize", action="store", type="int",
help="This is to be used in automatic mode "
"and will split the hosts into blocks of BLOCKSIZE for "
@@ -216,20 +211,9 @@ class MirrorSelect(object):
"routers which block 40+ requests at any given time. "
"Recommended parameters to pass are: -s3 -b10")
group.add_option(
- "-t", "--timeout", action="store", type="int",
- default="10", help="Timeout for deep mode. Defaults to 10 seconds.")
- group.add_option(
- "-s", "--servers", action="store", type="int", default=1,
- help="Specify Number of servers for Automatic Mode "
- "to select. this is only valid for download mirrors. "
- "If this is not specified, a default of 1 is used.")
- group.add_option(
"-d", "--debug", action="store", type="int", dest="verbosity",
default=1, help="debug mode, pass in the debug level [1-9]")
group.add_option(
- "-q", "--quiet", action="store_const", const=0, dest="verbosity",
- help="Quiet mode")
- group.add_option(
"-f", "--file", action="store", default='mirrorselect-test',
help="An alternate file to download for deep testing. "
"Please choose the file carefully as to not abuse the system "
@@ -241,10 +225,26 @@ class MirrorSelect(object):
help="An alternate file md5sum value used to compare the downloaded "
"file against for deep testing.")
group.add_option(
+ "-o", "--output", action="store_true", default=False,
+ help="Output Only Mode, this is especially useful "
+ "when being used during installation, to redirect "
+ "output to a file other than %s" % config_path)
+ group.add_option(
"-P", "--proxy", action="store",
default=None,
help="Proxy server to use if not the default proxy "
"in the environment")
+ group.add_option(
+ "-q", "--quiet", action="store_const", const=0, dest="verbosity",
+ help="Quiet mode")
+ group.add_option(
+ "-s", "--servers", action="store", type="int", default=1,
+ help="Specify Number of servers for Automatic Mode "
+ "to select. this is only valid for download mirrors. "
+ "If this is not specified, a default of 1 is used.")
+ group.add_option(
+ "-t", "--timeout", action="store", type="int",
+ default="10", help="Timeout for deep mode. Defaults to 10 seconds.")
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-05-28 19:44 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-05-28 19:44 UTC (permalink / raw
To: gentoo-commits
commit: 16c4aebc0a90b98c50f4df58abd63066196cefc7
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed May 28 19:43:46 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed May 28 19:43:46 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=16c4aebc
Update for recent ssl-fetch changes
---
| 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index eb26faf..b9ed15e 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -104,7 +104,15 @@ class Extractor(object):
self.output.print_info('Downloading a list of mirrors...\n')
- fetcher = Connector(self.output, self.proxies, USERAGENT)
+ # setup the ssl-fetch ouptut map
+ connector_output = {
+ 'info':self.output.write,
+ 'error': self.output.print_err,
+ 'args-info': {'level': 2},
+ 'args-error': {'level':0},
+ }
+
+ fetcher = Connector(connector_output, self.proxies, USERAGENT)
success, mirrorlist, timestamp = fetcher.fetch_content(url)
parser.parse(mirrorlist)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2014-05-28 21:08 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2014-05-28 21:08 UTC (permalink / raw
To: gentoo-commits
commit: 186426dd65b0b91daa9a5163900f2233ff197d61
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed May 28 20:52:38 2014 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Wed May 28 20:52:38 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=186426dd
Ssl-fetch renamed args-* to kwargs-*
---
| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index b9ed15e..ec5f299 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -108,8 +108,8 @@ class Extractor(object):
connector_output = {
'info':self.output.write,
'error': self.output.print_err,
- 'args-info': {'level': 2},
- 'args-error': {'level':0},
+ 'kwargs-info': {'level': 2},
+ 'kwargs-error': {'level':0},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2015-01-27 4:38 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2015-01-27 4:38 UTC (permalink / raw
To: gentoo-commits
commit: 9ec1e9649c980d32d23aab40eff3cac5e8f6f555
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 27 02:23:05 2015 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Jan 27 02:23:05 2015 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=9ec1e964
Update for ssl-fetch api change
---
| 1 +
1 file changed, 1 insertion(+)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index ec5f299..59efad1 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -109,6 +109,7 @@ class Extractor(object):
'info':self.output.write,
'error': self.output.print_err,
'kwargs-info': {'level': 2},
+ 'kwargs-debug': {'level':2},
'kwargs-error': {'level':0},
}
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2015-01-27 4:38 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2015-01-27 4:38 UTC (permalink / raw
To: gentoo-commits
commit: 71a32f7fea74caedb83d57408f8f2aa08e40176b
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 27 04:38:03 2015 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Jan 27 04:38:03 2015 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=71a32f7f
update version with latest release as a base
---
mirrorselect/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/version.py b/mirrorselect/version.py
index 12942ed..ce110c9 100644
--- a/mirrorselect/version.py
+++ b/mirrorselect/version.py
@@ -25,5 +25,5 @@ Distributed under the terms of the GNU General Public License v2
"""
-version = "git"
+version = "2.2.2-git"
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2015-01-27 18:20 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2015-01-27 18:20 UTC (permalink / raw
To: gentoo-commits
commit: e9dea3a98aa5031fecf1c7115de77e948e8d07ca
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 27 18:19:05 2015 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Jan 27 18:19:05 2015 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=e9dea3a9
Add missed output mode function assignment.
---
| 1 +
1 file changed, 1 insertion(+)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 59efad1..06dce6c 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -107,6 +107,7 @@ class Extractor(object):
# setup the ssl-fetch ouptut map
connector_output = {
'info':self.output.write,
+ 'debug': self.output.write,
'error': self.output.print_err,
'kwargs-info': {'level': 2},
'kwargs-debug': {'level':2},
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2016-11-15 1:16 Zac Medico
0 siblings, 0 replies; 86+ messages in thread
From: Zac Medico @ 2016-11-15 1:16 UTC (permalink / raw
To: gentoo-commits
commit: 613bb0efeebe6461b37df6baa862f4a3e67c94c7
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 14 07:16:51 2016 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Nov 14 07:23:19 2016 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=613bb0ef
deeptime: cancel alarm signal before handling socket.error (bug 523312)
In order to avoid a race condition, the alarm signal must be cancelled
before socket.error is handled.
X-Gentoo-Bug: 523312
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=523312
mirrorselect/selectors.py | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 1aa5f11..cf70b21 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -300,22 +300,24 @@ class Deep(object):
ips = []
for addr_family in self._addr_families:
try:
- signal.alarm(self._dns_timeout)
- for result in socket.getaddrinfo(url_parts.hostname, None,
- addr_family, socket.SOCK_STREAM, 0, socket.AI_ADDRCONFIG):
- family, _, __, ___, sockaddr = result
- ip = sockaddr[0]
- if family == socket.AF_INET6:
- ip = "[%s]" % ip
- ips.append(ip)
+ try:
+ signal.alarm(self._dns_timeout)
+ for result in socket.getaddrinfo(
+ url_parts.hostname, None, addr_family,
+ socket.SOCK_STREAM, 0, socket.AI_ADDRCONFIG):
+ family, _, __, ___, sockaddr = result
+ ip = sockaddr[0]
+ if family == socket.AF_INET6:
+ ip = "[%s]" % ip
+ ips.append(ip)
+ finally:
+ signal.alarm(0)
except socket.error as e:
self.output.write('deeptime(): dns error for host %s: %s\n'
% (url_parts.hostname, e), 2)
except TimeoutException:
self.output.write('deeptime(): dns timeout for host %s\n'
% url_parts.hostname, 2)
- finally:
- signal.alarm(0)
if not ips:
self.output.write('deeptime(): unable to resolve ip for host %s\n'
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2017-02-21 3:19 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2017-02-21 3:19 UTC (permalink / raw
To: gentoo-commits
commit: c91509d823c27fdb1527c61f77c959b305170915
Author: zakalwe <sc.contact <AT> gmail <DOT> com>
AuthorDate: Tue Feb 21 03:09:11 2017 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Feb 21 03:18:51 2017 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=c91509d8
main.py: Fix Bytes error in hosts list bug 610016
The code was only checking the first host which was not a bytes instance
in that case. This resulted in the remaining host entries to not be decoded.
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/main.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index d17a4f1..30345cc 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -108,8 +108,9 @@ class MirrorSelect(object):
else:
var = 'GENTOO_MIRRORS'
- if hasattr(hosts[0], 'decode'):
- hosts = [x.decode('utf-8') for x in hosts]
+ for i in range(0, len(hosts)):
+ if isinstance(hosts[i], 'bytes'):
+ hosts[i] = hosts[i].decode('utf-8')
if var == "sync-uri" and out:
mirror_string = '%s = %s' % (var, ' '.join(hosts))
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2017-02-21 3:19 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2017-02-21 3:19 UTC (permalink / raw
To: gentoo-commits
commit: 195bea73e1195b824eabdeb411713801f17d11a6
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 21 03:15:28 2017 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Feb 21 03:18:51 2017 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=195bea73
extractor.py: Add climit param for >=ssl-fetch-0.4
| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 06dce6c..dca8302 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -115,7 +115,7 @@ class Extractor(object):
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
- success, mirrorlist, timestamp = fetcher.fetch_content(url)
+ success, mirrorlist, timestamp = fetcher.fetch_content(url, climit=60)
parser.parse(mirrorlist)
if (not mirrorlist) or len(parser.tuples()) == 0:
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2017-02-21 4:44 Zac Medico
0 siblings, 0 replies; 86+ messages in thread
From: Zac Medico @ 2017-02-21 4:44 UTC (permalink / raw
To: gentoo-commits
commit: 18df6efa96f57830f26ca94c41e1f1e8cc30b3ca
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 21 04:44:03 2017 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 21 04:44:24 2017 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=18df6efa
main.py: fix isinstance(hosts[i], bytes)
Fixes: c91509d823c2 ("main.py: Fix Bytes error in hosts list bug 610016")
mirrorselect/main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 30345cc..4858875 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -109,7 +109,7 @@ class MirrorSelect(object):
var = 'GENTOO_MIRRORS'
for i in range(0, len(hosts)):
- if isinstance(hosts[i], 'bytes'):
+ if isinstance(hosts[i], bytes):
hosts[i] = hosts[i].decode('utf-8')
if var == "sync-uri" and out:
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2018-05-26 15:43 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2018-05-26 15:43 UTC (permalink / raw
To: gentoo-commits
commit: cf6769792d8334e6302a082350d5cf411ff4f3f7
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 18 20:14:25 2017 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Jul 18 20:15:02 2017 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=cf676979
main.py: Remove obsolete 'SYNC' option
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/main.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 4858875..04698f3 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -101,10 +101,7 @@ class MirrorSelect(object):
SYNC and GENTOO_MIRRORS make.conf variable target
"""
if sync:
- if 'repos.conf' in config_path:
- var = "sync-uri"
- else:
- var = 'SYNC'
+ var = "sync-uri"
else:
var = 'GENTOO_MIRRORS'
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2019-02-13 5:51 Zac Medico
0 siblings, 0 replies; 86+ messages in thread
From: Zac Medico @ 2019-02-13 5:51 UTC (permalink / raw
To: gentoo-commits
commit: 92d682ca0f8d173149df8511a06b0457ffbffa8d
Author: Daniel Harding <dharding <AT> living180 <DOT> net>
AuthorDate: Sat Sep 2 11:13:00 2017 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 13 05:50:12 2019 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=92d682ca
selectors.py: handle ssl.CertificateError (bug 604968)
Bug: https://bugs.gentoo.org/604968
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
mirrorselect/selectors.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index cf70b21..58a44a1 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -31,6 +31,7 @@ Distributed under the terms of the GNU General Public License v2
import math
import signal
import socket
+import ssl
import subprocess
import sys
import time
@@ -430,7 +431,7 @@ class Deep(object):
if len(ips) == 1:
test_url = url_unparse(url_parts)
return self._test_connection(test_url, url_parts, ip, [])
- except EnvironmentError as e:
+ except (EnvironmentError, ssl.CertificateError) as e:
self.output.write('deeptime(): connection to host %s '
'failed for ip %s:\n %s\n'
% (url_parts.hostname, ip, e), 2)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2019-02-13 8:09 Zac Medico
0 siblings, 0 replies; 86+ messages in thread
From: Zac Medico @ 2019-02-13 8:09 UTC (permalink / raw
To: gentoo-commits
commit: a6532e7c6b655ebba0dce53f92d9fca180b23be6
Author: Xiami <i <AT> f2light <DOT> com>
AuthorDate: Thu Dec 14 10:30:29 2017 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 13 08:06:57 2019 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=a6532e7c
selectors.py: handle ssl.CertificateError (bug 639156)
Bug: https://bugs.gentoo.org/639156
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
mirrorselect/selectors.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 58a44a1..33f7663 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -388,7 +388,7 @@ class Deep(object):
finally:
signal.alarm(0)
- except EnvironmentError as e:
+ except (EnvironmentError, ssl.CertificateError) as e:
self.output.write(('\ndeeptime(): download from host %s '
'failed for ip %s: %s\n') % (url_parts.hostname, ip, e), 2)
return (None, True)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2019-02-13 8:22 Zac Medico
0 siblings, 0 replies; 86+ messages in thread
From: Zac Medico @ 2019-02-13 8:22 UTC (permalink / raw
To: gentoo-commits
commit: 856abee86416d4b2159f81d34cf28ef3422b92ec
Author: Michel Ganguin <ganguin <AT> romandie <DOT> com>
AuthorDate: Mon Dec 31 21:54:29 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 13 08:20:47 2019 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=856abee8
selectors.py: Give urllib hostname info (bug 604968)
Give urllib hostname info such that:
* it will not fail when using HTTPS because of hostname mismatch (CertificateError)
* it will not fail when the server is a virtualhost
* it will not fail when the server validates ssl SNI
Bug: https://bugs.gentoo.org/566778
Bug: https://bugs.gentoo.org/604968
Bug: https://bugs.gentoo.org/639156
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
mirrorselect/selectors.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 33f7663..4b7e7a2 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -42,6 +42,7 @@ if sys.version_info[0] >= 3:
url_parse = urllib.parse.urlparse
url_unparse = urllib.parse.urlunparse
url_open = urllib.request.urlopen
+ url_request = urllib.request.Request
HTTPError = urllib.error.HTTPError
import http.client
IncompleteRead = http.client.IncompleteRead
@@ -51,6 +52,7 @@ else:
url_parse = urlparse.urlparse
url_unparse = urlparse.urlunparse
url_open = urllib2.urlopen
+ url_request = urllib2.Request
HTTPError = urllib2.HTTPError
import httplib
IncompleteRead = httplib.IncompleteRead
@@ -368,7 +370,9 @@ class Deep(object):
try:
signal.alarm(int(math.ceil(maxtime)))
stime = time.time()
- f = url_open(test_url)
+ r = url_request(test_url)
+ r.host = url_parts.netloc
+ f = url_open(r)
md5 = hashlib.md5(f.read()).hexdigest()
@@ -419,7 +423,9 @@ class Deep(object):
try:
try:
signal.alarm(self._connect_timeout)
- f = url_open(test_url)
+ r = url_request(test_url)
+ r.host = url_parts.netloc
+ f = url_open(r)
early_out = True
finally:
signal.alarm(0)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2019-05-27 17:22 Zac Medico
0 siblings, 0 replies; 86+ messages in thread
From: Zac Medico @ 2019-05-27 17:22 UTC (permalink / raw
To: gentoo-commits
commit: 195781b8fbfe3ee52ceb478de5058e2745d24aa2
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 27 17:02:02 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 27 17:07:45 2019 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=195781b8
netselect: use -4/-6 options (bug 582508)
Requires >=net-analyzer/netselect-0.4[ipv6(+)].
Bug: https://bugs.gentoo.org/582508
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
mirrorselect/selectors.py | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 4b7e7a2..e3f718c 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -65,6 +65,7 @@ class Shallow(object):
"""handles rapid server selection via netselect"""
def __init__(self, hosts, options, output):
+ self._options = options
self.output = output
self.urls = []
@@ -94,10 +95,18 @@ class Shallow(object):
host_string = ' '.join(hosts)
- self.output.write('\nnetselect(): running "netselect -s%d %s"\n'
- % (int(number), host_string), 2)
+ cmd = ['netselect', '-s%d' % (number,)]
+ if self._options.ipv4:
+ cmd.append('-4')
+ elif self._options.ipv6:
+ cmd.append('-6')
- proc = subprocess.Popen( ['netselect', '-s%d' % (number,)] + hosts,
+ cmd.extend(hosts)
+
+ self.output.write('\nnetselect(): running "%s"\n'
+ % ' '.join(cmd), 2)
+
+ proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate()
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2019-05-27 17:52 Zac Medico
0 siblings, 0 replies; 86+ messages in thread
From: Zac Medico @ 2019-05-27 17:52 UTC (permalink / raw
To: gentoo-commits
commit: 2bc49521708ec4cd6941bc725f7b4ab490ff77a9
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 27 17:51:38 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 27 17:52:05 2019 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=2bc49521
get_filesystem_mirrors: fix DeprecationWarning: invalid escape sequence \
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
mirrorselect/configs.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
index 8c4b4df..df718f6 100644
--- a/mirrorselect/configs.py
+++ b/mirrorselect/configs.py
@@ -172,7 +172,7 @@ def get_filesystem_mirrors(output, config_path, sync=False):
""" Search for 'var' in make.conf and extract value """
lex = shlex.shlex(f, posix=True)
- lex.wordchars = string.digits+letters+"~!@#$%*_\:;?,./-+{}"
+ lex.wordchars = string.digits + letters + r"~!@#$%*_\:;?,./-+{}"
lex.quotes = "\"'"
p = re.compile('rsync://|http://|ftp://', re.IGNORECASE)
while 1:
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2019-07-17 5:06 Zac Medico
0 siblings, 0 replies; 86+ messages in thread
From: Zac Medico @ 2019-07-17 5:06 UTC (permalink / raw
To: gentoo-commits
commit: 95ba3ca5e8db2a6b360e4abdcee026076652551d
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 13 05:00:33 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jul 13 05:09:20 2019 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=95ba3ca5
netselect: make netselect ipv6 support optional
Since netselect with ipv6 support does not work on a system where
ipv6 is disabled, use a NETSELECT_SUPPORTS_IPV4_IPV6 variable to
optionally enable the use of netselect -4/-6 options. The ebuild
will use sed to set NETSELECT_SUPPORTS_IPV4_IPV6 = True when
USE=ipv6 is enabled, and will have a dependency like
RDEPEND=">=net-analyzer/netselect-0.4[ipv6(+)?]".
Bug: https://bugs.gentoo.org/688214
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
mirrorselect/selectors.py | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index e3f718c..4564f11 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -61,6 +61,11 @@ else:
from mirrorselect.output import encoder, get_encoding, decode_selection
+# The netselect --ipv4 and --ipv6 options are supported only
+# with >=net-analyzer/netselect-0.4[ipv6(+)].
+NETSELECT_SUPPORTS_IPV4_IPV6 = False
+
+
class Shallow(object):
"""handles rapid server selection via netselect"""
@@ -96,10 +101,12 @@ class Shallow(object):
host_string = ' '.join(hosts)
cmd = ['netselect', '-s%d' % (number,)]
- if self._options.ipv4:
- cmd.append('-4')
- elif self._options.ipv6:
- cmd.append('-6')
+
+ if NETSELECT_SUPPORTS_IPV4_IPV6:
+ if self._options.ipv4:
+ cmd.append('-4')
+ elif self._options.ipv6:
+ cmd.append('-6')
cmd.extend(hosts)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2020-06-03 19:05 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2020-06-03 19:05 UTC (permalink / raw
To: gentoo-commits
commit: 7caac017833b01e13028658effc502430c56d770
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Jun 3 18:53:47 2020 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Jun 3 18:58:04 2020 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=7caac017
main.py: Add a -a, -s combo sanity check
This is to prevent all mirrors from being added to make.conf when the
-s option is also enabled. The -a option takes priority in the code, so
there is never any selection done. This options check will error out when
both options are enabled.
Reported by: toralf on #gentoo-dev
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/main.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index b49461b..c4f649f 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -271,6 +271,9 @@ class MirrorSelect(object):
if options.rsync and not (options.interactive or options.all_mirrors):
self.output.print_err('rsync servers can only be selected with -i or -a')
+ if options.servers and options.all_mirrors:
+ self.output.print_err('Choose at most one of -s or -a')
+
if options.interactive and (
options.deep or
options.blocksize or
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-30 23:12 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-30 23:12 UTC (permalink / raw
To: gentoo-commits
commit: a61f2d061134d1a61254da4cd8c30cc9b4749cdb
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon May 30 22:19:30 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon May 30 23:11:03 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=a61f2d06
selectors.py: Fix bug 771963 incorrect comparison
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/selectors.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 4564f11..74405a7 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -63,7 +63,7 @@ from mirrorselect.output import encoder, get_encoding, decode_selection
# The netselect --ipv4 and --ipv6 options are supported only
# with >=net-analyzer/netselect-0.4[ipv6(+)].
-NETSELECT_SUPPORTS_IPV4_IPV6 = False
+NETSELECT_SUPPORTS_IPV4_IPV6 = True
class Shallow(object):
@@ -260,7 +260,7 @@ class Deep(object):
for host in hosts:
prog += 1
- if self.test_file is not 'mirrorselect-test':
+ if self.test_file != 'mirrorselect-test':
self.output.print_info(
'Downloading %s files from each mirror... [%s of %s]'
% (self.test_file, prog, num_hosts) )
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-30 23:12 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-30 23:12 UTC (permalink / raw
To: gentoo-commits
commit: b7defa1505c2e1207dc514523c21995f707943f4
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon May 30 22:22:29 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon May 30 23:11:22 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=b7defa15
Remove obsolete __future__ import of print_function
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/configs.py | 3 ---
mirrorselect/main.py | 3 ---
mirrorselect/mirrorparser3.py | 2 --
3 files changed, 8 deletions(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
index 4cba14f..2dd6cba 100644
--- a/mirrorselect/configs.py
+++ b/mirrorselect/configs.py
@@ -29,9 +29,6 @@ Distributed under the terms of the GNU General Public License v2
"""
-from __future__ import print_function
-
-
import os
import re
import shlex
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index c4f649f..c3b5633 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -30,9 +30,6 @@ Distributed under the terms of the GNU General Public License v2
"""
-from __future__ import print_function
-
-
import os
import socket
import sys
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 16cb1c6..c9349cb 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -27,8 +27,6 @@ Distributed under the terms of the GNU General Public License v2
"""
-from __future__ import print_function
-
from xml.etree import ElementTree as ET
# old url's
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-31 2:22 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-31 2:22 UTC (permalink / raw
To: gentoo-commits
commit: 738c6f83b48c9f33ae17551b725756a1e94437fb
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May 31 02:06:54 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May 31 02:22:29 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=738c6f83
main.py: Fix bug 600572 Remove SYNC variable usage
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/main.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index b0a68cc..9cad25b 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -343,12 +343,12 @@ class MirrorSelect(object):
@rtype: string
'''
if rsync:
- # startwith repos.conf
+ # repos.conf
config_path = EPREFIX + '/etc/portage/repos.conf/gentoo.conf'
if not os.access(config_path, os.F_OK):
self.output.write("Failed access to gentoo.conf: "
"%s\n" % os.access(config_path, os.F_OK), 2)
- return get_make_conf_path(EPREFIX)
+ config_path = None
return config_path
return get_make_conf_path(EPREFIX)
@@ -363,12 +363,12 @@ class MirrorSelect(object):
self.output.verbosity = options.verbosity
self.output.write("main(); config_path = %s\n" % config_path, 2)
- # reset config_path to find repos.conf/gentoo.conf if it exists
+ # reset config_path to find repos.conf/gentoo.conf
if options.rsync:
config_path = self.get_conf_path(options.rsync)
self.output.write("main(); reset config_path = %s\n" % config_path, 2)
- else:
- self.output.write("main(); rsync = %s\n" % str(options.rsync),2)
+ if not config_path:
+ self.output.print_err("main(); Exiting due to missing repos.conf/gentoo.conf file\n")
fsmirrors = get_filesystem_mirrors(self.output,
config_path, options.rsync)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-31 2:22 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-31 2:22 UTC (permalink / raw
To: gentoo-commits
commit: 25535069d5943b6e7bbbe4607e40685a7ed36ea4
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May 31 01:32:43 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May 31 02:21:52 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=25535069
selectors.py: Bug 800149. Add general exception for any other server error
Catch any other server errors so that mirrorselect will ignore that
server without crashing.
It will also output the error and request it to be reported.
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/selectors.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 74405a7..35051ee 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -460,6 +460,11 @@ class Deep(object):
except TimeoutException:
self.output.write(('deeptime(): connection to host %s '
'timed out for ip %s\n') % (url_parts.hostname, ip), 2)
+ except Exception as e: # Add general exception to catch any other errors
+ self.output.write(('deeptime(): connection to host %s '
+ 'errored for ip %s\n %s\n'
+ ' Please file a bug for this error at bugs.gentoo.org')
+ % (url_parts.hostname, ip, e), 2)
return f, test_url, early_out
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-31 4:08 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-31 4:08 UTC (permalink / raw
To: gentoo-commits
commit: c920bf656ab8d41ecc637741d49053a594302c3c
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May 31 04:07:35 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May 31 04:07:35 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=c920bf65
version.py: Bump base version to 2.3.0 before release
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/version.py b/mirrorselect/version.py
index ce4f7f4..c8216df 100644
--- a/mirrorselect/version.py
+++ b/mirrorselect/version.py
@@ -25,5 +25,5 @@ Distributed under the terms of the GNU General Public License v2
"""
-version = "2.2.6-git"
+version = "2.3.0-git"
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-31 4:08 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-31 4:08 UTC (permalink / raw
To: gentoo-commits
commit: efa29410a1ae8194c9d40866a2c893d414639d5b
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May 31 03:34:17 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May 31 03:34:17 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=efa29410
selectors.py: Fix bug 698528 clear screen on dialog exit
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/selectors.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 35051ee..1b1312c 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -571,6 +571,7 @@ class Interactive(object):
self.urls = out.splitlines()
+ sys.stderr.write("\x1b[2J\x1b[H")
if self.urls:
if hasattr(self.urls[0], 'decode'):
self.urls = decode_selection(
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-31 18:39 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-31 18:39 UTC (permalink / raw
To: gentoo-commits
commit: c32caece8690c2a88bd12d4aa30cb27d8bdf25ef
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May 31 17:51:17 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May 31 17:51:17 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=c32caece
Remove remaining SYNC code
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/configs.py | 9 ++-------
mirrorselect/main.py | 4 ++--
2 files changed, 4 insertions(+), 9 deletions(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
index 2dd6cba..93b2108 100644
--- a/mirrorselect/configs.py
+++ b/mirrorselect/configs.py
@@ -138,12 +138,10 @@ def write_repos_conf(output, config_path, var, value):
" variable: %s\nChanges NOT SAVED" %var)
-def get_filesystem_mirrors(output, config_path, sync=False):
+def get_filesystem_mirrors(output, config_path):
"""Read the current mirrors and retain mounted filesystems mirrors
@param config_path: string
- @param sync: boolean, used to switch between SYNC and GENTOO_MIRRORS
- make.conf variable target
@rtype list
"""
@@ -158,10 +156,7 @@ def get_filesystem_mirrors(output, config_path, sync=False):
fsmirrors = []
- if sync:
- var = 'SYNC'
- else:
- var = 'GENTOO_MIRRORS'
+ var = 'GENTOO_MIRRORS'
output.write('get_filesystem_mirrors(): config_path = %s\n' % config_path, 2)
try:
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
old mode 100755
new mode 100644
index 9cad25b..8a3094e
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -95,7 +95,7 @@ class MirrorSelect(object):
@param out: boolean, used to redirect output to stdout
@param config_path; string
@param sync: boolean, used to switch between sync-uri repos.conf target,
- SYNC and GENTOO_MIRRORS make.conf variable target
+ and GENTOO_MIRRORS make.conf variable target
"""
if sync:
var = "sync-uri"
@@ -371,7 +371,7 @@ class MirrorSelect(object):
self.output.print_err("main(); Exiting due to missing repos.conf/gentoo.conf file\n")
fsmirrors = get_filesystem_mirrors(self.output,
- config_path, options.rsync)
+ config_path)
hosts = self.get_available_hosts(options)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-31 18:39 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-31 18:39 UTC (permalink / raw
To: gentoo-commits
commit: 1d39a41a244b49b3e1fe1739eb8fdd2a01783877
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May 31 17:53:22 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May 31 17:53:22 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=1d39a41a
configs.py: Add missing https:// to the re.compile
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/configs.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
index d72ddf3..6d901a8 100644
--- a/mirrorselect/configs.py
+++ b/mirrorselect/configs.py
@@ -165,7 +165,7 @@ def get_filesystem_mirrors(output, config_path):
lex = shlex.shlex(f, posix=True)
lex.wordchars = string.digits + letters + r"~!@#$%*_\:;?,./-+{}"
lex.quotes = "\"'"
- p = re.compile('rsync://|http://|ftp://', re.IGNORECASE)
+ p = re.compile('rsync://|http://|https://|ftp://', re.IGNORECASE)
while 1:
key = get_token(lex)
#output.write('get_filesystem_mirrors(): processing key = %s\n' % key, 2)
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-31 18:39 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-31 18:39 UTC (permalink / raw
To: gentoo-commits
commit: c9552deb2d4b50b26e493e4a04b6adc5c65ed4df
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May 31 15:01:15 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May 31 15:01:15 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=c9552deb
selectors.py: Change general exception output to print_warn
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/selectors.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 1b1312c..8b5e28b 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -461,10 +461,10 @@ class Deep(object):
self.output.write(('deeptime(): connection to host %s '
'timed out for ip %s\n') % (url_parts.hostname, ip), 2)
except Exception as e: # Add general exception to catch any other errors
- self.output.write(('deeptime(): connection to host %s '
+ self.output.print_warn(('deeptime(): connection to host %s '
'errored for ip %s\n %s\n'
' Please file a bug for this error at bugs.gentoo.org')
- % (url_parts.hostname, ip, e), 2)
+ % (url_parts.hostname, ip, e), 0)
return f, test_url, early_out
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2022-05-31 18:39 Brian Dolbec
0 siblings, 0 replies; 86+ messages in thread
From: Brian Dolbec @ 2022-05-31 18:39 UTC (permalink / raw
To: gentoo-commits
commit: 889a9ce7e114a453447d120dbea864899bfaae8e
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue May 31 17:52:26 2022 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue May 31 17:52:26 2022 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=889a9ce7
configs.py: Remove py2 code
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
mirrorselect/configs.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
index 93b2108..d72ddf3 100644
--- a/mirrorselect/configs.py
+++ b/mirrorselect/configs.py
@@ -37,10 +37,7 @@ import string
import sys
-try: # py2
- letters = string.letters
-except AttributeError: # py3
- letters = string.ascii_letters
+letters = string.ascii_letters
def get_make_conf_path(EPREFIX):
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2023-07-06 8:05 Sam James
0 siblings, 0 replies; 86+ messages in thread
From: Sam James @ 2023-07-06 8:05 UTC (permalink / raw
To: gentoo-commits
commit: e0fbc2fb33762211aa5e64175d525e66f4c257d4
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 6 08:04:42 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Thu Jul 6 08:04:42 2023 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=e0fbc2fb
main: Fix --all option parsing
Avoid --all erroring out with "ERROR: Choose at most one of -s or -a".
Bug: https://bugs.gentoo.org/872218
Fixes: 7caac017833b01e13028658effc502430c56d770
Thanks-to: <xpenev <AT> gmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>
mirrorselect/main.py | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/mirrorselect/main.py b/mirrorselect/main.py
index 31f8e7b..7780bb7 100755
--- a/mirrorselect/main.py
+++ b/mirrorselect/main.py
@@ -145,6 +145,11 @@ class MirrorSelect(object):
self.output.white(" interactive:"),
" # mirrorselect -i -r",
))
+
+ def set_servers(option, opt_str, value, parser):
+ set_servers.user_configured = True
+ setattr(parser.values, option.dest, value)
+
parser = OptionParser(
formatter=ColoredFormatter(self.output), description=desc,
version='Mirrorselect version: %s' % version)
@@ -236,8 +241,8 @@ class MirrorSelect(object):
"-q", "--quiet", action="store_const", const=0, dest="verbosity",
help="Quiet mode")
group.add_option(
- "-s", "--servers", action="store", type="int", default=1,
- help="Specify Number of servers for Automatic Mode "
+ "-s", "--servers", action="callback", callback=set_servers,
+ type="int", default=1, help="Specify Number of servers for Automatic Mode "
"to select. this is only valid for download mirrors. "
"If this is not specified, a default of 1 is used.")
group.add_option(
@@ -271,7 +276,7 @@ class MirrorSelect(object):
if options.rsync and not (options.interactive or options.all_mirrors):
self.output.print_err('rsync servers can only be selected with -i or -a')
- if options.servers and options.all_mirrors:
+ if options.all_mirrors and hasattr(set_servers, 'user_configured'):
self.output.print_err('Choose at most one of -s or -a')
if options.interactive and (
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2023-08-06 23:30 Sam James
0 siblings, 0 replies; 86+ messages in thread
From: Sam James @ 2023-08-06 23:30 UTC (permalink / raw
To: gentoo-commits
commit: fe715a306754a4df2d05a3a24034d015c16377bf
Author: Peter Levine <plevine457 <AT> gmail <DOT> com>
AuthorDate: Mon Jul 24 22:40:02 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Aug 6 23:28:34 2023 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=fe715a30
extractor.py: parse proto from the uri
The protocol can be parsed from the URI so we can get rid of the
protocol tag altogether.
Bug: https://bugs.gentoo.org/911183
Suggested-by: Florian Schmaus <flow <AT> gentoo.org>
Suggested-by: Sam James <sam <AT> gentoo.org>
Signed-off-by: Peter Levine <plevine457 <AT> gmail.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>
mirrorselect/mirrorparser3.py | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 089f949..444bc11 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -44,6 +44,22 @@ class MirrorParser3:
def _reset(self):
self._dict = {}
+ def _get_proto(self, uri=None):
+ if not uri: # Don't parse if empty
+ return None;
+ try:
+ import sys;
+ if sys.version_info[0] >= 3:
+ from urllib.parse import urlparse
+ return urlparse(uri).scheme
+ else:
+ from urllib2 import Request
+ return Request(uri).get_type()
+ except Exception as e: # Add general exception to catch errors
+ from mirrorselect.output import Output
+ Output.write(('_get_proto(): Exception while parsing the protocol '
+ 'for URI %s: %s\n')% (uri, e), 2)
+
def parse(self, text):
self._reset()
for mirrorgroup in ET.XML(text):
@@ -60,7 +76,7 @@ class MirrorParser3:
"region": mirrorgroup.get("region"),
"ipv4": e.get("ipv4"),
"ipv6": e.get("ipv6"),
- "proto": e.get("protocol"),
+ "proto": e.get("protocol") or self._get_proto(uri),
}
def tuples(self):
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2023-08-06 23:30 Sam James
0 siblings, 0 replies; 86+ messages in thread
From: Sam James @ 2023-08-06 23:30 UTC (permalink / raw
To: gentoo-commits
commit: ac250b126b8de24276cd4e9bdc4afab14a9c41e7
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Sun Aug 6 23:29:25 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sun Aug 6 23:29:42 2023 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=ac250b12
extractor.py: cleanup py2 compat
Bug: https://bugs.gentoo.org/911183
Signed-off-by: Sam James <sam <AT> gentoo.org>
mirrorselect/mirrorparser3.py | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 444bc11..9bca3e9 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -48,13 +48,8 @@ class MirrorParser3:
if not uri: # Don't parse if empty
return None;
try:
- import sys;
- if sys.version_info[0] >= 3:
- from urllib.parse import urlparse
- return urlparse(uri).scheme
- else:
- from urllib2 import Request
- return Request(uri).get_type()
+ from urllib.parse import urlparse
+ return urlparse(uri).scheme
except Exception as e: # Add general exception to catch errors
from mirrorselect.output import Output
Output.write(('_get_proto(): Exception while parsing the protocol '
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2023-08-07 0:14 Sam James
0 siblings, 0 replies; 86+ messages in thread
From: Sam James @ 2023-08-07 0:14 UTC (permalink / raw
To: gentoo-commits
commit: 1a2eb56c27f0b1c284a13400fa9ee864575e8ba1
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 7 00:01:45 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Aug 7 00:04:33 2023 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=1a2eb56c
Drop old URLs
Signed-off-by: Sam James <sam <AT> gentoo.org>
mirrorselect/mirrorparser3.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 133420a..4023973 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -28,11 +28,6 @@ Distributed under the terms of the GNU General Public License v2
from xml.etree import ElementTree as ET
-# old url's
-#MIRRORS_3_XML = 'http://www.gentoo.org/main/en/mirrors3.xml'
-#MIRRORS_RSYNC_DATA = 'http://www.gentoo.org/main/en/mirrors-rsync-data.xml'
-
-# new urls
MIRRORS_3_XML = 'https://api.gentoo.org/mirrors/distfiles.xml'
MIRRORS_RSYNC_DATA = 'https://api.gentoo.org/mirrors/rsync.xml'
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2023-08-07 0:14 Sam James
0 siblings, 0 replies; 86+ messages in thread
From: Sam James @ 2023-08-07 0:14 UTC (permalink / raw
To: gentoo-commits
commit: 1c6189be723e5048737f45cf531e35ec8725b727
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 7 00:01:10 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Aug 7 00:04:33 2023 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=1c6189be
Drop Python 2 support
Signed-off-by: Sam James <sam <AT> gentoo.org>
mirrorselect/mirrorparser3.py | 15 +++++----------
mirrorselect/selectors.py | 27 +++++++--------------------
2 files changed, 12 insertions(+), 30 deletions(-)
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index f89c61e..133420a 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
"""Mirrorselect 2.x
Tool for selecting Gentoo source and rsync mirrors.
-Copyright 2009-2012 Gentoo Foundation
+Copyright 2009-2023 Gentoo Authors
Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org>
Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org>
@@ -81,14 +81,9 @@ class MirrorParser3:
if __name__ == '__main__':
import sys
- if sys.version_info[0] >= 3:
- import urllib.request, urllib.parse, urllib.error
- parser = MirrorParser3()
- parser.parse(urllib.request.urlopen(MIRRORS_3_XML).read())
- else:
- import urllib
- parser = MirrorParser3()
- parser.parse(urllib.urlopen(MIRRORS_3_XML).read())
+ import urllib.request, urllib.parse, urllib.error
+ parser = MirrorParser3()
+ parser.parse(urllib.request.urlopen(MIRRORS_3_XML).read())
print('===== tuples')
print(parser.tuples())
print('===== uris')
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index a17a646..df66cc4 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -37,26 +37,13 @@ import sys
import time
import hashlib
-if sys.version_info[0] >= 3:
- import urllib.request, urllib.parse, urllib.error
- url_parse = urllib.parse.urlparse
- url_unparse = urllib.parse.urlunparse
- url_open = urllib.request.urlopen
- url_request = urllib.request.Request
- HTTPError = urllib.error.HTTPError
- import http.client
- IncompleteRead = http.client.IncompleteRead
-else:
- import urllib2
- import urlparse
- url_parse = urlparse.urlparse
- url_unparse = urlparse.urlunparse
- url_open = urllib2.urlopen
- url_request = urllib2.Request
- HTTPError = urllib2.HTTPError
- import httplib
- IncompleteRead = httplib.IncompleteRead
-
+import urllib.request, urllib.parse, urllib.error
+url_parse = urllib.parse.urlparse
+url_unparse = urllib.parse.urlunparse
+url_open = urllib.request.urlopen
+url_request = urllib.request.Request
+HTTPError = urllib.error.HTTPError
+import http.client
from mirrorselect.output import encoder, get_encoding, decode_selection
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2023-08-07 0:14 Sam James
0 siblings, 0 replies; 86+ messages in thread
From: Sam James @ 2023-08-07 0:14 UTC (permalink / raw
To: gentoo-commits
commit: 816c1882de41dcf4a2fc65e455f95f8d8197883e
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 7 00:09:58 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Aug 7 00:12:28 2023 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=816c1882
Run `autopep8`
Signed-off-by: Sam James <sam <AT> gentoo.org>
mirrorselect/mirrorparser3.py | 4 +++-
mirrorselect/selectors.py | 17 ++++++++---------
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/mirrorselect/mirrorparser3.py b/mirrorselect/mirrorparser3.py
index 37245ff..3e32880 100644
--- a/mirrorselect/mirrorparser3.py
+++ b/mirrorselect/mirrorparser3.py
@@ -86,7 +86,9 @@ class MirrorParser3:
if __name__ == "__main__":
import sys
- import urllib.request, urllib.parse, urllib.error
+ import urllib.request
+ import urllib.parse
+ import urllib.error
parser = MirrorParser3()
parser.parse(urllib.request.urlopen(MIRRORS_3_XML).read())
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 4ec4474..60956d4 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -27,6 +27,8 @@ Distributed under the terms of the GNU General Public License v2
"""
+from mirrorselect.output import encoder, get_encoding, decode_selection
+import http.client
import math
import signal
import socket
@@ -36,16 +38,15 @@ import sys
import time
import hashlib
-import urllib.request, urllib.parse, urllib.error
+import urllib.request
+import urllib.parse
+import urllib.error
url_parse = urllib.parse.urlparse
url_unparse = urllib.parse.urlunparse
url_open = urllib.request.urlopen
url_request = urllib.request.Request
HTTPError = urllib.error.HTTPError
-import http.client
-
-from mirrorselect.output import encoder, get_encoding, decode_selection
# The netselect --ipv4 and --ipv6 options are supported only
@@ -251,7 +252,6 @@ class Deep:
self.dl_failures = 0
for host in hosts:
-
prog += 1
if self.test_file != "mirrorselect-test":
self.output.print_info(
@@ -289,7 +289,7 @@ class Deep:
rethosts = []
for key in keys:
# self.output.write('deeptest(): adding rethost '
- #'%s, %s' % (key, top_hosts[key]), 2)
+ # '%s, %s' % (key, top_hosts[key]), 2)
rethosts.append(top_hosts[key])
self.output.write("deeptest(): final rethost %s\n" % (rethosts), 2)
@@ -514,7 +514,6 @@ class Deep:
or the slowest host.
"""
if len(host_dict) < maxlen: # still have room, and host is fast. add it.
-
self.output.write(
"_list_add(): added host %s. with a time of %s\n"
% (time_host[1], time_host[0]),
@@ -609,7 +608,7 @@ class Interactive:
dialog.extend(["20", "110", "14"])
- for (url, args) in sorted(
+ for url, args in sorted(
hosts, key=lambda x: (x[1]["country"].lower(), x[1]["name"].lower())
):
marker = ""
@@ -623,7 +622,7 @@ class Interactive:
continue
# dialog.append('"%s" "%s%s: %s" "OFF"'
- #% ( url, marker, args['country'], args['name']))
+ # % ( url, marker, args['country'], args['name']))
dialog.extend(
[
"%s" % url,
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2023-08-07 0:23 Sam James
0 siblings, 0 replies; 86+ messages in thread
From: Sam James @ 2023-08-07 0:23 UTC (permalink / raw
To: gentoo-commits
commit: 1c44e110a44341f72ea056132dcaf3c4bfff9273
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 7 00:21:20 2023 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Aug 7 00:21:20 2023 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=1c44e110
Rerun `autopep8` with `--aggressive
Signed-off-by: Sam James <sam <AT> gentoo.org>
mirrorselect/configs.py | 4 ++--
mirrorselect/output.py | 2 +-
mirrorselect/selectors.py | 15 +++++----------
3 files changed, 8 insertions(+), 13 deletions(-)
diff --git a/mirrorselect/configs.py b/mirrorselect/configs.py
index 39dcdab..c7935d7 100644
--- a/mirrorselect/configs.py
+++ b/mirrorselect/configs.py
@@ -164,7 +164,7 @@ def get_filesystem_mirrors(output, config_path):
lex.wordchars = string.digits + letters + r"~!@#$%*_\:;?,./-+{}"
lex.quotes = "\"'"
p = re.compile("rsync://|http://|https://|ftp://", re.IGNORECASE)
- while 1:
+ while True:
key = get_token(lex)
# output.write('get_filesystem_mirrors(): processing key = %s\n' % key, 2)
@@ -181,7 +181,7 @@ def get_filesystem_mirrors(output, config_path):
mirrorlist = val.rsplit()
output.write("get_filesystem_mirrors(): mirrorlist = %s\n" % mirrorlist, 2)
for mirror in mirrorlist:
- if p.match(mirror) == None:
+ if p.match(mirror) is None:
if os.access(mirror, os.F_OK):
output.write(
"get_filesystem_mirrors(): found file system mirror = %s\n"
diff --git a/mirrorselect/output.py b/mirrorselect/output.py
index 8b33cff..87a83b2 100644
--- a/mirrorselect/output.py
+++ b/mirrorselect/output.py
@@ -53,7 +53,7 @@ def decode_selection(selection):
def get_encoding(output):
- if hasattr(output, "encoding") and output.encoding != None:
+ if hasattr(output, "encoding") and output.encoding is not None:
return output.encoding
else:
encoding = locale.getpreferredencoding()
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
index 60956d4..7f9fddb 100644
--- a/mirrorselect/selectors.py
+++ b/mirrorselect/selectors.py
@@ -168,8 +168,7 @@ class Shallow:
% (number, block_size, block_index, len(host_blocks))
)
- host_ranking_keys = list(ret_hosts.keys())
- host_ranking_keys.sort()
+ host_ranking_keys = sorted(ret_hosts.keys())
for rank in host_ranking_keys[:number]:
top_hosts.append(ret_hosts[rank])
@@ -283,8 +282,7 @@ class Deep:
# can't just return the dict.values,
# because we want the fastest mirror first...
- keys = list(top_hosts.keys())
- keys.sort()
+ keys = sorted(top_hosts.keys())
rethosts = []
for key in keys:
@@ -521,8 +519,7 @@ class Deep:
)
host_dict.update(dict([time_host]))
- times = list(host_dict.keys())
- times.sort()
+ times = sorted(host_dict.keys())
else: # We need to make room in the dict before we add. Kill the slowest.
self.output.write(
@@ -530,14 +527,12 @@ class Deep:
% (time_host[1], time_host[0]),
2,
)
- times = list(host_dict.keys())
- times.sort()
+ times = sorted(host_dict.keys())
self.output.write("_list_add(): removing %s\n" % host_dict[times[-1]], 2)
del host_dict[times[-1]]
host_dict.update(dict([time_host]))
# done adding. now return the appropriate time
- times = list(host_dict.keys())
- times.sort()
+ times = sorted(host_dict.keys())
if len(host_dict) < maxlen: # check again to choose new timeout
self.output.write(
^ permalink raw reply related [flat|nested] 86+ messages in thread
* [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/
@ 2024-05-28 3:48 Sam James
0 siblings, 0 replies; 86+ messages in thread
From: Sam James @ 2024-05-28 3:48 UTC (permalink / raw
To: gentoo-commits
commit: 55402ec0df846a42dba44c295b5761a770724ce8
Author: Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Mon May 27 18:47:13 2024 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Tue May 28 03:48:00 2024 +0000
URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=55402ec0
extractor.py: Replace sslfetch with plain requests
sslfetch was a thin NIH wrapper around requests, and it is unmaintained.
Closes: https://bugs.gentoo.org/932145
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>
Closes: https://github.com/gentoo/mirrorselect/pull/1
Signed-off-by: Sam James <sam <AT> gentoo.org>
| 26 ++++++++++----------------
1 file changed, 10 insertions(+), 16 deletions(-)
--git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
index 4598b8b..7326c86 100644
--- a/mirrorselect/extractor.py
+++ b/mirrorselect/extractor.py
@@ -27,8 +27,9 @@ Distributed under the terms of the GNU General Public License v2
import os
+import requests
+
from mirrorselect.mirrorparser3 import MirrorParser3
-from sslfetch.connections import Connector
from mirrorselect.version import version
USERAGENT = "Mirrorselect-" + version
@@ -103,21 +104,14 @@ class Extractor:
self.output.print_info("Downloading a list of mirrors...\n")
- # setup the ssl-fetch ouptut map
- connector_output = {
- "info": self.output.write,
- "debug": self.output.write,
- "error": self.output.print_err,
- "kwargs-info": {"level": 2},
- "kwargs-debug": {"level": 2},
- "kwargs-error": {"level": 0},
- }
-
- fetcher = Connector(connector_output, self.proxies, USERAGENT)
- success, mirrorlist, timestamp = fetcher.fetch_content(url, climit=60)
- parser.parse(mirrorlist)
-
- if (not mirrorlist) or len(parser.tuples()) == 0:
+ response = requests.get(url,
+ timeout=60,
+ proxies=self.proxies,
+ headers={"User-Agent": USERAGENT})
+ if response:
+ parser.parse(response.text)
+
+ if len(parser.tuples()) == 0:
self.output.print_err(
"Could not get mirror list. " "Check your internet connection."
)
^ permalink raw reply related [flat|nested] 86+ messages in thread
end of thread, other threads:[~2024-05-28 3:48 UTC | newest]
Thread overview: 86+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2012-11-12 7:46 [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/ Brian Dolbec
-- strict thread matches above, loose matches on Subject: below --
2024-05-28 3:48 Sam James
2023-08-07 0:23 Sam James
2023-08-07 0:14 Sam James
2023-08-07 0:14 Sam James
2023-08-07 0:14 Sam James
2023-08-06 23:30 Sam James
2023-08-06 23:30 Sam James
2023-07-06 8:05 Sam James
2022-05-31 18:39 Brian Dolbec
2022-05-31 18:39 Brian Dolbec
2022-05-31 18:39 Brian Dolbec
2022-05-31 18:39 Brian Dolbec
2022-05-31 4:08 Brian Dolbec
2022-05-31 4:08 Brian Dolbec
2022-05-31 2:22 Brian Dolbec
2022-05-31 2:22 Brian Dolbec
2022-05-30 23:12 Brian Dolbec
2022-05-30 23:12 Brian Dolbec
2020-06-03 19:05 Brian Dolbec
2019-07-17 5:06 Zac Medico
2019-05-27 17:52 Zac Medico
2019-05-27 17:22 Zac Medico
2019-02-13 8:22 Zac Medico
2019-02-13 8:09 Zac Medico
2019-02-13 5:51 Zac Medico
2018-05-26 15:43 Brian Dolbec
2017-02-21 4:44 Zac Medico
2017-02-21 3:19 Brian Dolbec
2017-02-21 3:19 Brian Dolbec
2016-11-15 1:16 Zac Medico
2015-01-27 18:20 Brian Dolbec
2015-01-27 4:38 Brian Dolbec
2015-01-27 4:38 Brian Dolbec
2014-05-28 21:08 Brian Dolbec
2014-05-28 19:44 Brian Dolbec
2014-05-05 2:04 Brian Dolbec
2014-05-05 2:04 Brian Dolbec
2014-05-05 2:04 Brian Dolbec
2014-03-02 7:44 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
2014-03-02 7:44 ` [gentoo-commits] proj/mirrorselect:master " Brian Dolbec
2014-03-02 7:44 Brian Dolbec
2014-03-02 7:44 Brian Dolbec
2014-03-02 7:44 Brian Dolbec
2014-03-02 7:44 Brian Dolbec
2014-03-02 7:44 Brian Dolbec
2014-03-02 7:44 Brian Dolbec
2014-03-02 7:44 Brian Dolbec
2014-01-31 15:44 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
2014-03-02 7:44 ` [gentoo-commits] proj/mirrorselect:master " Brian Dolbec
2013-10-20 18:19 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec
2014-03-02 7:44 ` [gentoo-commits] proj/mirrorselect:master " Brian Dolbec
2013-10-19 9:06 Brian Dolbec
2013-10-19 9:06 Brian Dolbec
2013-10-19 9:06 Brian Dolbec
2013-10-18 14:26 Brian Dolbec
2013-10-18 6:46 Brian Dolbec
2013-10-18 6:46 Brian Dolbec
2013-10-18 6:46 Brian Dolbec
2013-10-18 6:46 Brian Dolbec
2013-10-17 14:26 Brian Dolbec
2013-10-17 6:57 Brian Dolbec
2013-10-17 3:16 Brian Dolbec
2013-10-17 3:16 Brian Dolbec
2013-10-17 3:16 Brian Dolbec
2013-10-16 9:17 Brian Dolbec
2013-10-16 8:36 Brian Dolbec
2013-10-16 8:36 Brian Dolbec
2013-10-16 8:36 Brian Dolbec
2013-10-16 8:36 Brian Dolbec
2013-10-15 22:43 Brian Dolbec
2013-10-15 14:39 Brian Dolbec
2013-10-15 14:39 Brian Dolbec
2013-10-15 14:39 Brian Dolbec
2013-10-15 7:27 Brian Dolbec
2013-10-15 3:51 Brian Dolbec
2013-10-15 3:51 Brian Dolbec
2013-03-10 13:07 Brian Dolbec
2012-12-16 2:38 Brian Dolbec
2012-12-15 21:25 Brian Dolbec
2012-11-15 3:53 Brian Dolbec
2012-11-15 3:44 Brian Dolbec
2012-11-14 20:28 Paul Varner
2012-11-12 21:41 Brian Dolbec
2012-11-12 20:37 Brian Dolbec
2012-11-12 15:56 Brian Dolbec
2012-11-12 7:46 Brian Dolbec
2012-11-12 7:46 Brian Dolbec
2012-11-12 7:46 Brian Dolbec
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox