From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 3D860138247 for ; Sun, 19 Jan 2014 22:36:48 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 96F4AE0E42; Sun, 19 Jan 2014 22:36:46 +0000 (UTC) Received: from mail-we0-f176.google.com (mail-we0-f176.google.com [74.125.82.176]) (using TLSv1 with cipher ECDHE-RSA-RC4-SHA (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 88E1FE0E3A for ; Sun, 19 Jan 2014 22:36:45 +0000 (UTC) Received: by mail-we0-f176.google.com with SMTP id t61so2006770wes.35 for ; Sun, 19 Jan 2014 14:36:44 -0800 (PST) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:mime-version:sender:in-reply-to:references:date :message-id:subject:from:to:content-type; bh=K8CCaesENZ8uJbAPuDQeEpCleUTXrFa3M0MIYj/f4qI=; b=H5s7XqzMpJHDvDtCrNCZOd2gAtFhjM1bHUaU92O2VYSYpOonny4R8aseHT5UoUwCp7 oetDG5QKJKV3M83UscxQL1YD771m4Wx0ISoAmxEGiFlc0aBtSDHXiZMbqc5vczAjKtOE 84llMRZtSXr/U8mVf+j3tsMsl7AdEUoOQ/+Q56whSnQTyFk8jok1SsJqbSuc/+TZDb1n OpNzDA/DNdTC6otW2tngw+l2xwxE+cFKQK3w2+uSZYKW+YqMgekNoiWnVtULNlPz4okU XGbn+LwNYE4e3syXR85Ght4n1O1TyjhLv/hV3gMhMy6hpFYstIKbYqEHCm2ahFvLq80X BLNg== X-Gm-Message-State: ALoCoQn1QZLRcJ2QTRyOHwLHl0r0bZ0uHtn5HiPJXPty/3JqHxx/3qHNTKCvQfhtu4NCC1vt+upE Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org MIME-Version: 1.0 X-Received: by 10.180.211.141 with SMTP id nc13mr11347813wic.1.1390171004049; Sun, 19 Jan 2014 14:36:44 -0800 (PST) Sender: antarus@scriptkitty.com Received: by 10.216.170.129 with HTTP; Sun, 19 Jan 2014 14:36:43 -0800 (PST) X-Originating-IP: [173.8.165.226] In-Reply-To: <43632318589c05c4959f9dcf0bb39a6b0b50e8b8.1390099978.git.wking@tremily.us> References: <43632318589c05c4959f9dcf0bb39a6b0b50e8b8.1390099978.git.wking@tremily.us> Date: Sun, 19 Jan 2014 14:36:43 -0800 X-Google-Sender-Auth: 75NEXCwXX8CnWkUd0HSLa6MTnsw Message-ID: Subject: Re: [gentoo-portage-dev] [PATCH 3/3] pym/portage/package/ebuild/fetch.py: Factor out _get_uris From: Alec Warner To: gentoo-portage-dev@lists.gentoo.org Content-Type: multipart/alternative; boundary=001a11c3709cdfb13204f05a6892 X-Archives-Salt: b72845a3-adb0-4cde-82be-d871d6f106ce X-Archives-Hash: 5f11095bb7d0f8dca8ec2e749f33fabf --001a11c3709cdfb13204f05a6892 Content-Type: text/plain; charset=UTF-8 On Sat, Jan 18, 2014 at 7:07 PM, W. Trevor King wrote: > The current fetch() function is quite long, which makes it hard to > know what I can change without adverse side effects. By pulling this > logic out of the main function, we get clearer logic in fetch() and > more explicit input for the config extraction. > > This block was especially complicated, so I also created the helper > functions _get_file_uri_tuples and _expand_mirror. I'd prefer if > _expand_mirror iterated through URIs instead of (group, URI) tuples, > but we need a distinct marker for third-party URIs to build > third_party_mirror_uris which is used to build primaryuri_dict which > is used way down in fetch(): > > if checksum_failure_count == \ > checksum_failure_primaryuri: > # Switch to "primaryuri" mode in order > # to increase the probablility of > # of success. > primaryuris = \ > primaryuri_dict.get(myfile) > if primaryuris: > uri_list.extend( > reversed(primaryuris)) > > I don't know if this is useful enough to motivate the uglier > _expandmirror return values, but I'll kick that can down the road for > now. > --- > pym/portage/package/ebuild/fetch.py | 197 > +++++++++++++++++++++--------------- > 1 file changed, 117 insertions(+), 80 deletions(-) > > diff --git a/pym/portage/package/ebuild/fetch.py > b/pym/portage/package/ebuild/fetch.py > index bd572fa..a617945 100644 > --- a/pym/portage/package/ebuild/fetch.py > +++ b/pym/portage/package/ebuild/fetch.py > @@ -15,9 +15,9 @@ import sys > import tempfile > > try: > - from urllib.parse import urlparse > + from urllib.parse import urlparse, urlunparse > except ImportError: > - from urlparse import urlparse > + from urlparse import urlparse, urlunparse > > import portage > portage.proxy.lazyimport.lazyimport(globals(), > @@ -297,6 +297,118 @@ def _get_fetch_resume_size(settings, default='350K'): > return v > > > +def _get_file_uri_tuples(uris): > + """Return a list of (filename, uri) tuples > + """ > As mike noted on another thread: ""Return a list of (filename, uri) tuples.""" + file_uri_tuples = [] > + # Check for 'items' attribute since OrderedDict is not a dict. > + if hasattr(uris, 'items'): > + for filename, uri_set in uris.items(): > + for uri in uri_set: > + file_uri_tuples.append((filename, uri)) > + if not uri_set: > + file_uri_tuples.append((filename, None)) > + else: > + for uri in uris: > + if urlparse(uri).scheme: > + file_uri_tuples.append( > + (os.path.basename(myuri), myuri)) > + else: > + file_uri_tuples.append( > + (os.path.basename(myuri), None)) > + return file_uri_tuples > + > + > +def _expand_mirror(uri, custom_mirrors=(), third_party_mirrors=()): > + """Replace the 'mirror://' scheme in the uri > Sentence should end in a period. > + > + Returns an iterable listing expanded (group, URI) tuples, > + where the group is either 'custom' or 'third-party'. > + """ > + parsed = urlparse(uri) > + mirror = parsed.netloc > + path = parsed.path > + if path: > + # Try user-defined mirrors first > + if mirror in custom_mirrors: > + for cmirr in custom_mirrors[mirror]: > + m_uri = urlparse(cmirr) > + yield ('custom', urlunparse(( > + cmirr.scheme, cmirr.netloc, path) + > + parsed[3:])) > + > + # now try the official mirrors > + if mirror in third_party_mirrors: > + uris = [] > + for locmirr in third_party_mirrors[mirror]: > + m_uri = urlparse(cmirr) > + uris.append(urlunparse(( > + cmirr.scheme, cmirr.netloc, path) + > + parsed[3:])) > + random.shuffle(uris) > + for uri in uris: > + yield ('third-party', uri) > + else: > + writemsg(_("Invalid mirror definition in SRC_URI:\n"), > + noiselevel=-1) > + writemsg(" %s\n" % (uri), noiselevel=-1) Is this a py3k thing? You should be able to write: writemsg(" %s\n" % uri, noiselevel=-1) The tuple is only needed for multiple arguments in py2k. + > + > +def _get_uris(uris, settings, custom_mirrors=(), locations=()): > I want you to be careful here. This is bad style when you are constructing mutable objects in parameter defaults: >>> def func(a=[]): ... a.append(5) ... >>> del func >>> def func(a=[]): ... a.append(5) ... print a ... >>> func() [5] >>> func() [5, 5] >>> func() [5, 5, 5] >>> func() [5, 5, 5, 5] >>> func() [5, 5, 5, 5, 5] That doesn't function as most would expect, because it turns out that 'a' is constructed at function definition time (not call time) and is re-used between function calls. Now of course, your code is constructing tuples, which are immutable. That is handy. I'm not sure if I appreciate that more or less than the other form.... def _get_uris(uris, settings, custom_mirrors=None, locations=None): if not custom_mirrors: custom_mirrors = () if not locations: locations = () Another advisable way to write it is to simply not have default arguments, and to force the caller to sync in an empty iterable: def_get_uris(uris, settings, custom_mirrors, locations): ... and so they will be forced to create the empty iterables at the call site. A brief discussion with Sebastian on irc seemed to indicate that he thought creating immutable objects in this way was permissible (we certainly do it often for strings) so I won't raise a fuss. I merely want to point out that you really need to be aware of whether the objects are mutable or immutable. > + third_party_mirrors = settings.thirdpartymirrors() > Why pass in all of settings? If you only need settings.thirdpartymirrors, then just ask for 'a thirdparty mirrors iterable' or whatever it is. You do not want the entire settings object, it is full of disgusting globals, and it will only screw you over later. Think about testing this function. Do you really want to try to construct some sort of 'testing' settings object, or simply construct a list? > + third_party_mirror_uris = {} > + filedict = OrderedDict() > + primaryuri_dict = {} > + for filename, uri in _get_file_uri_tuples(uris=uris): > + if filename not in filedict: > + filedict[filename] = [ > + os.path.join(location, 'distfiles', > filename) > + for location in locations] > + if uri is None: > + continue > + if uri.startswith('mirror://'): > + uris = _expand_mirror( > too many uri / uris variables... can we called this 'expanded_uris'? I'm really having trouble tracking what is what. Remember that 'uris' is already a parameter to this function. Are you shadowing it here, or trying to overwrite it? > + uri=uri, custom_mirrors=custom_mirrors, > + third_party_mirrors=third_party_mirrors) > + filedict[filename].extend(uri for group, uri in > uris) > group appears unused in your implicit iterable here. perhaps: filedict[filename].extend(uri for _, uri in uris) > + third_party_mirror_uris.setdefault(filename, > []).extend( > + uri for group, uri in uris > + if group == 'third-party') > I'm curious if this matters. We are iterator over 'uris' twice. Is it cheaper to do it once and build the iterables once? third_party_uris = [] for group, uri in uris: if group == 'third_party': third_party_uris.append(uri) filedict[filename].append(uri) I'm guessing the iterable is so small it doesn't matter. > + if not filedict[filename]: > + writemsg( > + _("No known mirror by the name: > %s\n") > + % (mirror,)) > + else: > + if restrict_fetch or force_mirror: > Are these globals or am I missing somethng? > + # Only fetch from specific mirrors is > allowed. > + continue > + primaryuris = primaryuri_dict.get(filename) > + if primaryuris is None: > + primaryuris = [] > + primaryuri_dict[filename] = primaryuris > + primaryuris.append(uri) > + > + # Order primaryuri_dict values to match that in SRC_URI. > + for uris in primaryuri_dict.values(): > + uris.reverse() > Is there any guaranteed ordering for dict.values()? I thought dict order was random (and they seriously make it random in modern python versions, to detect bugs.) How does this uris.reverse() match the SRC_URI ordering? > + > + # Prefer third_party_mirrors over normal mirrors in cases when > + # the file does not yet exist on the normal mirrors. > + for filename, uris in third_party_mirror_uris.items(): > + primaryuri_dict.setdefault(filename, []).extend(uris) > + > + # Now merge primaryuri values into filedict (includes mirrors > + # explicitly referenced in SRC_URI). > + if "primaryuri" in restrict: > same questoin here about where 'restrict' comes from. > + for filename, uris in filedict.items(): > + filedict[filename] = primaryuri_dict.get(filename, > []) + uris > + else: > + for filename in filedict: > + filedict[filename] += > primaryuri_dict.get(filename, []) > + > + return filedict, primaryuri_dict > + > + > def fetch(myuris, mysettings, listonly=0, fetchonly=0, > locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None, > allow_missing_digests=True): > @@ -328,7 +440,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, > # couple of checksum failures, to increase the probablility > # of success before checksum_failure_max_tries is reached. > checksum_failure_primaryuri = 2 > - thirdpartymirrors = mysettings.thirdpartymirrors() > > # In the background parallel-fetch process, it's safe to skip > checksum > # verification of pre-existing files in $DISTDIR that have the > correct > @@ -412,83 +523,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, > else: > locations = mymirrors > > - file_uri_tuples = [] > - # Check for 'items' attribute since OrderedDict is not a dict. > - if hasattr(myuris, 'items'): > - for myfile, uri_set in myuris.items(): > - for myuri in uri_set: > - file_uri_tuples.append((myfile, myuri)) > - if not uri_set: > - file_uri_tuples.append((myfile, None)) > - else: > - for myuri in myuris: > - if urlparse(myuri).scheme: > - > file_uri_tuples.append((os.path.basename(myuri), myuri)) > - else: > - > file_uri_tuples.append((os.path.basename(myuri), None)) > - > - filedict = OrderedDict() > - primaryuri_dict = {} > - thirdpartymirror_uris = {} > - for myfile, myuri in file_uri_tuples: > - if myfile not in filedict: > - filedict[myfile]=[] > - for y in range(0,len(locations)): > - > filedict[myfile].append(locations[y]+"/distfiles/"+myfile) > - if myuri is None: > - continue > - if myuri[:9]=="mirror://": > - eidx = myuri.find("/", 9) > - if eidx != -1: > - mirrorname = myuri[9:eidx] > - path = myuri[eidx+1:] > - > - # Try user-defined mirrors first > - if mirrorname in custommirrors: > - for cmirr in > custommirrors[mirrorname]: > - filedict[myfile].append( > - cmirr.rstrip("/") > + "/" + path) > - > - # now try the official mirrors > - if mirrorname in thirdpartymirrors: > - uris = [locmirr.rstrip("/") + "/" > + path \ > - for locmirr in > thirdpartymirrors[mirrorname]] > - random.shuffle(uris) > - filedict[myfile].extend(uris) > - > thirdpartymirror_uris.setdefault(myfile, []).extend(uris) > - > - if not filedict[myfile]: > - writemsg(_("No known mirror by the > name: %s\n") % (mirrorname)) > - else: > - writemsg(_("Invalid mirror definition in > SRC_URI:\n"), noiselevel=-1) > - writemsg(" %s\n" % (myuri), noiselevel=-1) > - else: > - if restrict_fetch or force_mirror: > - # Only fetch from specific mirrors is > allowed. > - continue > - primaryuris = primaryuri_dict.get(myfile) > - if primaryuris is None: > - primaryuris = [] > - primaryuri_dict[myfile] = primaryuris > - primaryuris.append(myuri) > - > - # Order primaryuri_dict values to match that in SRC_URI. > - for uris in primaryuri_dict.values(): > - uris.reverse() > - > - # Prefer thirdpartymirrors over normal mirrors in cases when > - # the file does not yet exist on the normal mirrors. > - for myfile, uris in thirdpartymirror_uris.items(): > - primaryuri_dict.setdefault(myfile, []).extend(uris) > - > - # Now merge primaryuri values into filedict (includes mirrors > - # explicitly referenced in SRC_URI). > - if "primaryuri" in restrict: > - for myfile, uris in filedict.items(): > - filedict[myfile] = primaryuri_dict.get(myfile, []) > + uris > - else: > - for myfile in filedict: > - filedict[myfile] += primaryuri_dict.get(myfile, []) > + filedict, primaryuri_dict = _get_uris( > + uris=myuris, settings=mysettings, > + custom_mirrors=custom_mirrors, locations=locations) > > can_fetch=True > > -- > 1.8.5.2.8.g0f6c0d1 > > > --001a11c3709cdfb13204f05a6892 Content-Type: text/html; charset=UTF-8 Content-Transfer-Encoding: quoted-printable
On S= at, Jan 18, 2014 at 7:07 PM, W. Trevor King <wking@tremily.us> wrote:
The current fetch() function is quite long, which makes it= hard to
know what I can change without adverse side effects. =C2=A0By pulling this<= br> logic out of the main function, we get clearer logic in fetch() and
more explicit input for the config extraction.

This block was especially complicated, so I also created the helper
functions _get_file_uri_tuples and _expand_mirror. =C2=A0I'd prefer if<= br> _expand_mirror iterated through URIs instead of (group, URI) tuples,
but we need a distinct marker for third-party URIs to build
third_party_mirror_uris which is used to build primaryuri_dict which
is used way down in fetch():

=C2=A0 if checksum_failure_count =3D=3D \
=C2=A0 =C2=A0 =C2=A0 checksum_failure_primaryuri:
=C2=A0 =C2=A0 =C2=A0 # Switch to "primaryuri" mode in order
=C2=A0 =C2=A0 =C2=A0 # to increase the probablility of
=C2=A0 =C2=A0 =C2=A0 # of success.
=C2=A0 =C2=A0 =C2=A0 primaryuris =3D \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 primaryuri_dict.get(myfile)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if primaryuris:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 uri_list.extend(
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 reversed(pri= maryuris))

I don't know if this is useful enough to motivate the uglier
_expandmirror return values, but I'll kick that can down the road for now.
---
=C2=A0pym/portage/package/ebuild/fetch.py | 197 +++++++++++++++++++++------= ---------
=C2=A01 file changed, 117 insertions(+), 80 deletions(-)

diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebui= ld/fetch.py
index bd572fa..a617945 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -15,9 +15,9 @@ import sys
=C2=A0import tempfile

=C2=A0try:
- =C2=A0 =C2=A0 =C2=A0 from urllib.parse import urlparse
+ =C2=A0 =C2=A0 =C2=A0 from urllib.parse import urlparse, urlunparse
=C2=A0except ImportError:
- =C2=A0 =C2=A0 =C2=A0 from urlparse import urlparse
+ =C2=A0 =C2=A0 =C2=A0 from urlparse import urlparse, urlunparse

=C2=A0import portage
=C2=A0portage.proxy.lazyimport.lazyimport(globals(),
@@ -297,6 +297,118 @@ def _get_fetch_resume_size(settings, default=3D'3= 50K'):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 return v


+def _get_file_uri_tuples(uris):
+ =C2=A0 =C2=A0 =C2=A0 """Return a list of (filename, uri) t= uples
+ =C2=A0 =C2=A0 =C2=A0 """

As mike noted on another thread:

""Re= turn a list of (filename, uri) tuples."""

+ =C2=A0 =C2=A0 =C2=A0 file_uri_tuples =3D []
+ =C2=A0 =C2=A0 =C2=A0 # Check for 'items' attribute since OrderedD= ict is not a dict.
+ =C2=A0 =C2=A0 =C2=A0 if hasattr(uris, 'items'):
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for filename, uri_set in= uris.items():
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 for uri in uri_set:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 file_uri_tuples.append((filename, uri))
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if not uri_set:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 file_uri_tuples.append((filename, None)) + =C2=A0 =C2=A0 =C2=A0 else:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for uri in uris:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if urlparse(uri).scheme:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 file_uri_tuples.append(
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 (os.path.basena= me(myuri), myuri))
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 else:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 file_uri_tuples.append(
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 (os.path.basena= me(myuri), None))
+ =C2=A0 =C2=A0 =C2=A0 return file_uri_tuples
+
+
+def _expand_mirror(uri, custom_mirrors=3D(), third_party_mirrors=3D()): + =C2=A0 =C2=A0 =C2=A0 """Replace the 'mirror://' sc= heme in the uri

Sentence should end in = a period.
=C2=A0
+
+ =C2=A0 =C2=A0 =C2=A0 Returns an iterable listing expanded (group, URI) tu= ples,
+ =C2=A0 =C2=A0 =C2=A0 where the group is either 'custom' or 't= hird-party'.
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
+ =C2=A0 =C2=A0 =C2=A0 parsed =3D urlparse(uri)
+ =C2=A0 =C2=A0 =C2=A0 mirror =3D parsed.netloc
+ =C2=A0 =C2=A0 =C2=A0 path =3D parsed.path
+ =C2=A0 =C2=A0 =C2=A0 if path:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # Try user-defined mirro= rs first
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if mirror in custom_mirr= ors:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 for cmirr in custom_mirrors[mirror]:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 m_uri =3D urlparse(cmirr)
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 yield ('custom', urlunparse((
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cmirr.scheme, c= mirr.netloc, path) +
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 parsed[3:])) +
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # now try the official m= irrors
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if mirror in third_party= _mirrors:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 uris =3D []
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 for locmirr in third_party_mirrors[mirror]:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 m_uri =3D urlparse(cmirr)
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 uris.append(urlunparse((
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cmirr.scheme, c= mirr.netloc, path) +
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 parsed[3:])) + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 random.shuffle(uris)
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 for uri in uris:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 yield ('third-party', uri)
+ =C2=A0 =C2=A0 =C2=A0 else:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 writemsg(_("Invalid= mirror definition in SRC_URI:\n"),
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0noiselevel=3D-1)
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 writemsg(" =C2=A0%s= \n" % (uri), noiselevel=3D-1)

Is this = a py3k thing? You should be able to write:

writems= g(" =C2=A0%s\n" % uri, noiselevel=3D-1)

The tuple is only needed for multiple arguments in py2k= .

=C2=A0
+
+
+def _get_uris(uris, settings, custom_mirrors=3D(), locations=3D()):

I want you to be careful here. This is bad= style when you are constructing mutable objects in parameter defaults:

>>> def func(a=3D[]):
... =C2= =A0 a.append(5)
...=C2=A0
>>> del func
>>> def f= unc(a=3D[]):
... =C2=A0 a.append(5)
... =C2=A0 print a<= /div>
...=C2=A0
>>> func()
[5]
&= gt;>> func()
[5, 5]
>>> func()
[5, 5, 5]
>= ;>> func()
[5, 5, 5, 5]
>>> func()
=
[5, 5, 5, 5, 5]

That doesn't functi= on as most would expect, because it turns out that 'a' is construct= ed at function definition time (not call time) and is re-used between funct= ion calls.

Now of course, your code is constructing tuples, which = are immutable. That is handy.

I'm not sure if = I appreciate that more or less than the other form....

def _get_uris(uris, settings, custom_mirrors=3DNone, locations=3DNone)= :
=C2=A0 if not custom_mirrors:
=C2=A0 =C2=A0 custo= m_mirrors =3D ()
=C2=A0 if not locations:
=C2=A0 =C2=A0= locations =3D ()

Another advisable way to write it is to simply not have default argume= nts, and to force the caller to sync in an empty iterable:

def_get_uris(uris, settings, custom_mirrors, locations):
...

and so they will be forced to create the = empty iterables at the call site.

A brief discussi= on with Sebastian on irc seemed to indicate that he thought creating immuta= ble objects in this way was permissible (we certainly do it often for strin= gs) so I won't raise a fuss. I merely want to point out that you really= need to be aware of whether the objects are mutable or immutable.
=C2=A0
+ =C2=A0 =C2=A0 =C2=A0 third_party_mirrors =3D settings.thirdpartymirrors()=

Why pass in all of settings? If you on= ly need settings.thirdpartymirrors, then just ask for 'a thirdparty mir= rors iterable' or whatever it is. You do not want the entire settings o= bject, it is full of disgusting globals, and it will only screw you over la= ter.

Think about testing this function. Do you really want t= o try to construct some sort of 'testing' settings object, or simpl= y construct a list?
=C2=A0
+ =C2=A0 =C2=A0 =C2=A0 third_party_mirror_uris =3D {}
+ =C2=A0 =C2=A0 =C2=A0 filedict =3D OrderedDict()
+ =C2=A0 =C2=A0 =C2=A0 primaryuri_dict =3D {}
+ =C2=A0 =C2=A0 =C2=A0 for filename, uri in _get_file_uri_tuples(uris=3Duri= s):
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if filename not in filed= ict:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 filedict[filename] =3D [
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 os.path.join(location, 'distfiles',= filename)
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for location in locations]
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if uri is None:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 continue
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if uri.startswith('m= irror://'):
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 uris =3D _expand_mirror(

too many u= ri / uris variables...

can we called this 'exp= anded_uris'?

I'm really having trouble tra= cking what is what. Remember that 'uris' is already a parameter to = this function. Are you shadowing it here, or trying to overwrite it?
=C2=A0
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 uri=3Duri, custom_mirrors=3Dcustom_mirrors,=
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 third_party_mirrors=3Dthird_party_mirrors)<= br> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 filedict[filename].extend(uri for group, uri in uris)
<= div>
group appears unused in your implicit iterable here.

perhaps:

filedict[filename].extend(uri for _, uri in uris)
=C2=A0
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 third_party_mirror_uris.setdefault(filename, []).extend(
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 uri for group, uri in uris
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if group =3D=3D 'third-party')
<= /blockquote>

I'm curious if this matters. We are ite= rator over 'uris' twice. Is it cheaper to do it once and build the = iterables once?

third_party_uris =3D []
for group, uri in uri= s:
=C2=A0 if group =3D=3D 'third_party':
=C2=A0= =C2=A0 third_party_uris.append(uri)
=C2=A0 filedict[filename].ap= pend(uri)

I'm guessing the iterable is so small it doesn't matter.=
=C2=A0=C2=A0
=C2=A0
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if not filedict[filename]:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 writemsg(
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 _("No know= n mirror by the name: %s\n")
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 % (mirror,)) + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 else:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if restrict_fetch or force_mirror:

= Are these globals or am I missing somethng?
=C2=A0
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # Only fetch from specific mirrors is allow= ed.
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 continue
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 primaryuris =3D primaryuri_dict.get(filename)
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if primaryuris is None:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 primaryuris =3D []
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 primaryuri_dict[filename] =3D primaryuris + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 primaryuris.append(uri)
+
+ =C2=A0 =C2=A0 =C2=A0 # Order primaryuri_dict values to match that in SRC_= URI.
+ =C2=A0 =C2=A0 =C2=A0 for uris in primaryuri_dict.values():
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 uris.reverse()

Is there any guaranteed ordering for dict.values= ()? I thought dict order was random (and they seriously make it random in m= odern python versions, to detect bugs.) How does this uris.reverse() match = the SRC_URI ordering?
=C2=A0
+
+ =C2=A0 =C2=A0 =C2=A0 # Prefer third_party_mirrors over normal mirrors in = cases when
+ =C2=A0 =C2=A0 =C2=A0 # the file does not yet exist on the normal mirrors.=
+ =C2=A0 =C2=A0 =C2=A0 for filename, uris in third_party_mirror_uris.items(= ):
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 primaryuri_dict.setdefau= lt(filename, []).extend(uris)
+
+ =C2=A0 =C2=A0 =C2=A0 # Now merge primaryuri values into filedict (include= s mirrors
+ =C2=A0 =C2=A0 =C2=A0 # explicitly referenced in SRC_URI).
+ =C2=A0 =C2=A0 =C2=A0 if "primaryuri" in restrict:

same questoin here about where 'restrict' co= mes from.
=C2=A0
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for filename, uris in fi= ledict.items():
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 filedict[filename] =3D primaryuri_dict.get(filename, []) + uris
+ =C2=A0 =C2=A0 =C2=A0 else:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for filename in filedict= :
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 filedict[filename] +=3D primaryuri_dict.get(filename, [])
+
+ =C2=A0 =C2=A0 =C2=A0 return filedict, primaryuri_dict
+
+
=C2=A0def fetch(myuris, mysettings, listonly=3D0, fetchonly=3D0,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 locks_in_subdir=3D".locks", use_locks= =3D1, try_mirrors=3D1, digests=3DNone,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 allow_missing_digests=3DTrue):
@@ -328,7 +440,6 @@ def fetch(myuris, mysettings, listonly=3D0, fetchonly= =3D0,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 # couple of checksum failures, to increase the = probablility
=C2=A0 =C2=A0 =C2=A0 =C2=A0 # of success before checksum_failure_max_tries = is reached.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 checksum_failure_primaryuri =3D 2
- =C2=A0 =C2=A0 =C2=A0 thirdpartymirrors =3D mysettings.thirdpartymirrors()=

=C2=A0 =C2=A0 =C2=A0 =C2=A0 # In the background parallel-fetch process, it&= #39;s safe to skip checksum
=C2=A0 =C2=A0 =C2=A0 =C2=A0 # verification of pre-existing files in $DISTDI= R that have the correct
@@ -412,83 +523,9 @@ def fetch(myuris, mysettings, listonly=3D0, fetchonly= =3D0,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 else:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 locations =3D mymir= rors

- =C2=A0 =C2=A0 =C2=A0 file_uri_tuples =3D []
- =C2=A0 =C2=A0 =C2=A0 # Check for 'items' attribute since OrderedD= ict is not a dict.
- =C2=A0 =C2=A0 =C2=A0 if hasattr(myuris, 'items'):
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for myfile, uri_set in m= yuris.items():
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 for myuri in uri_set:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 file_uri_tuples.append((myfile, myuri))
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if not uri_set:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 file_uri_tuples.append((myfile, None))
- =C2=A0 =C2=A0 =C2=A0 else:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for myuri in myuris:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if urlparse(myuri).scheme:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 file_uri_tuples.append((os.path.basename(my= uri), myuri))
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 else:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 file_uri_tuples.append((os.path.basename(my= uri), None))
-
- =C2=A0 =C2=A0 =C2=A0 filedict =3D OrderedDict()
- =C2=A0 =C2=A0 =C2=A0 primaryuri_dict =3D {}
- =C2=A0 =C2=A0 =C2=A0 thirdpartymirror_uris =3D {}
- =C2=A0 =C2=A0 =C2=A0 for myfile, myuri in file_uri_tuples:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if myfile not in filedic= t:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 filedict[myfile]=3D[]
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 for y in range(0,len(locations)):
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 filedict[myfile].append(locations[y]+"= /distfiles/"+myfile)
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if myuri is None:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 continue
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if myuri[:9]=3D=3D"= mirror://":
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 eidx =3D myuri.find("/", 9)
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if eidx !=3D -1:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 mirrorname =3D myuri[9:eidx]
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 path =3D myuri[eidx+1:]
-
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # Try user-defined mirrors first
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if mirrorname in custommirrors:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for cmirr in cu= stommirrors[mirrorname]:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 filedict[myfile].append(
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cmirr.rstrip("/") + &qu= ot;/" + path)
-
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # now try the official mirrors
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if mirrorname in thirdpartymirrors:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 uris =3D [locmi= rr.rstrip("/") + "/" + path \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 for locmirr in thirdpartymirrors[mirrorname]]
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 random.shuffle(= uris)
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 filedict[myfile= ].extend(uris)
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 thirdpartymirro= r_uris.setdefault(myfile, []).extend(uris)
-
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if not filedict[myfile]:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 writemsg(_(&quo= t;No known mirror by the name: %s\n") % (mirrorname))
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 else:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 writemsg(_("Invalid mirror definition = in SRC_URI:\n"), noiselevel=3D-1)
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 writemsg(" =C2=A0%s\n" % (myuri),= noiselevel=3D-1)
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 else:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if restrict_fetch or force_mirror:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # Only fetch from specific mirrors is allow= ed.
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 continue
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 primaryuris =3D primaryuri_dict.get(myfile)
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if primaryuris is None:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 primaryuris =3D []
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 primaryuri_dict[myfile] =3D primaryuris
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 primaryuris.append(myuri)
-
- =C2=A0 =C2=A0 =C2=A0 # Order primaryuri_dict values to match that in SRC_= URI.
- =C2=A0 =C2=A0 =C2=A0 for uris in primaryuri_dict.values():
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 uris.reverse()
-
- =C2=A0 =C2=A0 =C2=A0 # Prefer thirdpartymirrors over normal mirrors in ca= ses when
- =C2=A0 =C2=A0 =C2=A0 # the file does not yet exist on the normal mirrors.=
- =C2=A0 =C2=A0 =C2=A0 for myfile, uris in thirdpartymirror_uris.items(): - =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 primaryuri_dict.setdefau= lt(myfile, []).extend(uris)
-
- =C2=A0 =C2=A0 =C2=A0 # Now merge primaryuri values into filedict (include= s mirrors
- =C2=A0 =C2=A0 =C2=A0 # explicitly referenced in SRC_URI).
- =C2=A0 =C2=A0 =C2=A0 if "primaryuri" in restrict:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for myfile, uris in file= dict.items():
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 filedict[myfile] =3D primaryuri_dict.get(myfile, []) + uris
- =C2=A0 =C2=A0 =C2=A0 else:
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for myfile in filedict:<= br> - =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 filedict[myfile] +=3D primaryuri_dict.get(myfile, [])
+ =C2=A0 =C2=A0 =C2=A0 filedict, primaryuri_dict =3D _get_uris(
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 uris=3Dmyuris, settings= =3Dmysettings,
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 custom_mirrors=3Dcustom_= mirrors, locations=3Dlocations)

=C2=A0 =C2=A0 =C2=A0 =C2=A0 can_fetch=3DTrue

--
1.8.5.2.8.g0f6c0d1



--001a11c3709cdfb13204f05a6892--