From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 12D7113800E for ; Thu, 2 Aug 2012 00:57:30 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id D81B1E07D2; Thu, 2 Aug 2012 00:57:12 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id AB039E07CF for ; Thu, 2 Aug 2012 00:57:12 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 142B91B4025 for ; Thu, 2 Aug 2012 00:57:12 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 544F9E543F for ; Thu, 2 Aug 2012 00:57:10 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <1343868855.cbebf76d8e5666aad4984f87c2be83d474fe5a7e.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: pym/portage/util/ X-VCS-Repository: proj/portage X-VCS-Files: pym/portage/util/_urlopen.py X-VCS-Directories: pym/portage/util/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: cbebf76d8e5666aad4984f87c2be83d474fe5a7e X-VCS-Branch: master Date: Thu, 2 Aug 2012 00:57:10 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 0d9bfb2a-edef-4861-b602-cf24a9f26f33 X-Archives-Hash: 5f542da3bdc7e950d19cceed54ab3e68 commit: cbebf76d8e5666aad4984f87c2be83d474fe5a7e Author: W-Mark Kubacki hurrikane de> AuthorDate: Wed Aug 1 18:36:31 2012 +0000 Commit: Zac Medico gentoo org> CommitDate: Thu Aug 2 00:54:15 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cbebf76d Add support for HTTP compression (bzip2, gzip and deflate). --- pym/portage/util/_urlopen.py | 32 +++++++++++++++++++++++++++++++- 1 files changed, 31 insertions(+), 1 deletions(-) diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py index 4296188..bcd8f7c 100644 --- a/pym/portage/util/_urlopen.py +++ b/pym/portage/util/_urlopen.py @@ -1,6 +1,7 @@ # Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import io import sys from datetime import datetime from time import mktime @@ -33,7 +34,7 @@ def urlopen(url, if_modified_since=None): request.add_header('User-Agent', 'Gentoo Portage') if if_modified_since: request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since)) - opener = urllib_request.build_opener() + opener = urllib_request.build_opener(CompressedResponseProcessor) hdl = opener.open(request) if hdl.headers.get('last-modified', ''): try: @@ -77,3 +78,32 @@ def _http_to_timestamp(http_datetime_string): tuple = parsedate(http_datetime_string) timestamp = mktime(tuple) return str(long(timestamp)) + +class CompressedResponseProcessor(urllib_request.BaseHandler): + # Handler for compressed responses. + + def http_request(self, req): + req.add_header('Accept-Encoding', 'bzip2,gzip,deflate') + return req + https_request = http_request + + def http_response(self, req, response): + decompressed = None + if response.headers.get('content-encoding') == 'bzip2': + import bz2 + decompressed = io.BytesIO(bz2.decompress(response.read())) + elif response.headers.get('content-encoding') == 'gzip': + from gzip import GzipFile + decompressed = GzipFile(fileobj=io.BytesIO(response.read()), mode='r') + elif response.headers.get('content-encoding') == 'deflate': + import zlib + try: + decompressed = io.BytesIO(zlib.decompress(response.read())) + except zlib.error: # they ignored RFC1950 + decompressed = io.BytesIO(zlib.decompress(response.read(), -zlib.MAX_WBITS)) + if decompressed: + old_response = response + response = urllib_request.addinfourl(decompressed, old_response.headers, old_response.url, old_response.code) + response.msg = old_response.msg + return response + https_response = http_response