From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 490D7138A1A for ; Fri, 16 Jan 2015 11:25:34 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 454AEE084A; Fri, 16 Jan 2015 11:25:33 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id A3547E0831 for ; Fri, 16 Jan 2015 11:25:32 +0000 (UTC) Received: from big_daddy.dol-sen.ca (S010634bdfa9ecf80.vc.shawcable.net [96.49.31.57]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) (Authenticated sender: dolsen) by smtp.gentoo.org (Postfix) with ESMTPSA id A69D8340659 for ; Fri, 16 Jan 2015 11:25:31 +0000 (UTC) Date: Fri, 16 Jan 2015 03:25:12 -0800 From: Brian Dolbec To: gentoo-portage-dev@lists.gentoo.org Subject: Re: [gentoo-portage-dev] [PATCH] BinpkgExtractorAsync: xz and gzip decompression (142579) Message-ID: <20150116032512.53021d8f.dolsen@gentoo.org> In-Reply-To: <1421397031-9158-1-git-send-email-zmedico@gentoo.org> References: <20150115211837.042a950a.dolsen@gentoo.org> <1421397031-9158-1-git-send-email-zmedico@gentoo.org> Organization: Gentoo Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org MIME-Version: 1.0 Content-Type: text/plain; charset=US-ASCII Content-Transfer-Encoding: 7bit X-Archives-Salt: 15b91dda-a226-435a-ab5c-a707207d6f12 X-Archives-Hash: 23b8a5f1c4a2623e55eeb7ab265f3202 On Fri, 16 Jan 2015 00:30:31 -0800 Zac Medico wrote: > This adds support for using a binary package's compression header to > determine the compression type, providing forward-compatibility for > xz and gzip decompression. The file name extension is disregared, so > that it will be possible to use a compression-independent file naming > scheme in the future (see bug #150031 for discussion about proposed > file naming schemes). > > Currently, only decompression is supported. It's useful to provide > forward-compatibility now, so that binhost clients will be prepared > to handle future binhost servers that use xz or gzip compression. > > X-Gentoo-Bug: 142579 > X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=142579 > --- oooh, now that is sooooooo much better :D Merge approved! (P.S. too pretty to erase the code below ;) > pym/_emerge/BinpkgExtractorAsync.py | 25 ++++++++++--- > pym/portage/util/compression_probe.py | 68 > +++++++++++++++++++++++++++++++++++ 2 files changed, 89 > insertions(+), 4 deletions(-) create mode 100644 > pym/portage/util/compression_probe.py > > diff --git a/pym/_emerge/BinpkgExtractorAsync.py > b/pym/_emerge/BinpkgExtractorAsync.py index be74c2f..6aaa448 100644 > --- a/pym/_emerge/BinpkgExtractorAsync.py > +++ b/pym/_emerge/BinpkgExtractorAsync.py > @@ -1,8 +1,13 @@ > # Copyright 1999-2013 Gentoo Foundation > # Distributed under the terms of the GNU General Public License v2 > > +import logging > + > from _emerge.SpawnProcess import SpawnProcess > import portage > +from portage.localization import _ > +from portage.util.compression_probe import (compression_probe, > + _decompressors) > import signal > import subprocess > > @@ -20,19 +25,31 @@ class BinpkgExtractorAsync(SpawnProcess): > if b"--xattrs" in output: > tar_options = "--xattrs" > > - # Add -q to bzip2 opts, in order to avoid "trailing > garbage after > - # EOF ignored" warning messages due to xpak trailer. > + decomp_cmd = _decompressors.get( > + compression_probe(self.pkg_path)) > + if decomp_cmd is None: > + self.scheduler.output("!!! %s\n" % > + _("File compression header > unrecognized: %s") % > + self.pkg_path, log_path=self.logfile, > + background=self.background, > level=logging.ERROR) > + self.returncode = 1 > + self._async_wait() > + return > + > + # Add -q to decomp_cmd opts, in order to avoid > "trailing garbage > + # after EOF ignored" warning messages due to xpak > trailer. # SIGPIPE handling (128 + SIGPIPE) should be compatible with > # assert_sigpipe_ok() that's used by the ebuild > unpack() helper. self.args = [self._shell_binary, "-c", > - > ("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | > tar -xp %s -C %s -f - ; " + \ > + ("%s -cq -- %s | tar -xp %s -C %s -f - ; " + > \ "p=(${PIPESTATUS[@]}) ; " + \ > "if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; > then " % (128 + signal.SIGPIPE) + \ "echo bzip2 failed with status > ${p[0]} ; exit ${p[0]} ; fi ; " + \ "if [ ${p[1]} != 0 ] ; then " + \ > "echo tar failed with status ${p[1]} ; exit > ${p[1]} ; fi ; " + \ "exit 0 ;") % \ > - (portage._shell_quote(self.pkg_path), > + (decomp_cmd, > + portage._shell_quote(self.pkg_path), > tar_options, > portage._shell_quote(self.image_dir))] > > diff --git a/pym/portage/util/compression_probe.py > b/pym/portage/util/compression_probe.py new file mode 100644 > index 0000000..1dc3547 > --- /dev/null > +++ b/pym/portage/util/compression_probe.py > @@ -0,0 +1,68 @@ > +# Copyright 2015 Gentoo Foundation > +# Distributed under the terms of the GNU General Public License v2 > + > +import errno > +import re > +import sys > + > +if sys.hexversion >= 0x3000000: > + basestring = str > + > +from portage import _encodings, _unicode_encode > +from portage.exception import FileNotFound, PermissionDenied > + > +_decompressors = { > + "bzip2": > "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}", > + "gzip": "gzip -d", > + "xz": "xz -d", > +} > + > +_compression_re = re.compile(b'^(' + > + b'(?P\x42\x5a\x68\x39)|' + > + b'(?P\x1f\x8b)|' + > + b'(?P\xfd\x37\x7a\x58\x5a\x00))') > + > +def compression_probe(f): > + """ > + Identify the compression type of a file. Returns one of the > + following identifier strings: > + > + bzip2 > + gzip > + xz > + > + @param f: a file path, or file-like object > + @type f: str or file > + @return: a string identifying the compression type, or None > if the > + compression type is unrecognized > + @rtype str or None > + """ > + > + open_file = isinstance(f, basestring) > + if open_file: > + try: > + f = open(_unicode_encode(f, > + encoding=_encodings['fs'], > errors='strict'), mode='rb') > + except IOError as e: > + if e.errno == PermissionDenied.errno: > + raise PermissionDenied(f) > + elif e.errno in (errno.ENOENT, errno.ESTALE): > + raise FileNotFound(f) > + else: > + raise > + > + try: > + return _compression_probe_file(f) > + finally: > + if open_file: > + f.close() > + > +def _compression_probe_file(f): > + > + m = _compression_re.match(f.read(6)) > + if m is not None: > + for k, v in m.groupdict().items(): > + if v is not None: > + return k > + > + return None -- Brian Dolbec