[gentoo-commits] proj/portage:master commit in: pym/_emerge/, pym/portage/util/

[gentoo-commits] proj/portage:master commit in: pym/_emerge/, pym/portage/util/

[Zac Medico]

commit:     6b3d262e6316073a2a3be81086c05891d970ae2a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Jan  6 02:36:11 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jan 16 17:07:11 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=6b3d262e

BinpkgExtractorAsync: xz and gzip decompression (142579)

This adds support for using a binary package's compression header to
determine the compression type, providing forward-compatibility for
xz and gzip decompression. The file name extension is disregared, so
that it will be possible to use a compression-independent file naming
scheme in the future (see bug #150031 for discussion about proposed
file naming schemes).

Currently, only decompression is supported. It's useful to provide
forward-compatibility now, so that binhost clients will be prepared
to handle future binhost servers that use xz or gzip compression.

X-Gentoo-Bug: 142579
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=142579
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

---
 pym/_emerge/BinpkgExtractorAsync.py   | 25 ++++++++++---
 pym/portage/util/compression_probe.py | 68 +++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 4 deletions(-)

diff --git a/pym/_emerge/BinpkgExtractorAsync.py b/pym/_emerge/BinpkgExtractorAsync.py
index be74c2f..6aaa448 100644
--- a/pym/_emerge/BinpkgExtractorAsync.py
+++ b/pym/_emerge/BinpkgExtractorAsync.py
@@ -1,8 +1,13 @@
 # Copyright 1999-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import logging
+
 from _emerge.SpawnProcess import SpawnProcess
 import portage
+from portage.localization import _
+from portage.util.compression_probe import (compression_probe,
+	_decompressors)
 import signal
 import subprocess
 
@@ -20,19 +25,31 @@ class BinpkgExtractorAsync(SpawnProcess):
 			if b"--xattrs" in output:
 				tar_options = "--xattrs"
 
-		# Add -q to bzip2 opts, in order to avoid "trailing garbage after
-		# EOF ignored" warning messages due to xpak trailer.
+		decomp_cmd = _decompressors.get(
+			compression_probe(self.pkg_path))
+		if decomp_cmd is None:
+			self.scheduler.output("!!! %s\n" %
+				_("File compression header unrecognized: %s") %
+				self.pkg_path, log_path=self.logfile,
+				background=self.background, level=logging.ERROR)
+			self.returncode = 1
+			self._async_wait()
+			return
+
+		# Add -q to decomp_cmd opts, in order to avoid "trailing garbage
+		# after EOF ignored" warning messages due to xpak trailer.
 		# SIGPIPE handling (128 + SIGPIPE) should be compatible with
 		# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
 		self.args = [self._shell_binary, "-c",
-			("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp %s -C %s -f - ; " + \
+			("%s -cq -- %s | tar -xp %s -C %s -f - ; " + \
 			"p=(${PIPESTATUS[@]}) ; " + \
 			"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
 			"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
 			"if [ ${p[1]} != 0 ] ; then " + \
 			"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
 			"exit 0 ;") % \
-			(portage._shell_quote(self.pkg_path),
+			(decomp_cmd,
+			portage._shell_quote(self.pkg_path),
 			tar_options,
 			portage._shell_quote(self.image_dir))]
 

diff --git a/pym/portage/util/compression_probe.py b/pym/portage/util/compression_probe.py
new file mode 100644
index 0000000..1dc3547
--- /dev/null
+++ b/pym/portage/util/compression_probe.py
@@ -0,0 +1,68 @@
+# Copyright 2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+from portage import _encodings, _unicode_encode
+from portage.exception import FileNotFound, PermissionDenied
+
+_decompressors = {
+	"bzip2": "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}",
+	"gzip": "gzip -d",
+	"xz": "xz -d",
+}
+
+_compression_re = re.compile(b'^(' +
+	b'(?P<bzip2>\x42\x5a\x68\x39)|' +
+	b'(?P<gzip>\x1f\x8b)|' +
+	b'(?P<xz>\xfd\x37\x7a\x58\x5a\x00))')
+
+def compression_probe(f):
+	"""
+	Identify the compression type of a file. Returns one of the
+	following identifier strings:
+
+		bzip2
+		gzip
+		xz
+
+	@param f: a file path, or file-like object
+	@type f: str or file
+	@return: a string identifying the compression type, or None if the
+		compression type is unrecognized
+	@rtype str or None
+	"""
+
+	open_file = isinstance(f, basestring)
+	if open_file:
+		try:
+			f = open(_unicode_encode(f,
+				encoding=_encodings['fs'], errors='strict'), mode='rb')
+		except IOError as e:
+			if e.errno == PermissionDenied.errno:
+				raise PermissionDenied(f)
+			elif e.errno in (errno.ENOENT, errno.ESTALE):
+				raise FileNotFound(f)
+			else:
+				raise
+
+	try:
+		return _compression_probe_file(f)
+	finally:
+		if open_file:
+			f.close()
+
+def _compression_probe_file(f):
+
+	m = _compression_re.match(f.read(6))
+	if m is not None:
+		for k, v in m.groupdict().items():
+			if v is not None:
+				return k
+
+	return None

[gentoo-commits] proj/portage:master commit in: pym/_emerge/, pym/portage/util/

[Michał Górny]

commit:     daeb752c842e892a6055d686173ec9ab0f9961de
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Wed Nov 11 21:09:26 2015 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sun Nov 15 21:18:45 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=daeb752c

Warn if LC_CTYPE does not transform ASCII chars like POSIX

Output a warning if LC_CTYPE is set to a value that causes libc
toupper() and/or tolower() conversions not apply correctly to printable
ASCII characters.

 pym/_emerge/actions.py     |  3 ++
 pym/portage/util/locale.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)

diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 7f1cb59..e03e8d4 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -27,6 +27,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.debug',
 	'portage.news:count_unread_news,display_news_notifications',
 	'portage.util._get_vm_info:get_vm_info',
+	'portage.util.locale:check_locale',
 	'portage.emaint.modules.sync.sync:SyncRepos',
 	'_emerge.chk_updated_cfg_files:chk_updated_cfg_files',
 	'_emerge.help:help@emerge_help',
@@ -2467,6 +2468,8 @@ def validate_ebuild_environment(trees):
 		for line in textwrap.wrap(msg, 65):
 			out.ewarn(line)
 
+	check_locale()
+
 def check_procfs():
 	procfs_path = '/proc'
 	if platform.system() not in ("Linux",) or \

diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
new file mode 100644
index 0000000..12161eb
--- /dev/null
+++ b/pym/portage/util/locale.py
@@ -0,0 +1,90 @@
+#-*- coding:utf-8 -*-
+# Copyright 2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+"""
+Function to check whether the current used LC_CTYPE handles case
+transformations of ASCII characters in a way compatible with the POSIX
+locale.
+"""
+from __future__ import unicode_literals
+
+import logging
+import os
+import textwrap
+import traceback
+
+from portage.util import writemsg_level
+from portage.util._ctypes import find_library, LoadLibrary
+
+
+def _check_locale():
+	"""
+	The inner locale check function.
+	"""
+
+	libc_fn = find_library("c")
+	if libc_fn is None:
+		return None
+	libc = LoadLibrary(libc_fn)
+	if libc is None:
+		return None
+
+	lc = list(range(ord('a'), ord('z')+1))
+	uc = list(range(ord('A'), ord('Z')+1))
+	rlc = [libc.tolower(c) for c in uc]
+	ruc = [libc.toupper(c) for c in lc]
+
+	if lc != rlc or uc != ruc:
+		msg = ("WARNING: The LC_CTYPE variable is set to a locale " +
+			"that specifies transformation between lowercase " +
+			"and uppercase ASCII characters that is different than " +
+			"the one specified by POSIX locale. This can break " +
+			"ebuilds and cause issues in programs that rely on " +
+			"the common character conversion scheme. " +
+			"Please consider enabling another locale (such as " +
+			"en_US.UTF-8) in /etc/locale.gen and setting it " +
+			"as LC_CTYPE in make.conf.")
+		msg = [l for l in textwrap.wrap(msg, 70)]
+		msg.append("")
+		chars = lambda l: ''.join(chr(x) for x in l)
+		if uc != ruc:
+			msg.extend([
+				"  %s -> %s" % (chars(lc), chars(ruc)),
+				"  %28s: %s" % ('expected', chars(uc))])
+		if lc != rlc:
+			msg.extend([
+				"  %s -> %s" % (chars(uc), chars(rlc)),
+				"  %28s: %s" % ('expected', chars(lc))])
+		writemsg_level("".join(["!!! %s\n" % l for l in msg]),
+			level=logging.ERROR, noiselevel=-1)
+		return False
+
+	return True
+
+
+def check_locale():
+	"""
+	Check whether the locale is sane. Returns True if it is, prints
+	warning and returns False if it is not. Returns None if the check
+	can not be executed due to platform limitations.
+	"""
+
+	pid = os.fork()
+	if pid == 0:
+		try:
+			ret = _check_locale()
+			if ret is None:
+				os._exit(2)
+			else:
+				os._exit(0 if ret else 1)
+		except Exception:
+			traceback.print_exc()
+			os._exit(2)
+
+	pid2, ret = os.waitpid(pid, 0)
+	assert pid == pid2
+	if os.WIFEXITED(ret):
+		ret = os.WEXITSTATUS(ret)
+		if ret != 2:
+			return ret == 0
+	return None