public inbox for gentoo-portage-dev@lists.gentoo.org
 help / color / mirror / Atom feed
* [gentoo-portage-dev] Resending patchset for C module to help check locale
@ 2016-09-14 21:01 Anthony G. Basile
  2016-09-15  3:22 ` Brian Dolbec
  0 siblings, 1 reply; 2+ messages in thread
From: Anthony G. Basile @ 2016-09-14 21:01 UTC (permalink / raw
  To: gentoo-portage-dev

[-- Attachment #1: Type: text/plain, Size: 211 bytes --]

As per Brian's request on IRC, here's the patchset for adding a C module
to help check locales.

-- 
Anthony G. Basile, Ph. D.
Chair of Information Technology
D'Youville College
Buffalo, NY 14201
(716) 829-8197

[-- Attachment #2: 0001-pym-portage-util-locale.py-fix-decoding-for-python2-.patch --]
[-- Type: text/plain, Size: 1374 bytes --]

From 76131d44e1731c876840e7925ccff2468ea157a7 Mon Sep 17 00:00:00 2001
From: "Anthony G. Basile" <blueness@gentoo.org>
Date: Wed, 25 May 2016 08:48:32 -0400
Subject: [PATCH 1/3] pym/portage/util/locale.py: fix decoding for python2 plus
 some locales

When using python2 with some locales, like turkish, chr() is passed values not in
range(128) which cannot be decoded as ASCII, thus throwing a UnicodeDecodeError
exception.  We use _unicode_decode() from portage.util to address this.

Signed-off-by: Anthony G. Basile <blueness@gentoo.org>
---
 pym/portage/util/locale.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
index 2a15ea1..093eb86 100644
--- a/pym/portage/util/locale.py
+++ b/pym/portage/util/locale.py
@@ -15,7 +15,7 @@ import textwrap
 import traceback
 
 import portage
-from portage.util import writemsg_level
+from portage.util import _unicode_decode, writemsg_level
 from portage.util._ctypes import find_library, LoadLibrary
 
 
@@ -62,7 +62,7 @@ def _check_locale(silent):
 			"as LC_CTYPE in make.conf.")
 		msg = [l for l in textwrap.wrap(msg, 70)]
 		msg.append("")
-		chars = lambda l: ''.join(chr(x) for x in l)
+		chars = lambda l: ''.join(_unicode_decode(chr(x)) for x in l)
 		if uc != ruc:
 			msg.extend([
 				"  %s -> %s" % (chars(lc), chars(ruc)),
-- 
2.7.3


[-- Attachment #3: 0002-setup.py-add-stub-for-building-custom-modules-in-C-C.patch --]
[-- Type: text/plain, Size: 1867 bytes --]

From 831af672827380a928e8e07c3fb5ecd01545e3c9 Mon Sep 17 00:00:00 2001
From: "Anthony G. Basile" <blueness@gentoo.org>
Date: Thu, 19 May 2016 06:52:43 -0400
Subject: [PATCH 2/3] setup.py: add stub for building custom modules in C/C++

Currently portage doesn't include any custom modules written in C/C++.
This commit introduces stub code for building such modules in setup.py.

Signed-off-by: Anthony G. Basile <blueness@gentoo.org>
---
 setup.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2220d23..8d20355 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 from __future__ import print_function
 
-from distutils.core import setup, Command
+from distutils.core import setup, Command, Extension
 from distutils.command.build import build
 from distutils.command.build_scripts import build_scripts
 from distutils.command.clean import clean
@@ -30,6 +30,9 @@ import sys
 # TODO:
 # - smarter rebuilds of docs w/ 'install_docbook' and 'install_epydoc'.
 
+# Dictionary of scripts.  The structure is
+#   key   = location in filesystem to install the scripts
+#   value = list of scripts, path relative to top source directory
 x_scripts = {
 	'bin': [
 		'bin/ebuild', 'bin/egencache', 'bin/emerge', 'bin/emerge-webrsync',
@@ -41,6 +44,10 @@ x_scripts = {
 	],
 }
 
+# Dictionary custom modules written in C/C++ here.  The structure is
+#   key   = module name
+#   value = list of C/C++ source code, path relative to top source directory
+x_c_helpers = {}
 
 class x_build(build):
 	""" Build command with extra build_man call. """
@@ -636,6 +643,8 @@ setup(
 		['$sysconfdir/portage/repo.postsync.d', ['cnf/repo.postsync.d/example']],
 	],
 
+	ext_modules = [Extension(name=n, sources=m) for n, m in x_c_helpers.items()],
+
 	cmdclass = {
 		'build': x_build,
 		'build_man': build_man,
-- 
2.7.3


[-- Attachment #4: 0003-pym-portage-util-locale.py-add-a-C-module-to-help-ch.patch --]
[-- Type: text/plain, Size: 4075 bytes --]

From ee31675efc06d1df6806cfcdfc4102271e596c96 Mon Sep 17 00:00:00 2001
From: "Anthony G. Basile" <blueness@gentoo.org>
Date: Fri, 27 May 2016 09:47:34 -0400
Subject: [PATCH 3/3] pym/portage/util/locale.py: add a C module to help check
 locale

The current method to check for a sane system locale is to use python's
ctypes.util.find_library() to construct a full library path to the
system libc.so and pass that path to ctypes.CDLL() so we can call
toupper() and tolower() directly.  However, this gets bogged down in
implementation details and fails with musl.

We work around this design flaw in ctypes with a small python module
written in C which provides thin wrappers to toupper() and tolower(),
and only fall back on the current ctypes-based check when this module
is not available.

This has been tested on glibc, uClibc and musl systems.

X-Gentoo-bug: 571444
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444

Signed-off-by: Anthony G. Basile <blueness@gentoo.org>
---
 pym/portage/util/locale.py | 16 ++++++-----
 setup.py                   |  6 +++-
 src/portage_util_libc.c    | 68 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 82 insertions(+), 8 deletions(-)
 create mode 100644 src/portage_util_libc.c

diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
index 093eb86..5b09945 100644
--- a/pym/portage/util/locale.py
+++ b/pym/portage/util/locale.py
@@ -34,13 +34,15 @@ def _check_locale(silent):
 	"""
 	The inner locale check function.
 	"""
-
-	libc_fn = find_library("c")
-	if libc_fn is None:
-		return None
-	libc = LoadLibrary(libc_fn)
-	if libc is None:
-		return None
+	try:
+		from portage.util import libc
+	except ImportError:
+		libc_fn = find_library("c")
+		if libc_fn is None:
+			return None
+		libc = LoadLibrary(libc_fn)
+		if libc is None:
+			return None
 
 	lc = list(range(ord('a'), ord('z')+1))
 	uc = list(range(ord('A'), ord('Z')+1))
diff --git a/setup.py b/setup.py
index 8d20355..d37ab29 100755
--- a/setup.py
+++ b/setup.py
@@ -47,7 +47,11 @@ x_scripts = {
 # Dictionary custom modules written in C/C++ here.  The structure is
 #   key   = module name
 #   value = list of C/C++ source code, path relative to top source directory
-x_c_helpers = {}
+x_c_helpers = {
+	'portage.util.libc' : [
+		'src/portage_util_libc.c',
+	],
+}
 
 class x_build(build):
 	""" Build command with extra build_man call. """
diff --git a/src/portage_util_libc.c b/src/portage_util_libc.c
new file mode 100644
index 0000000..977b954
--- /dev/null
+++ b/src/portage_util_libc.c
@@ -0,0 +1,68 @@
+/* Copyright 2005-2016 Gentoo Foundation
+ * Distributed under the terms of the GNU General Public License v2
+ */
+
+#include <Python.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+static PyObject * _libc_tolower(PyObject *, PyObject *);
+static PyObject * _libc_toupper(PyObject *, PyObject *);
+
+static PyMethodDef LibcMethods[] = {
+	{"tolower", _libc_tolower, METH_VARARGS, "Convert to lower case using system locale."},
+	{"toupper", _libc_toupper, METH_VARARGS, "Convert to upper case using system locale."},
+	{NULL, NULL, 0, NULL}
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+	PyModuleDef_HEAD_INIT,
+	"libc",								/* m_name */
+	"Module for converting case using the system locale",		/* m_doc */
+	-1,								/* m_size */
+	LibcMethods,							/* m_methods */
+	NULL,								/* m_reload */
+	NULL,								/* m_traverse */
+	NULL,								/* m_clear */
+	NULL,								/* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit_libc(void)
+{
+	PyObject *m;
+	m = PyModule_Create(&moduledef);
+	return m;
+}
+#else
+PyMODINIT_FUNC
+initlibc(void)
+{
+	Py_InitModule("libc", LibcMethods);
+}
+#endif
+
+
+static PyObject *
+_libc_tolower(PyObject *self, PyObject *args)
+{
+	int c;
+
+	if (!PyArg_ParseTuple(args, "i", &c))
+		return NULL;
+
+	return Py_BuildValue("i", tolower(c));
+}
+
+
+static PyObject *
+_libc_toupper(PyObject *self, PyObject *args)
+{
+	int c;
+
+	if (!PyArg_ParseTuple(args, "i", &c))
+		return NULL;
+
+	return Py_BuildValue("i", toupper(c));
+}
-- 
2.7.3


^ permalink raw reply related	[flat|nested] 2+ messages in thread

* Re: [gentoo-portage-dev] Resending patchset for C module to help check locale
  2016-09-14 21:01 [gentoo-portage-dev] Resending patchset for C module to help check locale Anthony G. Basile
@ 2016-09-15  3:22 ` Brian Dolbec
  0 siblings, 0 replies; 2+ messages in thread
From: Brian Dolbec @ 2016-09-15  3:22 UTC (permalink / raw
  To: gentoo-portage-dev

On Wed, 14 Sep 2016 17:01:27 -0400
"Anthony G. Basile" <basile@opensource.dyc.edu> wrote:

> As per Brian's request on IRC, here's the patchset for adding a C
> module to help check locales.
> 

Merged

-- 
Brian Dolbec <dolsen>



^ permalink raw reply	[flat|nested] 2+ messages in thread

end of thread, other threads:[~2016-09-15  3:22 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2016-09-14 21:01 [gentoo-portage-dev] Resending patchset for C module to help check locale Anthony G. Basile
2016-09-15  3:22 ` Brian Dolbec

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox