[gentoo-portage-dev] [PATCH 1/2] setup.py: add stub for building custom modules in C/C++

[gentoo-portage-dev] [PATCH 1/2] setup.py: add stub for building custom modules in C/C++

[Anthony G. Basile]

From: "Anthony G. Basile" <blueness@gentoo.org>

Currently portage doesn't include any custom modules written in C/C++.
This commit introduces stub code for building such modules in setup.py.

Signed-off-by: Anthony G. Basile <blueness@gentoo.org>
---
 setup.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 75c4bcb..25429bc 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 from __future__ import print_function
 
-from distutils.core import setup, Command
+from distutils.core import setup, Command, Extension
 from distutils.command.build import build
 from distutils.command.build_scripts import build_scripts
 from distutils.command.clean import clean
@@ -30,6 +30,9 @@ import sys
 # TODO:
 # - smarter rebuilds of docs w/ 'install_docbook' and 'install_epydoc'.
 
+# Dictionary of scripts.  The structure is
+#   key   = location in filesystem to install the scripts
+#   value = list of scripts, path relative to top source directory
 x_scripts = {
 	'bin': [
 		'bin/ebuild', 'bin/egencache', 'bin/emerge', 'bin/emerge-webrsync',
@@ -41,6 +44,10 @@ x_scripts = {
 	],
 }
 
+# Dictionary custom modules written in C/C++ here.  The structure is
+#   key   = module name
+#   value = list of C/C++ source code, path relative to top source directory
+x_c_helpers = {}
 
 class x_build(build):
 	""" Build command with extra build_man call. """
@@ -636,6 +643,8 @@ setup(
 		['$sysconfdir/portage/repo.postsync.d', ['cnf/repo.postsync.d/example']],
 	],
 
+	ext_modules = [Extension(name=n, sources=m) for n, m in x_c_helpers.items()],
+
 	cmdclass = {
 		'build': x_build,
 		'build_man': build_man,
-- 
2.7.3

[gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to check locale

[Anthony G. Basile]

From: "Anthony G. Basile" <blueness@gentoo.org>

The current method to check for the system locale is to use python's
ctypes.util.find_library() to construct a full library path to the
system libc.so which is then passed to ctypes.CDLL().  However,
this gets bogged down in implementation dependant details and
fails with musl.

We work around this design flaw in ctypes with a small python module
written in C called 'portage_c_check_locale', and only fall back on
the current ctypes-based check when this module is not available.

This has been tested on glibc, uClibc and musl systems.

X-Gentoo-bug: 571444
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
Signed-off-by: Anthony G. Basile <blueness@gentoo.org>
---
 pym/portage/util/locale.py |  40 +++++++++----
 setup.py                   |   6 +-
 src/check_locale.c         | 144 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 178 insertions(+), 12 deletions(-)
 create mode 100644 src/check_locale.c

diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
index 2a15ea1..fc5d052 100644
--- a/pym/portage/util/locale.py
+++ b/pym/portage/util/locale.py
@@ -30,17 +30,17 @@ locale_categories = (
 _check_locale_cache = {}
 
 
-def _check_locale(silent):
+def _ctypes_check_locale():
 	"""
-	The inner locale check function.
+	Check for locale using ctypes.
 	"""
 
 	libc_fn = find_library("c")
 	if libc_fn is None:
-		return None
+		return (None, "")
 	libc = LoadLibrary(libc_fn)
 	if libc is None:
-		return None
+		return (None, "")
 
 	lc = list(range(ord('a'), ord('z')+1))
 	uc = list(range(ord('A'), ord('Z')+1))
@@ -48,9 +48,6 @@ def _check_locale(silent):
 	ruc = [libc.toupper(c) for c in lc]
 
 	if lc != rlc or uc != ruc:
-		if silent:
-			return False
-
 		msg = ("WARNING: The LC_CTYPE variable is set to a locale " +
 			"that specifies transformation between lowercase " +
 			"and uppercase ASCII characters that is different than " +
@@ -71,11 +68,32 @@ def _check_locale(silent):
 			msg.extend([
 				"  %s -> %s" % (chars(uc), chars(rlc)),
 				"  %28s: %s" % ('expected', chars(lc))])
-		writemsg_level("".join(["!!! %s\n" % l for l in msg]),
-			level=logging.ERROR, noiselevel=-1)
-		return False
+		msg = "".join(["!!! %s\n" % l for l in msg]),
+		return (False, msg)
+
+	return (True, "")
+
+
+def _check_locale(silent):
+	"""
+	The inner locale check function.
+	"""
+
+	try:
+		from portage_c_check_locale import _c_check_locale
+		(ret, msg) = _c_check_locale()
+	except ImportError:
+		writemsg_level("!!! Unable to import portage_c_check_locale\n",
+			level=logging.WARNING, noiselevel=-1)
+		(ret, msg) = _ctypes_check_locale()
+
+	if ret:
+		return True
+
+	if not silent:
+		writemsg_level(msg, level=logging.ERROR, noiselevel=-1)
 
-	return True
+	return False
 
 
 def check_locale(silent=False, env=None):
diff --git a/setup.py b/setup.py
index 25429bc..e44ac41 100755
--- a/setup.py
+++ b/setup.py
@@ -47,7 +47,11 @@ x_scripts = {
 # Dictionary custom modules written in C/C++ here.  The structure is
 #   key   = module name
 #   value = list of C/C++ source code, path relative to top source directory
-x_c_helpers = {}
+x_c_helpers = {
+	'portage_c_check_locale' : [
+		'src/check_locale.c',
+	],
+}
 
 class x_build(build):
 	""" Build command with extra build_man call. """
diff --git a/src/check_locale.c b/src/check_locale.c
new file mode 100644
index 0000000..9762ef2
--- /dev/null
+++ b/src/check_locale.c
@@ -0,0 +1,144 @@
+/* Copyright 2005-2015 Gentoo Foundation
+ * Distributed under the terms of the GNU General Public License v2
+ */
+
+#include <Python.h>
+#include <ctype.h>
+#include <string.h>
+
+static PyObject * portage_c_check_locale(PyObject *, PyObject *);
+
+static PyMethodDef CheckLocaleMethods[] = {
+	{"_c_check_locale", portage_c_check_locale, METH_NOARGS, "Check the system locale."},
+	{NULL, NULL, 0, NULL}
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+	PyModuleDef_HEAD_INIT,
+	"portage_c_check_locale",					/* m_name */
+	"Module for checking the system locale for portage",		/* m_doc */
+	-1,								/* m_size */
+	CheckLocaleMethods,						/* m_methods */
+	NULL,								/* m_reload */
+	NULL,								/* m_traverse */
+	NULL,								/* m_clear */
+	NULL,								/* m_free */
+};
+#endif
+
+static PyObject *LocaleError;
+
+PyMODINIT_FUNC
+#if PY_MAJOR_VERSION >= 3
+PyInit_portage_c_check_locale(void)
+#else
+initportage_c_check_locale(void)
+#endif
+{
+	PyObject *m;
+
+#if PY_MAJOR_VERSION >= 3
+	m = PyModule_Create(&moduledef);
+#else
+	m = Py_InitModule("portage_c_check_locale", CheckLocaleMethods);
+#endif
+
+	if (m == NULL)
+#if PY_MAJOR_VERSION >= 3
+		return NULL;
+#else
+		return;
+#endif
+
+	LocaleError = PyErr_NewException("locale.LocaleError", NULL, NULL);
+	Py_INCREF(LocaleError);
+	PyModule_AddObject(m, "LocaleError", LocaleError);
+
+#if PY_MAJOR_VERSION >= 3
+	return m;
+#else
+	return;
+#endif
+}
+
+
+static void
+error_msg(char msg[], int c[], int rc[], int ac[])
+{
+	int i, p;
+
+	strcat(msg, "  ");
+	p = strlen(msg);
+	for (i = 'a'; i <= 'z'; i++)
+		msg[p++] = c[i-'a'];
+
+	strcat(msg, " -> ");
+	p = strlen(msg);
+	for (i = 'a'; i <= 'z'; i++)
+		msg[p++] = rc[i-'a'];
+	strcat(msg, "\n");
+
+	strcat(msg, "                      expected: ");
+	p = strlen(msg);
+	for (i = 'a'; i <= 'z'; i++)
+		msg[p++] = ac[i-'a'];
+	strcat(msg, "\n\n");
+}
+
+
+static PyObject *
+portage_c_check_locale(PyObject *self, PyObject *args)
+{
+	int i, upper = 1, lower = 1;
+	int lc[26], uc[26], rlc[26], ruc[26];
+	char msg[1024];
+
+	memset(msg, 0, 1024);
+
+	for (i = 'a'; i <= 'z'; i++) {
+		lc[i-'a'] = i;
+		ruc[i-'a'] = toupper(i);
+	}
+
+	for (i = 'A'; i <= 'Z'; i++) {
+		uc[i-'A'] = i;
+		rlc[i-'A'] = tolower(i);
+	}
+
+	for (i = 'a'; i <= 'z'; i++)
+		if(lc[i-'a'] != rlc[i-'a']) {
+			lower = 0;
+			break;
+		}
+
+	for (i = 'A'; i <= 'Z'; i++)
+		if(uc[i-'A'] != ruc[i-'A']) {
+			upper = 0;
+			break;
+		}
+
+	if (lower == 0 || upper == 0) {
+		strcpy(msg,
+			"!!! WARNING: The LC_CTYPE variable is set to a locale that specifies\n"
+			"!!! transformation between lowercase and uppercase ASCII characters that\n"
+			"!!! is different than the one specified by POSIX locale. This can break\n"
+			"!!! ebuilds and cause issues in programs that rely on the common character\n"
+			"!!! conversion scheme.  Please consider enabling another locale (such as\n"
+			"!!! en_US.UTF-8) in /etc/locale.gen and setting it as LC_CTYPE in\n"
+			"!!! make.conf.\n\n"
+		);
+
+		if (lower == 0)
+			error_msg(msg, lc, ruc, uc);
+
+		if (upper == 0)
+			error_msg(msg, uc, rlc, lc);
+	}
+
+#if PY_MAJOR_VERSION >= 3
+	return Py_BuildValue("iy", lower && upper, msg);
+#else
+	return Py_BuildValue("is", lower && upper, msg);
+#endif
+}
-- 
2.7.3

Re: [gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to check locale

[Anthony G. Basile]

On 5/19/16 8:43 AM, Anthony G. Basile wrote:
> From: "Anthony G. Basile" <blueness@gentoo.org>

> 
> This has been tested on glibc, uClibc and musl systems.

To be clear here, I needed the patch from bug #583412 to fix an
independent bug in portage + python2.7 + turkish (and possibly other)
locale.  Arfrever is going to commit that upstream, so it'll eventually
trickle down to us.


-- 
Anthony G. Basile, Ph. D.
Chair of Information Technology
D'Youville College
Buffalo, NY 14201
(716) 829-8197

Re: [gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to check locale

[Michał Górny]

Dnia 19 maja 2016 14:43:38 CEST, "Anthony G. Basile" <basile@opensource.dyc.edu> napisał(a):
>From: "Anthony G. Basile" <blueness@gentoo.org>
>
>The current method to check for the system locale is to use python's
>ctypes.util.find_library() to construct a full library path to the
>system libc.so which is then passed to ctypes.CDLL().  However,
>this gets bogged down in implementation dependant details and
>fails with musl.
>
>We work around this design flaw in ctypes with a small python module
>written in C called 'portage_c_check_locale', and only fall back on
>the current ctypes-based check when this module is not available.
>
>This has been tested on glibc, uClibc and musl systems.
>
>X-Gentoo-bug: 571444
>X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
>Signed-off-by: Anthony G. Basile <blueness@gentoo.org>

To be honest, I don't like this. Do we really want to duplicate all this, including duplicating the complete messages? This really looks messy and unmaintainable.

The only reason libc functions were being used was to workaround the hacky built-in case conversion functions. And this is the only part where CDLL was really used.

So please change this to provide trivial wrappers around the C library functions, and use them alternatively to ones obtained using CDLL. With a single common code doing all the check logic and messaging.


>---
> pym/portage/util/locale.py |  40 +++++++++----
> setup.py                   |   6 +-
>src/check_locale.c         | 144
>+++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 178 insertions(+), 12 deletions(-)
> create mode 100644 src/check_locale.c
>
>diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
>index 2a15ea1..fc5d052 100644
>--- a/pym/portage/util/locale.py
>+++ b/pym/portage/util/locale.py
>@@ -30,17 +30,17 @@ locale_categories = (
> _check_locale_cache = {}
> 
> 
>-def _check_locale(silent):
>+def _ctypes_check_locale():
> 	"""
>-	The inner locale check function.
>+	Check for locale using ctypes.
> 	"""
> 
> 	libc_fn = find_library("c")
> 	if libc_fn is None:
>-		return None
>+		return (None, "")
> 	libc = LoadLibrary(libc_fn)
> 	if libc is None:
>-		return None
>+		return (None, "")
> 
> 	lc = list(range(ord('a'), ord('z')+1))
> 	uc = list(range(ord('A'), ord('Z')+1))
>@@ -48,9 +48,6 @@ def _check_locale(silent):
> 	ruc = [libc.toupper(c) for c in lc]
> 
> 	if lc != rlc or uc != ruc:
>-		if silent:
>-			return False
>-
> 		msg = ("WARNING: The LC_CTYPE variable is set to a locale " +
> 			"that specifies transformation between lowercase " +
> 			"and uppercase ASCII characters that is different than " +
>@@ -71,11 +68,32 @@ def _check_locale(silent):
> 			msg.extend([
> 				"  %s -> %s" % (chars(uc), chars(rlc)),
> 				"  %28s: %s" % ('expected', chars(lc))])
>-		writemsg_level("".join(["!!! %s\n" % l for l in msg]),
>-			level=logging.ERROR, noiselevel=-1)
>-		return False
>+		msg = "".join(["!!! %s\n" % l for l in msg]),
>+		return (False, msg)
>+
>+	return (True, "")
>+
>+
>+def _check_locale(silent):
>+	"""
>+	The inner locale check function.
>+	"""
>+
>+	try:
>+		from portage_c_check_locale import _c_check_locale
>+		(ret, msg) = _c_check_locale()
>+	except ImportError:
>+		writemsg_level("!!! Unable to import portage_c_check_locale\n",
>+			level=logging.WARNING, noiselevel=-1)
>+		(ret, msg) = _ctypes_check_locale()
>+
>+	if ret:
>+		return True
>+
>+	if not silent:
>+		writemsg_level(msg, level=logging.ERROR, noiselevel=-1)
> 
>-	return True
>+	return False
> 
> 
> def check_locale(silent=False, env=None):
>diff --git a/setup.py b/setup.py
>index 25429bc..e44ac41 100755
>--- a/setup.py
>+++ b/setup.py
>@@ -47,7 +47,11 @@ x_scripts = {
> # Dictionary custom modules written in C/C++ here.  The structure is
> #   key   = module name
>#   value = list of C/C++ source code, path relative to top source
>directory
>-x_c_helpers = {}
>+x_c_helpers = {
>+	'portage_c_check_locale' : [
>+		'src/check_locale.c',
>+	],
>+}
> 
> class x_build(build):
> 	""" Build command with extra build_man call. """
>diff --git a/src/check_locale.c b/src/check_locale.c
>new file mode 100644
>index 0000000..9762ef2
>--- /dev/null
>+++ b/src/check_locale.c
>@@ -0,0 +1,144 @@
>+/* Copyright 2005-2015 Gentoo Foundation
>+ * Distributed under the terms of the GNU General Public License v2
>+ */
>+
>+#include <Python.h>
>+#include <ctype.h>
>+#include <string.h>
>+
>+static PyObject * portage_c_check_locale(PyObject *, PyObject *);
>+
>+static PyMethodDef CheckLocaleMethods[] = {
>+	{"_c_check_locale", portage_c_check_locale, METH_NOARGS, "Check the
>system locale."},
>+	{NULL, NULL, 0, NULL}
>+};
>+
>+#if PY_MAJOR_VERSION >= 3
>+static struct PyModuleDef moduledef = {
>+	PyModuleDef_HEAD_INIT,
>+	"portage_c_check_locale",					/* m_name */
>+	"Module for checking the system locale for portage",		/* m_doc */
>+	-1,								/* m_size */
>+	CheckLocaleMethods,						/* m_methods */
>+	NULL,								/* m_reload */
>+	NULL,								/* m_traverse */
>+	NULL,								/* m_clear */
>+	NULL,								/* m_free */
>+};
>+#endif
>+
>+static PyObject *LocaleError;
>+
>+PyMODINIT_FUNC
>+#if PY_MAJOR_VERSION >= 3
>+PyInit_portage_c_check_locale(void)
>+#else
>+initportage_c_check_locale(void)
>+#endif
>+{
>+	PyObject *m;
>+
>+#if PY_MAJOR_VERSION >= 3
>+	m = PyModule_Create(&moduledef);
>+#else
>+	m = Py_InitModule("portage_c_check_locale", CheckLocaleMethods);
>+#endif
>+
>+	if (m == NULL)
>+#if PY_MAJOR_VERSION >= 3
>+		return NULL;
>+#else
>+		return;
>+#endif
>+
>+	LocaleError = PyErr_NewException("locale.LocaleError", NULL, NULL);
>+	Py_INCREF(LocaleError);
>+	PyModule_AddObject(m, "LocaleError", LocaleError);
>+
>+#if PY_MAJOR_VERSION >= 3
>+	return m;
>+#else
>+	return;
>+#endif
>+}
>+
>+
>+static void
>+error_msg(char msg[], int c[], int rc[], int ac[])
>+{
>+	int i, p;
>+
>+	strcat(msg, "  ");
>+	p = strlen(msg);
>+	for (i = 'a'; i <= 'z'; i++)
>+		msg[p++] = c[i-'a'];
>+
>+	strcat(msg, " -> ");
>+	p = strlen(msg);
>+	for (i = 'a'; i <= 'z'; i++)
>+		msg[p++] = rc[i-'a'];
>+	strcat(msg, "\n");
>+
>+	strcat(msg, "                      expected: ");
>+	p = strlen(msg);
>+	for (i = 'a'; i <= 'z'; i++)
>+		msg[p++] = ac[i-'a'];
>+	strcat(msg, "\n\n");
>+}
>+
>+
>+static PyObject *
>+portage_c_check_locale(PyObject *self, PyObject *args)
>+{
>+	int i, upper = 1, lower = 1;
>+	int lc[26], uc[26], rlc[26], ruc[26];
>+	char msg[1024];
>+
>+	memset(msg, 0, 1024);
>+
>+	for (i = 'a'; i <= 'z'; i++) {
>+		lc[i-'a'] = i;
>+		ruc[i-'a'] = toupper(i);
>+	}
>+
>+	for (i = 'A'; i <= 'Z'; i++) {
>+		uc[i-'A'] = i;
>+		rlc[i-'A'] = tolower(i);
>+	}
>+
>+	for (i = 'a'; i <= 'z'; i++)
>+		if(lc[i-'a'] != rlc[i-'a']) {
>+			lower = 0;
>+			break;
>+		}
>+
>+	for (i = 'A'; i <= 'Z'; i++)
>+		if(uc[i-'A'] != ruc[i-'A']) {
>+			upper = 0;
>+			break;
>+		}
>+
>+	if (lower == 0 || upper == 0) {
>+		strcpy(msg,
>+			"!!! WARNING: The LC_CTYPE variable is set to a locale that
>specifies\n"
>+			"!!! transformation between lowercase and uppercase ASCII
>characters that\n"
>+			"!!! is different than the one specified by POSIX locale. This can
>break\n"
>+			"!!! ebuilds and cause issues in programs that rely on the common
>character\n"
>+			"!!! conversion scheme.  Please consider enabling another locale
>(such as\n"
>+			"!!! en_US.UTF-8) in /etc/locale.gen and setting it as LC_CTYPE
>in\n"
>+			"!!! make.conf.\n\n"
>+		);
>+
>+		if (lower == 0)
>+			error_msg(msg, lc, ruc, uc);
>+
>+		if (upper == 0)
>+			error_msg(msg, uc, rlc, lc);
>+	}
>+
>+#if PY_MAJOR_VERSION >= 3
>+	return Py_BuildValue("iy", lower && upper, msg);
>+#else
>+	return Py_BuildValue("is", lower && upper, msg);
>+#endif
>+}


-- 
Best regards,
Michał Górny (by phone)

Re: [gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to check locale

[Anthony G. Basile]

On 5/19/16 9:38 AM, Michał Górny wrote:
> Dnia 19 maja 2016 14:43:38 CEST, "Anthony G. Basile" <basile@opensource.dyc.edu> napisał(a):
>> From: "Anthony G. Basile" <blueness@gentoo.org>
>>
>> The current method to check for the system locale is to use python's
>> ctypes.util.find_library() to construct a full library path to the
>> system libc.so which is then passed to ctypes.CDLL().  However,
>> this gets bogged down in implementation dependant details and
>> fails with musl.
>>
>> We work around this design flaw in ctypes with a small python module
>> written in C called 'portage_c_check_locale', and only fall back on
>> the current ctypes-based check when this module is not available.
>>
>> This has been tested on glibc, uClibc and musl systems.
>>
>> X-Gentoo-bug: 571444
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
>> Signed-off-by: Anthony G. Basile <blueness@gentoo.org>
> 
> To be honest, I don't like this. Do we really want to duplicate all this, including duplicating the complete messages? This really looks messy and unmaintainable.
> 
> The only reason libc functions were being used was to workaround the hacky built-in case conversion functions. And this is the only part where CDLL was really used.
> 
> So please change this to provide trivial wrappers around the C library functions, and use them alternatively to ones obtained using CDLL. With a single common code doing all the check logic and messaging.
> 
> 

ctypes is itself a problem and so hacky python built-ins were replaced
by more hacky python.  CDLL shouldn't be used at all.  The other problem
is that non utf-8 encodings cause problems much earlier in the portage
codebase than the test for a sane environment, which is a bigger problem
than this small issue.  No one checked what happens with python2.7 +
portage + exotic locale.  Since I don't know where this might head in
the future, I kinda like the standalone potential.  The only repeated
code here is the message.  Nonetheless, I can reduce this to just two
functions, and do something like the following.  I assume that's what
you're suggesting:

try:
	from portage_c_convert_case import _c_toupper, _c_tolower
	libc_toupper = _c_toupper
	libc_lolower = _c_tolower
except ImportError:
	libc_fn = find_library("c")
	if libc_fn is None:
		return None
	libc = LoadLibrary(libc_fn)
	if libc is None:
		return None
	libc_toupper = libc.toupper
	libc_tolower = libc.tolower


Incidentally, another approach, one that I use in bash is as follows.  I
think it requires bash4, and I'm not sure how to pull this into python
gracefully.

l="abcdefghijklmnopqrstuvwxyz"
u="ABCDEFGHIJKLMNOPQRSTUVWXYZ"
ru=${l^^}
rl=${u,,}
[[ $l == $rl ]] && echo "lower case okay" || echo "lower case bad"
[[ $u == $ru ]] && echo "upper case okay" || echo "upper case bad"

-- 
Anthony G. Basile, Ph. D.
Chair of Information Technology
D'Youville College
Buffalo, NY 14201
(716) 829-8197

Re: [gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to check locale

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 3369 bytes --]

On Fri, 20 May 2016 06:59:14 -0400
"Anthony G. Basile" <basile@opensource.dyc.edu> wrote:

> On 5/19/16 9:38 AM, Michał Górny wrote:
> > Dnia 19 maja 2016 14:43:38 CEST, "Anthony G. Basile" <basile@opensource.dyc.edu> napisał(a):  
> >> From: "Anthony G. Basile" <blueness@gentoo.org>
> >>
> >> The current method to check for the system locale is to use python's
> >> ctypes.util.find_library() to construct a full library path to the
> >> system libc.so which is then passed to ctypes.CDLL().  However,
> >> this gets bogged down in implementation dependant details and
> >> fails with musl.
> >>
> >> We work around this design flaw in ctypes with a small python module
> >> written in C called 'portage_c_check_locale', and only fall back on
> >> the current ctypes-based check when this module is not available.
> >>
> >> This has been tested on glibc, uClibc and musl systems.
> >>
> >> X-Gentoo-bug: 571444
> >> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
> >> Signed-off-by: Anthony G. Basile <blueness@gentoo.org>  
> > 
> > To be honest, I don't like this. Do we really want to duplicate all this, including duplicating the complete messages? This really looks messy and unmaintainable.
> > 
> > The only reason libc functions were being used was to workaround the hacky built-in case conversion functions. And this is the only part where CDLL was really used.
> > 
> > So please change this to provide trivial wrappers around the C library functions, and use them alternatively to ones obtained using CDLL. With a single common code doing all the check logic and messaging.
> > 
> >   
> 
> ctypes is itself a problem and so hacky python built-ins were replaced
> by more hacky python.  CDLL shouldn't be used at all.  The other problem
> is that non utf-8 encodings cause problems much earlier in the portage
> codebase than the test for a sane environment, which is a bigger problem
> than this small issue.  No one checked what happens with python2.7 +
> portage + exotic locale.  Since I don't know where this might head in
> the future, I kinda like the standalone potential.  The only repeated
> code here is the message.  Nonetheless, I can reduce this to just two
> functions, and do something like the following.  I assume that's what
> you're suggesting:
> 
> try:
> 	from portage_c_convert_case import _c_toupper, _c_tolower
> 	libc_toupper = _c_toupper
> 	libc_lolower = _c_tolower
> except ImportError:
> 	libc_fn = find_library("c")
> 	if libc_fn is None:
> 		return None
> 	libc = LoadLibrary(libc_fn)
> 	if libc is None:
> 		return None
> 	libc_toupper = libc.toupper
> 	libc_tolower = libc.tolower
> 
> 
> Incidentally, another approach, one that I use in bash is as follows.  I
> think it requires bash4, and I'm not sure how to pull this into python
> gracefully.
> 
> l="abcdefghijklmnopqrstuvwxyz"
> u="ABCDEFGHIJKLMNOPQRSTUVWXYZ"
> ru=${l^^}
> rl=${u,,}
> [[ $l == $rl ]] && echo "lower case okay" || echo "lower case bad"
> [[ $u == $ru ]] && echo "upper case okay" || echo "upper case bad"

Exactly as pointed out above. You have to import tolower()
and toupper() from libc as Python case conversion functions don't give
the same results as 'pure' libc used by bash.

-- 
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 949 bytes --]

[gentoo-portage-dev] [PATCH 1/2] setup.py: add stub for building custom modules in C/C++

[Anthony G. Basile]

From: "Anthony G. Basile" <blueness@gentoo.org>

Currently portage doesn't include any custom modules written in C/C++.
This commit introduces stub code for building such modules in setup.py.

Signed-off-by: Anthony G. Basile <blueness@gentoo.org>
---
 setup.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 75c4bcb..25429bc 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 from __future__ import print_function
 
-from distutils.core import setup, Command
+from distutils.core import setup, Command, Extension
 from distutils.command.build import build
 from distutils.command.build_scripts import build_scripts
 from distutils.command.clean import clean
@@ -30,6 +30,9 @@ import sys
 # TODO:
 # - smarter rebuilds of docs w/ 'install_docbook' and 'install_epydoc'.
 
+# Dictionary of scripts.  The structure is
+#   key   = location in filesystem to install the scripts
+#   value = list of scripts, path relative to top source directory
 x_scripts = {
 	'bin': [
 		'bin/ebuild', 'bin/egencache', 'bin/emerge', 'bin/emerge-webrsync',
@@ -41,6 +44,10 @@ x_scripts = {
 	],
 }
 
+# Dictionary custom modules written in C/C++ here.  The structure is
+#   key   = module name
+#   value = list of C/C++ source code, path relative to top source directory
+x_c_helpers = {}
 
 class x_build(build):
 	""" Build command with extra build_man call. """
@@ -636,6 +643,8 @@ setup(
 		['$sysconfdir/portage/repo.postsync.d', ['cnf/repo.postsync.d/example']],
 	],
 
+	ext_modules = [Extension(name=n, sources=m) for n, m in x_c_helpers.items()],
+
 	cmdclass = {
 		'build': x_build,
 		'build_man': build_man,
-- 
2.7.3