[gentoo-portage-dev] [PATCH 1/2] portage.util.configparser: Provide common code to handle cp imports

[gentoo-portage-dev] [PATCH 1/2] portage.util.configparser: Provide common code to handle cp imports

[Michał Górny]

Provide a common code unit to handle portable *ConfigParser imports
for all supported Python versions.
---
 pym/portage/_sets/__init__.py      | 10 ++--------
 pym/portage/repository/config.py   |  9 +--------
 pym/portage/util/_desktop_entry.py |  7 ++-----
 pym/portage/util/configparser.py   | 22 ++++++++++++++++++++++
 4 files changed, 27 insertions(+), 21 deletions(-)
 create mode 100644 pym/portage/util/configparser.py

diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py
index 3203521..ec42f7c 100644
--- a/pym/portage/_sets/__init__.py
+++ b/pym/portage/_sets/__init__.py
@@ -9,14 +9,6 @@ __all__ = ["SETPREFIX", "get_boolean", "SetConfigError",
 import io
 import logging
 import sys
-try:
-	from configparser import NoOptionError, ParsingError
-	if sys.hexversion >= 0x3020000:
-		from configparser import ConfigParser as SafeConfigParser
-	else:
-		from configparser import SafeConfigParser
-except ImportError:
-	from ConfigParser import SafeConfigParser, NoOptionError, ParsingError
 import portage
 from portage import os
 from portage import load_mod
@@ -29,6 +21,8 @@ from portage.const import _ENABLE_SET_CONFIG
 from portage.exception import PackageSetNotFound
 from portage.localization import _
 from portage.util import writemsg_level
+from portage.util.configparser import (SafeConfigParser,
+	NoOptionError, ParsingError)
 
 SETPREFIX = "@"
 
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 00319fe..9039886 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -9,14 +9,6 @@ import warnings
 import sys
 import re
 
-try:
-	from configparser import Error as ConfigParserError
-	if sys.hexversion >= 0x3020000:
-		from configparser import ConfigParser as SafeConfigParser
-	else:
-		from configparser import SafeConfigParser
-except ImportError:
-	from ConfigParser import SafeConfigParser, Error as ConfigParserError
 import portage
 from portage import eclass_cache, os
 from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH,
@@ -25,6 +17,7 @@ from portage.eapi import eapi_allows_directories_on_profile_level_and_repository
 from portage.env.loaders import KeyValuePairFileLoader
 from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split,
 	stack_lists, writemsg, writemsg_level, _recursive_file_list)
+from portage.util.configparser import SafeConfigParser, ConfigParserError
 from portage.util._path import isdir_raise_eaccess
 from portage.util.path import first_existing
 from portage.localization import _
diff --git a/pym/portage/util/_desktop_entry.py b/pym/portage/util/_desktop_entry.py
index 0b49547..95a015e 100644
--- a/pym/portage/util/_desktop_entry.py
+++ b/pym/portage/util/_desktop_entry.py
@@ -6,14 +6,11 @@ import re
 import subprocess
 import sys
 
-try:
-	from configparser import Error as ConfigParserError, RawConfigParser
-except ImportError:
-	from ConfigParser import Error as ConfigParserError, RawConfigParser
-
 import portage
 from portage import _encodings, _unicode_encode, _unicode_decode
 from portage.util import writemsg
+from portage.util.configparser import ConfigParserError, RawConfigParser
+
 
 def parse_desktop_entry(path):
 	"""
diff --git a/pym/portage/util/configparser.py b/pym/portage/util/configparser.py
new file mode 100644
index 0000000..d305052
--- /dev/null
+++ b/pym/portage/util/configparser.py
@@ -0,0 +1,22 @@
+# Copyright 2016 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+__all__ = ['ConfigParserError', 'NoOptionError', 'ParsingError',
+	'RawConfigParser', 'SafeConfigParser']
+
+# the following scary compatibility thing provides two classes:
+# - SafeConfigParser that provides safe interpolation for values,
+# - RawConfigParser that provides no interpolation for values.
+
+import sys
+
+try:
+	from configparser import (Error as ConfigParserError,
+		NoOptionError, ParsingError, RawConfigParser)
+	if sys.hexversion >= 0x3020000:
+		from configparser import ConfigParser as SafeConfigParser
+	else:
+		from configparser import SafeConfigParser
+except ImportError:
+	from ConfigParser import (Error as ConfigParserError,
+		NoOptionError, ParsingError, RawConfigParser, SafeConfigParser)
-- 
2.8.3

[gentoo-portage-dev] [PATCH 2/2] portage.util.configparser: Commonize portable config file reading routine

[Michał Górny]

---
 pym/portage/_sets/__init__.py      | 29 ++-----------------
 pym/portage/repository/config.py   | 38 ++-----------------------
 pym/portage/util/_desktop_entry.py | 20 ++-----------
 pym/portage/util/configparser.py   | 57 +++++++++++++++++++++++++++++++++++++-
 4 files changed, 64 insertions(+), 80 deletions(-)

diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py
index ec42f7c..6d69bda 100644
--- a/pym/portage/_sets/__init__.py
+++ b/pym/portage/_sets/__init__.py
@@ -22,7 +22,7 @@ from portage.exception import PackageSetNotFound
 from portage.localization import _
 from portage.util import writemsg_level
 from portage.util.configparser import (SafeConfigParser,
-	NoOptionError, ParsingError)
+	NoOptionError, ParsingError, read_configs)
 
 SETPREFIX = "@"
 
@@ -50,32 +50,7 @@ class SetConfig(object):
 			})
 
 		if _ENABLE_SET_CONFIG:
-			# use read_file/readfp in order to control decoding of unicode
-			try:
-				# Python >=3.2
-				read_file = self._parser.read_file
-			except AttributeError:
-				read_file = self._parser.readfp
-
-			for p in paths:
-				f = None
-				try:
-					f = io.open(_unicode_encode(p,
-						encoding=_encodings['fs'], errors='strict'),
-						mode='r', encoding=_encodings['repo.content'],
-						errors='replace')
-				except EnvironmentError:
-					pass
-				else:
-					try:
-						read_file(f)
-					except ParsingError as e:
-						writemsg_level(_unicode_decode(
-							_("!!! Error while reading sets config file: %s\n")
-							) % e, level=logging.ERROR, noiselevel=-1)
-				finally:
-					if f is not None:
-						f.close()
+			read_configs(self._parser, paths)
 		else:
 			self._create_default_config()
 
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 9039886..a23f4bd 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -17,7 +17,8 @@ from portage.eapi import eapi_allows_directories_on_profile_level_and_repository
 from portage.env.loaders import KeyValuePairFileLoader
 from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split,
 	stack_lists, writemsg, writemsg_level, _recursive_file_list)
-from portage.util.configparser import SafeConfigParser, ConfigParserError
+from portage.util.configparser import (SafeConfigParser, ConfigParserError,
+	read_configs)
 from portage.util._path import isdir_raise_eaccess
 from portage.util.path import first_existing
 from portage.localization import _
@@ -542,15 +543,6 @@ class RepoConfigLoader(object):
 		"""Parse files in paths to load config"""
 		parser = SafeConfigParser(defaults=default_opts)
 
-		# use read_file/readfp in order to control decoding of unicode
-		try:
-			# Python >=3.2
-			read_file = parser.read_file
-			source_kwarg = 'source'
-		except AttributeError:
-			read_file = parser.readfp
-			source_kwarg = 'filename'
-
 		recursive_paths = []
 		for p in paths:
 			if isinstance(p, basestring):
@@ -558,31 +550,7 @@ class RepoConfigLoader(object):
 			else:
 				recursive_paths.append(p)
 
-		for p in recursive_paths:
-			if isinstance(p, basestring):
-				f = None
-				try:
-					f = io.open(_unicode_encode(p,
-						encoding=_encodings['fs'], errors='strict'),
-						mode='r', encoding=_encodings['repo.content'],
-						errors='replace')
-				except EnvironmentError:
-					pass
-				else:
-					# The 'source' keyword argument is needed since otherwise
-					# ConfigParser in Python <3.3.3 may throw a TypeError
-					# because it assumes that f.name is a native string rather
-					# than binary when constructing error messages.
-					kwargs = {source_kwarg: p}
-					read_file(f, **portage._native_kwargs(kwargs))
-				finally:
-					if f is not None:
-						f.close()
-			elif isinstance(p, io.StringIO):
-				kwargs = {source_kwarg: "<io.StringIO>"}
-				read_file(p, **portage._native_kwargs(kwargs))
-			else:
-				raise TypeError("Unsupported type %r of element %r of 'paths' argument" % (type(p), p))
+		read_configs(parser, recursive_paths)
 
 		prepos['DEFAULT'] = RepoConfig("DEFAULT",
 			parser.defaults(), local_config=local_config)
diff --git a/pym/portage/util/_desktop_entry.py b/pym/portage/util/_desktop_entry.py
index 95a015e..4fe4194 100644
--- a/pym/portage/util/_desktop_entry.py
+++ b/pym/portage/util/_desktop_entry.py
@@ -9,7 +9,8 @@ import sys
 import portage
 from portage import _encodings, _unicode_encode, _unicode_decode
 from portage.util import writemsg
-from portage.util.configparser import ConfigParserError, RawConfigParser
+from portage.util.configparser import (ConfigParserError, RawConfigParser,
+	read_configs)
 
 
 def parse_desktop_entry(path):
@@ -20,22 +21,7 @@ def parse_desktop_entry(path):
 	"""
 	parser = RawConfigParser()
 
-	# use read_file/readfp in order to control decoding of unicode
-	try:
-		# Python >=3.2
-		read_file = parser.read_file
-	except AttributeError:
-		read_file = parser.readfp
-
-	with io.open(_unicode_encode(path,
-		encoding=_encodings['fs'], errors='strict'),
-		mode='r', encoding=_encodings['repo.content'],
-		errors='replace') as f:
-		content = f.read()
-
-	# In Python 3.2, read_file does not support bytes in file names
-	# (see bug #429544), so use StringIO to hide the file name.
-	read_file(io.StringIO(content))
+	read_configs(parser, [path])
 
 	return parser
 
diff --git a/pym/portage/util/configparser.py b/pym/portage/util/configparser.py
index d305052..fb1a351 100644
--- a/pym/portage/util/configparser.py
+++ b/pym/portage/util/configparser.py
@@ -2,12 +2,13 @@
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ['ConfigParserError', 'NoOptionError', 'ParsingError',
-	'RawConfigParser', 'SafeConfigParser']
+	'RawConfigParser', 'SafeConfigParser', 'read_configs']
 
 # the following scary compatibility thing provides two classes:
 # - SafeConfigParser that provides safe interpolation for values,
 # - RawConfigParser that provides no interpolation for values.
 
+import io
 import sys
 
 try:
@@ -20,3 +21,57 @@ try:
 except ImportError:
 	from ConfigParser import (Error as ConfigParserError,
 		NoOptionError, ParsingError, RawConfigParser, SafeConfigParser)
+
+from portage import _encodings
+from portage import _native_kwargs
+from portage import _unicode_encode
+
+
+if sys.hexversion >= 0x3000000:
+	# pylint: disable=W0622
+	basestring = str
+
+
+def read_configs(parser, paths):
+	"""
+	Read configuration files from given paths into the specified
+	ConfigParser, handling path encoding portably.
+	@param parser: target *ConfigParser instance
+	@type parser: SafeConfigParser or RawConfigParser
+	@param paths: list of paths to read
+	@type paths: iterable
+	"""
+	# use read_file/readfp in order to control decoding of unicode
+	try:
+		# Python >=3.2
+		read_file = parser.read_file
+		source_kwarg = 'source'
+	except AttributeError:
+		read_file = parser.readfp
+		source_kwarg = 'filename'
+
+	for p in paths:
+		if isinstance(p, basestring):
+			f = None
+			try:
+				f = io.open(_unicode_encode(p,
+					encoding=_encodings['fs'], errors='strict'),
+					mode='r', encoding=_encodings['repo.content'],
+					errors='replace')
+			except EnvironmentError:
+				pass
+			else:
+				# The 'source' keyword argument is needed since otherwise
+				# ConfigParser in Python <3.3.3 may throw a TypeError
+				# because it assumes that f.name is a native string rather
+				# than binary when constructing error messages.
+				kwargs = {source_kwarg: p}
+				read_file(f, **_native_kwargs(kwargs))
+			finally:
+				if f is not None:
+					f.close()
+		elif isinstance(p, io.StringIO):
+			kwargs = {source_kwarg: "<io.StringIO>"}
+			read_file(p, **_native_kwargs(kwargs))
+		else:
+			raise TypeError("Unsupported type %r of element %r of 'paths' argument" % (type(p), p))
-- 
2.8.3

Re: [gentoo-portage-dev] [PATCH 2/2] portage.util.configparser: Commonize portable config file reading routine

[Zac Medico]

On 05/22/2016 01:41 AM, Michał Górny wrote:
> ---
>  pym/portage/_sets/__init__.py      | 29 ++-----------------
>  pym/portage/repository/config.py   | 38 ++-----------------------
>  pym/portage/util/_desktop_entry.py | 20 ++-----------
>  pym/portage/util/configparser.py   | 57 +++++++++++++++++++++++++++++++++++++-
>  4 files changed, 64 insertions(+), 80 deletions(-)

These patches look good. Note that the _native_kwargs function is not
really needed anymore, because it was just a workaround for this issue
which only affected python 2.6.4 and earlier:

   http://bugs.python.org/issue4978
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 2/2] portage.util.configparser: Commonize portable config file reading routine

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 880 bytes --]

On Sun, 22 May 2016 11:04:51 -0700
Zac Medico <zmedico@gentoo.org> wrote:

> On 05/22/2016 01:41 AM, Michał Górny wrote:
> > ---
> >  pym/portage/_sets/__init__.py      | 29 ++-----------------
> >  pym/portage/repository/config.py   | 38 ++-----------------------
> >  pym/portage/util/_desktop_entry.py | 20 ++-----------
> >  pym/portage/util/configparser.py   | 57 +++++++++++++++++++++++++++++++++++++-
> >  4 files changed, 64 insertions(+), 80 deletions(-)  
> 
> These patches look good. Note that the _native_kwargs function is not
> really needed anymore, because it was just a workaround for this issue
> which only affected python 2.6.4 and earlier:
> 
>    http://bugs.python.org/issue4978

Pushed now, thanks.

I'll do a global _native_kwargs cleanup in a separate patch.

-- 
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 949 bytes --]