[gentoo-dev] [PATCH python-utils-r1] Move python_fix_shebang to python-utils-r1 and rewrite it clean.

[gentoo-dev] [PATCH python-utils-r1] Move python_fix_shebang to python-utils-r1 and rewrite it clean.

[Michał Górny]

It serves both as public shebang fixing function and replacement of
_python_rewrite_shebang internal function. For the sake of having common
code and consistent behavior.

Notes on the 'new' function:

1. takes a list of files and/or directories to fix. Directories are
processed recursively,

2. files, depending on the shebang:

a) with shebang matching $EPYTHON (e.g. already having pythonX.Y) are
skipped silently (but see 3),

b) with shebang 'compatible' with $EPYTHON (e.g. python3 -> python3.2,
but not python2 -> python3.2) are mangled verbosely,

c) with shebang 'incompatible' with $EPYTHON (e.g. python3 -> python2.7,
python3.2 -> python3.3) raise a fatal error,

d) with shebang not looking like Python at all are skipped when
processing directory recursively and raise a fatal error when are
specified directly.

3. there are two new QA warnings:

a) when specified file (or all files in the directory) has proper
shebang already -- likely attempting double mangling,

b) when specified directory contains no Python files.
---
 eclass/python-r1.eclass        |   6 +-
 eclass/python-single-r1.eclass |  44 -----------
 eclass/python-utils-r1.eclass  | 170 +++++++++++++++++++++++------------------
 3 files changed, 100 insertions(+), 120 deletions(-)

diff --git a/eclass/python-r1.eclass b/eclass/python-r1.eclass
index 0fb188d..876ce7e 100644
--- a/eclass/python-r1.eclass
+++ b/eclass/python-r1.eclass
@@ -783,6 +783,8 @@ python_replicate_script() {
 	debug-print-function ${FUNCNAME} "${@}"
 
 	_python_replicate_script() {
+		local _PYTHON_FIX_SHEBANG_QUIET=1
+
 		if _python_want_python_exec2; then
 			local PYTHON_SCRIPTDIR
 			python_export PYTHON_SCRIPTDIR
@@ -792,7 +794,7 @@ python_replicate_script() {
 				doexe "${files[@]}"
 			)
 
-			_python_rewrite_shebang "${EPYTHON}" \
+			python_fix_shebang \
 				"${files[@]/*\//${D%/}/${PYTHON_SCRIPTDIR}/}"
 		else
 			local f
@@ -800,7 +802,7 @@ python_replicate_script() {
 				cp -p "${f}" "${f}-${EPYTHON}" || die
 			done
 
-			_python_rewrite_shebang "${EPYTHON}" \
+			python_fix_shebang \
 				"${files[@]/%/-${EPYTHON}}"
 		fi
 	}
diff --git a/eclass/python-single-r1.eclass b/eclass/python-single-r1.eclass
index 7ce57c6..2a3a3fc 100644
--- a/eclass/python-single-r1.eclass
+++ b/eclass/python-single-r1.eclass
@@ -261,49 +261,5 @@ python-single-r1_pkg_setup() {
 	python_setup
 }
 
-# @FUNCTION: python_fix_shebang
-# @USAGE: <path>...
-# @DESCRIPTION:
-# Replace the shebang in Python scripts with the current Python
-# implementation (EPYTHON). If a directory is passed, works recursively
-# on all Python scripts.
-#
-# Only files having a 'python' shebang will be modified; other files
-# will be skipped. If a script has a complete shebang matching
-# the chosen interpreter version, it is left unmodified. If a script has
-# a complete shebang matching other version, the command dies.
-python_fix_shebang() {
-	debug-print-function ${FUNCNAME} "${@}"
-
-	[[ ${1} ]] || die "${FUNCNAME}: no paths given"
-	[[ ${EPYTHON} ]] || die "${FUNCNAME}: EPYTHON unset (pkg_setup not called?)"
-
-	local path f
-	for path; do
-		while IFS= read -r -d '' f; do
-			local shebang=$(head -n 1 "${f}")
-
-			case "${shebang}" in
-				'#!'*${EPYTHON}*)
-					debug-print "${FUNCNAME}: in file ${f#${D}}"
-					debug-print "${FUNCNAME}: shebang matches EPYTHON: ${shebang}"
-					;;
-				'#!'*python[23].[0123456789]*|'#!'*pypy-c*|'#!'*jython*)
-					debug-print "${FUNCNAME}: in file ${f#${D}}"
-					debug-print "${FUNCNAME}: incorrect specific shebang: ${shebang}"
-
-					die "${f#${D}} has a specific Python shebang not matching EPYTHON"
-					;;
-				'#!'*python*)
-					debug-print "${FUNCNAME}: in file ${f#${D}}"
-					debug-print "${FUNCNAME}: rewriting shebang: ${shebang}"
-
-					einfo "Fixing shebang in ${f#${D}}"
-					_python_rewrite_shebang "${f}"
-			esac
-		done < <(find "${path}" -type f -print0)
-	done
-}
-
 _PYTHON_SINGLE_R1=1
 fi
diff --git a/eclass/python-utils-r1.eclass b/eclass/python-utils-r1.eclass
index 81d6691..e292760 100644
--- a/eclass/python-utils-r1.eclass
+++ b/eclass/python-utils-r1.eclass
@@ -478,79 +478,6 @@ python_get_scriptdir() {
 	echo "${PYTHON_SCRIPTDIR}"
 }
 
-# @FUNCTION: _python_rewrite_shebang
-# @USAGE: [<EPYTHON>] <path>...
-# @INTERNAL
-# @DESCRIPTION:
-# Replaces 'python' executable in the shebang with the executable name
-# of the specified interpreter. If no EPYTHON value (implementation) is
-# used, the current ${EPYTHON} will be used.
-#
-# All specified files must start with a 'python' shebang. A file not
-# having a matching shebang will be refused. The exact shebang style
-# will be preserved in order not to break anything.
-#
-# Example conversions:
-# @CODE
-# From: #!/usr/bin/python -R
-# To: #!/usr/bin/python2.7 -R
-#
-# From: #!/usr/bin/env FOO=bar python
-# To: #!/usr/bin/env FOO=bar python2.7
-# @CODE
-_python_rewrite_shebang() {
-	debug-print-function ${FUNCNAME} "${@}"
-
-	local impl
-	case "${1}" in
-		python*|jython*|pypy*)
-			impl=${1}
-			shift
-			;;
-		*)
-			impl=${EPYTHON}
-			[[ ${impl} ]] || die "${FUNCNAME}: no impl nor EPYTHON"
-			;;
-	esac
-	debug-print "${FUNCNAME}: implementation: ${impl}"
-
-	local f
-	for f; do
-		local from shebang
-		read -r shebang < "${f}"
-		shebang=${shebang%$'\r'}
-		debug-print "${FUNCNAME}: path = ${f}"
-		debug-print "${FUNCNAME}: shebang = ${shebang}"
-
-		if [[ "${shebang} " == *"${impl} "* ]]; then
-			# skip files with correct impl
-			continue
-		elif [[ "${shebang} " == *'python '* ]]; then
-			from=python
-		elif [[ "${shebang} " == *'python2 '* ]]; then
-			from=python2
-		elif [[ "${shebang} " == *'python3 '* ]]; then
-			from=python3
-		else
-			eerror "A file does not seem to have a supported shebang:"
-			eerror "  file: ${f}"
-			eerror "  shebang: ${shebang}"
-			die "${FUNCNAME}: ${f} does not seem to have a valid shebang"
-		fi
-
-		if { [[ ${from} == python2 ]] && python_is_python3 "${impl}"; } \
-				|| { [[ ${from} == python3 ]] && ! python_is_python3 "${impl}"; } then
-			eerror "A file does have shebang not supporting requested impl:"
-			eerror "  file: ${f}"
-			eerror "  shebang: ${shebang}"
-			eerror "  impl: ${impl}"
-			die "${FUNCNAME}: ${f} does have shebang not supporting ${EPYTHON}"
-		fi
-
-		sed -i -e "1s:${from}:${impl}:" "${f}" || die
-	done
-}
-
 # @FUNCTION: _python_ln_rel
 # @USAGE: <from> <to>
 # @INTERNAL
@@ -743,7 +670,8 @@ python_newexe() {
 
 	# don't use this at home, just call python_doscript() instead
 	if [[ ${_PYTHON_REWRITE_SHEBANG} ]]; then
-		_python_rewrite_shebang "${ED%/}/${d}/${newfn}"
+		local _PYTHON_FIX_SHEBANG_QUIET=1
+		python_fix_shebang "${ED%/}/${d}/${newfn}"
 	fi
 }
 
@@ -1006,6 +934,100 @@ python_is_python3() {
 	[[ ${impl} == python3* ]]
 }
 
+# @FUNCTION: python_fix_shebang
+# @USAGE: <path>...
+# @DESCRIPTION:
+# Replace the shebang in Python scripts with the current Python
+# implementation (EPYTHON). If a directory is passed, works recursively
+# on all Python scripts.
+#
+# Only files having a 'python*' shebang will be modified. Files with
+# other shebang will either be skipped when working recursively
+# on a directory or treated as error when specified explicitly.
+#
+# Shebangs matching explicitly current Python version will be left
+# unmodified. Shebangs requesting another Python version will be treated
+# as fatal error.
+python_fix_shebang() {
+	debug-print-function ${FUNCNAME} "${@}"
+
+	[[ ${1} ]] || die "${FUNCNAME}: no paths given"
+	[[ ${EPYTHON} ]] || die "${FUNCNAME}: EPYTHON unset (pkg_setup not called?)"
+
+	local path f
+	for path; do
+		local any_correct any_fixed is_recursive
+
+		[[ -d ${path} ]] && is_recursive=1
+
+		while IFS= read -r -d '' f; do
+			local shebang=$(head -n 1 "${f}")
+			local error
+
+			case "${shebang} " in
+				'#!'*"${EPYTHON} "*)
+					debug-print "${FUNCNAME}: in file ${f#${D}}"
+					debug-print "${FUNCNAME}: shebang matches EPYTHON: ${shebang}"
+
+					# Nothing to do, move along.
+					any_correct=1
+					;;
+				'#!'*python" "*|'#!'*python[23]" "*)
+					debug-print "${FUNCNAME}: in file ${f#${D}}"
+					debug-print "${FUNCNAME}: rewriting shebang: ${shebang}"
+
+					# Note: for internal use.
+					if [[ ! ${_PYTHON_FIX_SHEBANG_QUIET} ]]; then
+						einfo "Fixing shebang in ${f#${D}}."
+					fi
+
+					local from
+					if [[ "${shebang} " == *'python2 '* ]]; then
+						from=python2
+						python_is_python3 "${EPYTHON}" && error=1
+					elif [[ "${shebang} " == *'python3 '* ]]; then
+						from=python3
+						python_is_python3 "${EPYTHON}" || error=1
+					else
+						from=python
+					fi
+
+					if [[ ! ${error} ]]; then
+						sed -i -e "1s:${from}:${EPYTHON}:" "${f}" || die
+						any_fixed=1
+					fi
+					;;
+				'#!'*python[23].[0123456789]" "*|'#!'*pypy" "*|'#!'*jython[23].[0123456789]" "*)
+					# Explicit mismatch.
+					error=1
+					;;
+				*)
+					# Non-Python shebang. Allowed in recursive mode,
+					# disallowed when specifying file explicitly.
+					[[ ${is_recursive} ]] || error=1
+					;;
+			esac
+
+			if [[ ${error} ]]; then
+				eerror "The file has incompatible shebang:"
+				eerror "  file: ${f#${D}}"
+				eerror "  current shebang: ${shebang}"
+				eerror "  requested impl: ${EPYTHON}"
+				die "${FUNCNAME}: conversion of incompatible shebang requested"
+			fi
+		done < <(find "${path}" -type f -print0)
+
+		if [[ ! ${any_fixed} ]]; then
+			eqawarn "QA warning: ${FUNCNAME}, ${path#${D}} did not match any fixable files."
+			if [[ ${any_correct} ]]; then
+				eqawarn "All files have ${EPYTHON} shebang already."
+			else
+				eqawarn "There are no Python files in specified directory."
+			fi
+		fi
+	done
+}
+
 # @FUNCTION: _python_want_python_exec2
 # @INTERNAL
 # @DESCRIPTION:
-- 
1.9.3

[gentoo-dev] Re: [PATCH python-utils-r1] Move python_fix_shebang to python-utils-r1 and rewrite it clean.

[Mike Gilbert]

On Tue, May 20, 2014 at 2:19 PM, Michał Górny <mgorny@gentoo.org> wrote:
> It serves both as public shebang fixing function and replacement of
> _python_rewrite_shebang internal function. For the sake of having common
> code and consistent behavior.
>
> Notes on the 'new' function:
>
> 1. takes a list of files and/or directories to fix. Directories are
> processed recursively,
>
> 2. files, depending on the shebang:
>
> a) with shebang matching $EPYTHON (e.g. already having pythonX.Y) are
> skipped silently (but see 3),
>
> b) with shebang 'compatible' with $EPYTHON (e.g. python3 -> python3.2,
> but not python2 -> python3.2) are mangled verbosely,
>
> c) with shebang 'incompatible' with $EPYTHON (e.g. python3 -> python2.7,
> python3.2 -> python3.3) raise a fatal error,
>

This fatal behavior is a little annoying. I seem to recall people
asking if there is a way to just force it, no matter if the shebang is
'compatible' or not.

Unfortunately, I do not have example handy, but I know I have been
asked the question.

Re: [gentoo-dev] Re: [PATCH python-utils-r1] Move python_fix_shebang to python-utils-r1 and rewrite it clean.

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 1637 bytes --]

Dnia 2014-05-20, o godz. 14:27:04
Mike Gilbert <floppym@gentoo.org> napisał(a):

> On Tue, May 20, 2014 at 2:19 PM, Michał Górny <mgorny@gentoo.org> wrote:
> > It serves both as public shebang fixing function and replacement of
> > _python_rewrite_shebang internal function. For the sake of having common
> > code and consistent behavior.
> >
> > Notes on the 'new' function:
> >
> > 1. takes a list of files and/or directories to fix. Directories are
> > processed recursively,
> >
> > 2. files, depending on the shebang:
> >
> > a) with shebang matching $EPYTHON (e.g. already having pythonX.Y) are
> > skipped silently (but see 3),
> >
> > b) with shebang 'compatible' with $EPYTHON (e.g. python3 -> python3.2,
> > but not python2 -> python3.2) are mangled verbosely,
> >
> > c) with shebang 'incompatible' with $EPYTHON (e.g. python3 -> python2.7,
> > python3.2 -> python3.3) raise a fatal error,
> >
> 
> This fatal behavior is a little annoying. I seem to recall people
> asking if there is a way to just force it, no matter if the shebang is
> 'compatible' or not.

I literally spent weeks cleaning up this function and you already
request new features :P.

> Unfortunately, I do not have example handy, but I know I have been
> asked the question.

Well, I think it was about some random package having files with
'python3' shebang while they worked with python2.7 as well. I guess we
may add some -f/--force option at a point. While at it, it would
probably be good if we could work on some consistency in argument
parsing of various helper functions.

-- 
Best regards,
Michał Górny

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 966 bytes --]

Re: [gentoo-dev] [PATCH python-utils-r1] Move python_fix_shebang to python-utils-r1 and rewrite it clean.

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 313 bytes --]

Dnia 2014-05-20, o godz. 20:19:58
Michał Górny <mgorny@gentoo.org> napisał(a):

> It serves both as public shebang fixing function and replacement of
> _python_rewrite_shebang internal function. For the sake of having common
> code and consistent behavior.

Committed.

-- 
Best regards,
Michał Górny

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 966 bytes --]