[gentoo-dev] tree-sitter-grammar.eclass: support new upstream bindings

[gentoo-dev] tree-sitter-grammar.eclass: support new upstream bindings

[Arthur Zamarin]

In latest tree-sitter, we now have much better build system (a good
Makefile!), and much nicer to use python bindings per language. So
add support for them in the eclass, being mostly backwards compatible
with previous eclass (in the PR there are more commits which fix all
broken stuff).

Pull request: https://github.com/gentoo/gentoo/pull/35750

[gentoo-dev] [PATCH 1/2] tree-sitter-grammar.eclass: support for new upstream makefile

[Arthur Zamarin]

The build system for tree-sitters now generates a much better
Makefile we can use to build the parser and grammar into a good C
library.
This also matches the build procedure used by upstream, making our
reports easier for them to debug (we hit this issue in an old bug
report on memory leak with tree-sitter-bash).

Signed-off-by: Arthur Zamarin <arthurzam@gentoo.org>
---
 eclass/tree-sitter-grammar.eclass | 64 +++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 21 deletions(-)

diff --git a/eclass/tree-sitter-grammar.eclass b/eclass/tree-sitter-grammar.eclass
index b2563220cfc..13539daf7e6 100644
--- a/eclass/tree-sitter-grammar.eclass
+++ b/eclass/tree-sitter-grammar.eclass
@@ -1,10 +1,11 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 # @ECLASS: tree-sitter-grammar.eclass
 # @MAINTAINER:
 # Matthew Smith <matthew@gentoo.org>
 # Nick Sarnie <sarnex@gentoo.org>
+# Arthur Zamarin <arthurzam@gentoo.org>
 # @AUTHOR:
 # Matthew Smith <matthew@gentoo.org>
 # @SUPPORTED_EAPIS: 8
@@ -22,7 +23,7 @@ inherit edo multilib toolchain-funcs
 
 SRC_URI="https://github.com/tree-sitter/${PN}/archive/${TS_PV:-v${PV}}.tar.gz
 	-> ${P}.tar.gz"
-S="${WORKDIR}"/${PN}-${TS_PV:-${PV}}/src
+S="${WORKDIR}"/${PN}-${TS_PV:-${PV}}
 
 BDEPEND+=" test? ( dev-util/tree-sitter-cli )"
 IUSE+=" test"
@@ -44,15 +45,16 @@ _get_tsg_abi_ver() {
 	# This sed script finds ABI definition string in parser source file,
 	# substitutes all the string until the ABI number, and prints remains
 	# (the ABI number itself)
-	sed -n 's/#define LANGUAGE_VERSION //p' "${S}"/parser.c ||
+	sed -n 's/#define LANGUAGE_VERSION //p' "${S}"/src/parser.c ||
 		die "Unable to extract ABI version for this grammar"
 }
 
-# @FUNCTION: tree-sitter-grammar_src_compile
+# @FUNCTION: _tree-sitter-grammar_legacy_compile
+# @INTERNAL
 # @DESCRIPTION:
-# Compiles the Tree Sitter parser as a shared library.
-tree-sitter-grammar_src_compile() {
-	debug-print-function ${FUNCNAME} "${@}"
+# Compiles the Tree Sitter parser as a shared library, the legacy way.
+_tree-sitter-grammar_legacy_compile() {
+	cd "${S}/src" || die
 
 	# Grammars always contain parser.c, and sometimes a scanner.c,
 	# or scanner.cc.
@@ -60,17 +62,17 @@ tree-sitter-grammar_src_compile() {
 	tc-export CC CXX
 	# We want to use the bundled parser.h, not anything lurking on the system, hence -I
 	# See https://github.com/tree-sitter/tree-sitter-bash/issues/199#issuecomment-1694416505
-	export CFLAGS="${CFLAGS} -fPIC -I. -Itree_sitter"
-	export CXXFLAGS="${CXXFLAGS} -fPIC -I. -Itree_sitter"
+	local -x CFLAGS="${CFLAGS} -fPIC -I. -Itree_sitter"
+	local -x CXXFLAGS="${CXXFLAGS} -fPIC -I. -Itree_sitter"
 
 	local objects=( parser.o )
-	if [[ -f "${S}"/scanner.c || -f "${S}"/scanner.cc ]]; then
+	if [[ -f "${S}"/src/scanner.c || -f "${S}"/src/scanner.cc ]]; then
 		objects+=( scanner.o )
 	fi
 	emake "${objects[@]}"
 
 	local link="$(tc-getCC) ${CFLAGS}"
-	if [[ -f "${S}/scanner.cc" ]]; then
+	if [[ -f "${S}/src/scanner.cc" ]]; then
 		link="$(tc-getCXX) ${CXXFLAGS}"
 	fi
 
@@ -84,10 +86,24 @@ tree-sitter-grammar_src_compile() {
 	edo ${link} ${LDFLAGS} \
 			-shared \
 			*.o \
-			${soname_args} \
+			"${soname_args}" \
 			-o "${WORKDIR}"/${soname}
 }
 
+tree-sitter-grammar_src_compile() {
+	debug-print-function ${FUNCNAME} "${@}"
+
+	# legacy grammars don't have a pyproject.toml
+	if [[ -f "${S}/pyproject.toml" ]]; then
+		sed -e "/SONAME_MINOR :=/s/:=.*$/:= $(_get_tsg_abi_ver)/" -i "${S}/Makefile" || die
+		emake \
+			PREFIX="${EPREFIX}/usr" \
+			LIBDIR="${EPREFIX}/usr/$(get_libdir)"
+	else
+		_tree-sitter-grammar_legacy_compile
+	fi
+}
+
 # @FUNCTION: tree-sitter-grammar_src_test
 # @DESCRIPTION:
 # Runs the Tree Sitter parser's test suite.
@@ -95,20 +111,26 @@ tree-sitter-grammar_src_compile() {
 tree-sitter-grammar_src_test() {
 	debug-print-function ${FUNCNAME} "${@}"
 
-	(cd .. && tree-sitter test) || die "Test suite failed"
+	tree-sitter test || die "Test suite failed"
 }
 
-# @FUNCTION: tree-sitter-grammar_src_install
-# @DESCRIPTION:
-# Installs the Tree Sitter parser library.
 tree-sitter-grammar_src_install() {
 	debug-print-function ${FUNCNAME} "${@}"
 
-	local soname=lib${PN}$(get_libname $(_get_tsg_abi_ver))
-
-	dolib.so "${WORKDIR}/${soname}"
-	dosym "${soname}" \
-		  /usr/$(get_libdir)/lib${PN}$(get_libname)
+	# legacy grammars don't have a pyproject.toml
+	if [[ -f "${S}/pyproject.toml" ]]; then
+		emake \
+			PREFIX="${EPREFIX}/usr" \
+			LIBDIR="${EPREFIX}/usr/$(get_libdir)" \
+			DESTDIR="${D}/" \
+			install
+		find "${D}" -name '*.a' -delete || die "failed to remove static libraries"
+	else
+		local soname=lib${PN}$(get_libname $(_get_tsg_abi_ver))
+
+		dolib.so "${WORKDIR}/${soname}"
+		dosym "${soname}" /usr/$(get_libdir)/lib${PN}$(get_libname)
+	fi
 }
 
 fi
-- 
2.44.0

[gentoo-dev] [PATCH 2/2] tree-sitter-grammar.eclass: support opt in python bindings

[Arthur Zamarin]

New tree-sitter cli generated bindings and code around grammars and
parsers now support bulding a python wheel which supply much better
API and library for consumers in python bindings.

Currently I've added only python as a binding languages, even though
rust, swift, and go are also available. We should add them when we
see a request for them. Python will be needed for pkgcheck.

When we opt in into python bindings, we call the matching distutils
phase functions when `use python` is true.

Signed-off-by: Arthur Zamarin <arthurzam@gentoo.org>
---
 eclass/tree-sitter-grammar.eclass | 85 ++++++++++++++++++++++++++++++-
 1 file changed, 84 insertions(+), 1 deletion(-)

diff --git a/eclass/tree-sitter-grammar.eclass b/eclass/tree-sitter-grammar.eclass
index 13539daf7e6..b04d5ee1103 100644
--- a/eclass/tree-sitter-grammar.eclass
+++ b/eclass/tree-sitter-grammar.eclass
@@ -36,6 +36,43 @@ RESTRICT+=" !test? ( test )"
 # Used to override upstream tag name if tagged differently, e.g. most releases
 # are v${PV} but some are tagged as rust-${PV}.
 
+# @ECLASS_VARIABLE: TS_BINDINGS
+# @PRE_INHERIT
+# @DEFAULT_UNSET
+# @DESCRIPTION:
+# Array of bindings language to build. Currently only "python" is supported.
+
+for _BINDING in "${TS_BINDINGS[@]}"; do
+	case ${_BINDING} in
+		python)
+			DISTUTILS_EXT=1
+			DISTUTILS_OPTIONAL=1
+			DISTUTILS_USE_PEP517=setuptools
+			PYTHON_COMPAT=( python3_{10..12} )
+			inherit distutils-r1
+
+			IUSE+=" python"
+			REQUIRED_USE+=" python? ( ${PYTHON_REQUIRED_USE} )"
+
+			DEPEND+=" python? (
+				${PYTHON_DEPS}
+			)"
+			RDEPEND+=" python? (
+				${PYTHON_DEPS}
+				>=dev-python/tree-sitter-0.21.0[${PYTHON_USEDEP}]
+			)"
+			BDEPEND+=" python? (
+				${DISTUTILS_DEPS}
+				dev-python/wheel[${PYTHON_USEDEP}]
+			)"
+			;;
+		*)
+			die "Unknown binding: ${_BINDING}"
+			;;
+	esac
+done
+unset _BINDING
+
 # @FUNCTION: _get_tsg_abi_ver
 # @INTERNAL
 # @DESCRIPTION:
@@ -49,6 +86,34 @@ _get_tsg_abi_ver() {
 		die "Unable to extract ABI version for this grammar"
 }
 
+tree-sitter-grammar_src_prepare() {
+	debug-print-function ${FUNCNAME} "${@}"
+
+	default
+
+	local binding
+	for binding in "${TS_BINDINGS[@]}"; do
+		case ${binding} in
+			python)
+				use python && distutils-r1_src_prepare
+				;;
+		esac
+	done
+}
+
+tree-sitter-grammar_src_configure() {
+	debug-print-function ${FUNCNAME} "${@}"
+
+	local binding
+	for binding in "${TS_BINDINGS[@]}"; do
+		case ${binding} in
+			python)
+				use python && distutils-r1_src_configure
+				;;
+		esac
+	done
+}
+
 # @FUNCTION: _tree-sitter-grammar_legacy_compile
 # @INTERNAL
 # @DESCRIPTION:
@@ -102,6 +167,15 @@ tree-sitter-grammar_src_compile() {
 	else
 		_tree-sitter-grammar_legacy_compile
 	fi
+
+	local binding
+	for binding in "${TS_BINDINGS[@]}"; do
+		case ${binding} in
+			python)
+				use python && distutils-r1_src_compile
+				;;
+		esac
+	done
 }
 
 # @FUNCTION: tree-sitter-grammar_src_test
@@ -131,8 +205,17 @@ tree-sitter-grammar_src_install() {
 		dolib.so "${WORKDIR}/${soname}"
 		dosym "${soname}" /usr/$(get_libdir)/lib${PN}$(get_libname)
 	fi
+
+	local binding
+	for binding in "${TS_BINDINGS[@]}"; do
+		case ${binding} in
+			python)
+				use python && distutils-r1_src_install
+				;;
+		esac
+	done
 }
 
 fi
 
-EXPORT_FUNCTIONS src_compile src_test src_install
+EXPORT_FUNCTIONS src_prepare src_configure src_compile src_test src_install
-- 
2.44.0