[gentoo-portage-dev] Proposal of "uncooperative-root" in SUPPORTED_FEATURES

[gentoo-portage-dev] Proposal of "uncooperative-root" in SUPPORTED_FEATURES

[X dej]

[-- Attachment #1: Type: text/plain, Size: 1560 bytes --]

Hello to all,

Summary: I have made many mods to sys-app/portage (2013 version). The result is
a proposal of patch for latest sys-app/portage, see below.

Main result of my mods to gentoo prefix:

  FEATURES=uncooperative-root emerge -bv =firefox-17

now works on EPREFIX on Android-4.1 ARM. I use neither fakeroot nor GNURoot.
root access never required. Distributed as a .tbz2 and a .ebuild by
gentooandroid.sf.net, along with the necessary stage3.

@dol-sen advised me one week ago on #gentoo-portage to post a rebased patch for
HEAD of git.overlays.gentoo.org/proj/portage.git : I just hosted this patch on
sf.net/projects/gentooandroid/files/sys-app_portage-current-HEAD_patch/download
and it is also attached to this post, and to previous post on gentoo-dev
mailing list; Joshua Kinard from there just advised me to post this patch at
gentoo-portage-dev (here).

I request comments and discussions about this patch. Summary:

pym/portage/const.py     + (this is SUPPORTED_FEATURES+=uncooperative-root)
cnf/make.conf.example    ++++++++++++++++++++++++++
bin/ebuild-helpers/emake ++++++++
bin/ebuild.sh            --+++++++
bin/misc-functions.sh    -+++++
bin/phase-helpers.sh     +++++

My aim is to take your remarks into account, then to build an overlay (will be
hosted at endrood.sf.net) with up-to-date versions of all my mods. The final
result will be a tested version of above patch, together with lastest builds
for firefox, squeak, scala, gnucash, ...

Thanks for you attention.

Xdej (I disclosed my real identity to Rick "Zero_Chaos" Farina).

[-- Attachment #2: sys-app_portage-current-HEAD_patch.txt --]
[-- Type: text/plain, Size: 5415 bytes --]

Patch to apply against HEAD of sys-app/portage project.
I did git clone git://git.overlays.gentoo.org/proj/portage.git on 2014-07-04
13:47:53.442947599 +0000.

diff -ru old_portage/pym/portage/const.py portage/pym/portage/const.py
--- old_portage/pym/portage/const.py	2014-07-04 16:35:37.462942289 +0200
+++ portage/pym/portage/const.py	2014-07-04 17:03:00.347420753 +0200
@@ -119,6 +119,7 @@
 	"other",
 )
 SUPPORTED_FEATURES       = frozenset([
+	"uncooperative-root",
 	"assume-digests",
 	"binpkg-logs",
 	"buildpkg",
diff -ru old_portage/cnf/make.conf.example portage/cnf/make.conf.example
--- old_portage/cnf/make.conf.example	2014-07-04 16:52:24.438859863 +0200
+++ portage/cnf/make.conf.example	2014-07-04 17:08:10.658545393 +0200
@@ -362,3 +362,29 @@
 #                               ${PACKAGE} - see description of PORTAGE_ELOG_COMMAND
 #                               ${HOST} - FQDN of the host portage is running on
 #PORTAGE_ELOG_MAILSUBJECT="[portage] ebuild log for \${PACKAGE} on \${HOST}"
+#
+# Variant for Prefix installations
+# ================================
+#
+#MAKEOPTS="-j2 TMPDIR=${EPREFIX}/tmp CONFIG_SHELL=${EPREFIX}/bin/sh SHELL=${EPREFIX}/bin/bash"
+#PORTAGE_PYTHON=$EPREFIX/usr/bin/python # e.g. for portage/bin/misc-functions.sh
+#PREFIX_DISABLE_GEN_USR_LDSCRIPT=set # e.g. for net-libs/libtirpc-0.2.2-r1.ebuild
+#FEATURES="uncooperative-root"
+#USE="uncooperative-root"
+# 
+#     has uncooperative-root ${FEATURES}
+# is a condition meaning that /tmp, /bin/bash,
+# /bin/sh, /dev, /usr/bin/{env,find,file} may not be available at buiding time
+# and despite being in various standards (FHS, ...) root administrator will not
+# cooperate or will break compatibility without notice. They are to be used now
+# in .eclass and in sys-app/portage (typical example: Android; needed for
+# gentooandroid.sourceforge.net).
+# It will likely be used in any *.ebuild file whose EAPI officially contains
+# SUPPORTED_FEATURES=uncooperative-root.
+# 
+#     use uncooperative-root
+# is the equivalent condition to use in files *.ebuild whose EAPI doesn't
+# officially contain SUPPORTED_FEATURES=uncooperative-root.  To enable that
+# condition, a file *.ebuild not inherite-ing anything should contain
+# IUSE=-uncooperative-root for that (otherwise, prefix.eclass will have
+# IUSE_IMPLICIT=uncooperative-root even if not directly inherited).
diff -ru old_portage/bin/ebuild-helpers/emake portage/bin/ebuild-helpers/emake
--- old_portage/bin/ebuild-helpers/emake	2014-07-04 16:17:50.847841909 +0200
+++ portage/bin/ebuild-helpers/emake	2014-07-04 16:30:26.020612108 +0200
@@ -11,6 +11,14 @@
 
 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh
 
+#${MAKEOPTS} is not transmitted to things called by make
+has uncooperative-root ${FEATURES} && { 
+  export TMPDIR="${EPREFIX}"/tmp
+  export BASH="${EPREFIX}"/bin/bash
+  export CONFIG_SHELL="${EPREFIX}"/bin/sh
+}
+
+
 if [[ $PORTAGE_QUIET != 1 ]] ; then
 	(
 	for arg in ${MAKE:-make} $MAKEOPTS $EXTRA_EMAKE "$@" ; do
diff -ru old_portage/bin/ebuild.sh portage/bin/ebuild.sh
--- old_portage/bin/ebuild.sh	2014-07-04 15:59:24.022279274 +0200
+++ portage/bin/ebuild.sh	2014-07-04 16:01:08.613886159 +0200
@@ -90,8 +90,13 @@
 __qa_call() {
 	local shopts=$(shopt) OLDIFS="$IFS"
 	local retval
-	"$@"
-	retval=$?
+        if has uncooperative-root ${FEATURES} ; then
+		uncooperative-root-wrapper "$@" # from prefix.eclass
+		retval=$?
+	else
+		"$@"
+		retval=$?
+	fi
 	set +e
 	[[ $shopts != $(shopt) ]] &&
 		eqawarn "QA Notice: Global shell options changed and were not restored while calling '$*'"
diff -ru old_portage/bin/misc-functions.sh portage/bin/misc-functions.sh
--- old_portage/bin/misc-functions.sh	2014-07-04 16:03:59.416144645 +0200
+++ portage/bin/misc-functions.sh	2014-07-04 16:57:13.327124754 +0200
@@ -820,7 +820,7 @@
 	rm -f "${ED}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!"
 
 	if has multilib-strict ${FEATURES} && \
-	   [[ -x /usr/bin/file && -x /usr/bin/find ]] && \
+	   [[ -x "`has uncooperative-root ${FEATURES} && echo ${EPREFIX}`/usr/bin/file" && -x "`has uncooperative-root ${FEATURES} && echo ${EPREFIX}`/usr/bin/find" ]] && \
 	   [[ -n ${MULTILIB_STRICT_DIRS} && -n ${MULTILIB_STRICT_DENY} ]]
 	then
 		rm -f "${T}/multilib-strict.log"
@@ -1193,6 +1193,10 @@
 	fi
 	[ -n "${md5_hash}" ] && \
 		echo ${md5_hash} > "${PORTAGE_BUILDDIR}"/build-info/BINPKGMD5
+  	# inform about trial patching done by uncooperative-root
+  	has uncooperative-root ${FEATURES} && bzip2 -c \
+  	  < "${PORTAGE_BUILDDIR}"/temp/build.log \
+  	  > "${PORTAGE_BUILDDIR}"/build-info/build.log.bz2
 	__vecho ">>> Done."
 
 	# cleanup our temp tree
diff -ru old_portage/bin/phase-helpers.sh portage/bin/phase-helpers.sh
--- old_portage/bin/phase-helpers.sh	2014-07-04 16:27:01.655594020 +0200
+++ portage/bin/phase-helpers.sh	2014-07-04 16:27:22.888250815 +0200
@@ -471,6 +471,10 @@
 	local x
 	local pid=${BASHPID:-$(__bashpid)}
 
+	has uncooperative-root ${FEATURES} && { 
+		export TMPDIR="${EPREFIX}"/tmp
+		export CONFIG_SHELL="${EPREFIX}"/bin/sh
+	}
 	if ! ___eapi_has_prefix_variables; then
 		local EPREFIX=
 	fi
@@ -592,6 +596,7 @@
 
 einstall() {
 	# CONF_PREFIX is only set if they didn't pass in libdir above.
+	has uncooperative-root ${FEATURES} && export TMPDIR="${EPREFIX}"/tmp
 	local LOCAL_EXTRA_EINSTALL="${EXTRA_EINSTALL}"
 	if ! ___eapi_has_prefix_variables; then
 		local ED=${D}

Re: [gentoo-portage-dev] Proposal of "uncooperative-root" in SUPPORTED_FEATURES

[Greg Turner]

[-- Attachment #1: Type: text/plain, Size: 4559 bytes --]

For a while my cygwin port needed something like this.  The problem was,
as, no doubt, the gentleman with the ridiculous name is all too familiar,
that you are fighting against a very long-standing convention.  Anything
based on autotools tends to be a serious disaster, for example.

There was another problem -- more of an institutional/human-nature thing
than a technical problem.  When I was on this same mission as you,
silly-name, it quickly became clear that people didn't give a shit and
didn't want those patches.

I hope those of you who I'm referring to won't be offended by that -- I'm
not making a normative statement here, just what I believe to be a factual
one.  Indeed there were many good reasons for them to feel that way,
including:

1. They (the people who didn't give a shit) didn't share my problem or
understand why I should have it.  The explanation, in my case, was
nontrivial.

2. The result of my approach was to greatly enlarge many core-system
ebuilds with huge patches to the upstream code; those evaluating those
patches would have been (I don't think I ever actually submitted them) the
same whose responsibility it would have become to maintain them.

In other words, my patches would have amounted to a request to sign the
decision-maker up for a bunch of ongoing busy-work, which they had no
personal stake in.

3. The number of people needing this feature would most likely have been
very small (of course, arguably, that number could have grown, once the
thing started to get off the ground.  But that still left me with a
chicken-and-egg problem even if it solved the justification problem).

4. It made alot of "tidy" ebuilds into "untidy" ebuilds.  This creates a
very real aesthetic problem and disinclines people to look on the
proposition favorably, myself included.  Humans like their technology to
look pretty.

I eventually found other ways to deal with it, and was able to drop those
patches from my tree.  But I lost many days of work screwing around with it.

I guess my only advice to you is, I strongly suggest you find ways to solve
this problem abstractly, rather than at an ebuild level, wherever possible.

One way you might look at is LD_PRELOAD trickery, allowing you to simply
lie about /bin/{ba,}sh and some other files' contents.  That probably
wouldn't have worked in my case, but I suspect it might in yours.  If that
won't work, then I guess my next idea would be to maintain a library of
code-injection hacks that could automatically fix those out-of-prefix
references in scripts and C code.

Abstract approaches at least stand a chance by avoiding problems #2 and #4,
which, I suspect, would be the big ones if you ever intend to upstream this
feature.  As for the idea of maintaining a huge overlay... sure, go for it,
but be warned: it's really hard.

-gmt

On Sat, Jul 5, 2014 at 1:24 AM, X dej <
dreplaceelettereejbyeletterea@gmail.com> wrote:

> Hello to all,
>
> Summary: I have made many mods to sys-app/portage (2013 version). The
> result is
> a proposal of patch for latest sys-app/portage, see below.
>
> Main result of my mods to gentoo prefix:
>
>   FEATURES=uncooperative-root emerge -bv =firefox-17
>
> now works on EPREFIX on Android-4.1 ARM. I use neither fakeroot nor
> GNURoot.
> root access never required. Distributed as a .tbz2 and a .ebuild by
> gentooandroid.sf.net, along with the necessary stage3.
>
> @dol-sen advised me one week ago on #gentoo-portage to post a rebased
> patch for
> HEAD of git.overlays.gentoo.org/proj/portage.git : I just hosted this
> patch on
>
> sf.net/projects/gentooandroid/files/sys-app_portage-current-HEAD_patch/download
> and it is also attached to this post, and to previous post on gentoo-dev
> mailing list; Joshua Kinard from there just advised me to post this patch
> at
> gentoo-portage-dev (here).
>
> I request comments and discussions about this patch. Summary:
>
> pym/portage/const.py     + (this is SUPPORTED_FEATURES+=uncooperative-root)
> cnf/make.conf.example    ++++++++++++++++++++++++++
> bin/ebuild-helpers/emake ++++++++
> bin/ebuild.sh            --+++++++
> bin/misc-functions.sh    -+++++
> bin/phase-helpers.sh     +++++
>
> My aim is to take your remarks into account, then to build an overlay
> (will be
> hosted at endrood.sf.net) with up-to-date versions of all my mods. The
> final
> result will be a tested version of above patch, together with lastest
> builds
> for firefox, squeak, scala, gnucash, ...
>
> Thanks for you attention.
>
> Xdej (I disclosed my real identity to Rick "Zero_Chaos" Farina).
>

[-- Attachment #2: Type: text/html, Size: 5764 bytes --]

Re: [gentoo-portage-dev] Proposal of "uncooperative-root" in SUPPORTED_FEATURES

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 6753 bytes --]

Dnia 2014-07-05, o godz. 10:24:03
X dej <dreplaceelettereejbyeletterea@gmail.com> napisał(a):

> Patch to apply against HEAD of sys-app/portage project.
> I did git clone git://git.overlays.gentoo.org/proj/portage.git on 2014-07-04
> 13:47:53.442947599 +0000.
> 
> diff -ru old_portage/pym/portage/const.py portage/pym/portage/const.py
> --- old_portage/pym/portage/const.py	2014-07-04 16:35:37.462942289 +0200
> +++ portage/pym/portage/const.py	2014-07-04 17:03:00.347420753 +0200
> @@ -119,6 +119,7 @@
>  	"other",
>  )
>  SUPPORTED_FEATURES       = frozenset([
> +	"uncooperative-root",

I don't want to nitpick but the name seems very unclear. The feature
name should tell what it does or enables.

>  	"assume-digests",
>  	"binpkg-logs",
>  	"buildpkg",
> diff -ru old_portage/cnf/make.conf.example portage/cnf/make.conf.example
> --- old_portage/cnf/make.conf.example	2014-07-04 16:52:24.438859863 +0200
> +++ portage/cnf/make.conf.example	2014-07-04 17:08:10.658545393 +0200
> @@ -362,3 +362,29 @@
>  #                               ${PACKAGE} - see description of PORTAGE_ELOG_COMMAND
>  #                               ${HOST} - FQDN of the host portage is running on
>  #PORTAGE_ELOG_MAILSUBJECT="[portage] ebuild log for \${PACKAGE} on \${HOST}"
> +#
> +# Variant for Prefix installations
> +# ================================
> +#
> +#MAKEOPTS="-j2 TMPDIR=${EPREFIX}/tmp CONFIG_SHELL=${EPREFIX}/bin/sh SHELL=${EPREFIX}/bin/bash"
> +#PORTAGE_PYTHON=$EPREFIX/usr/bin/python # e.g. for portage/bin/misc-functions.sh
> +#PREFIX_DISABLE_GEN_USR_LDSCRIPT=set # e.g. for net-libs/libtirpc-0.2.2-r1.ebuild
> +#FEATURES="uncooperative-root"
> +#USE="uncooperative-root"

A general remark: wouldn't it be better to substitute this to the main
part of make.conf.example when 'use prefix' is in use?

> +# 
> +#     has uncooperative-root ${FEATURES}
> +# is a condition meaning that /tmp, /bin/bash,
> +# /bin/sh, /dev, /usr/bin/{env,find,file} may not be available at buiding time
> +# and despite being in various standards (FHS, ...) root administrator will not
> +# cooperate or will break compatibility without notice. They are to be used now
> +# in .eclass and in sys-app/portage (typical example: Android; needed for
> +# gentooandroid.sourceforge.net).
> +# It will likely be used in any *.ebuild file whose EAPI officially contains
> +# SUPPORTED_FEATURES=uncooperative-root.
> +# 
> +#     use uncooperative-root
> +# is the equivalent condition to use in files *.ebuild whose EAPI doesn't
> +# officially contain SUPPORTED_FEATURES=uncooperative-root.  To enable that
> +# condition, a file *.ebuild not inherite-ing anything should contain
> +# IUSE=-uncooperative-root for that (otherwise, prefix.eclass will have
> +# IUSE_IMPLICIT=uncooperative-root even if not directly inherited).

This doesn't belong in make.conf.example but rather in the manpage.
Which is a good point to note that you haven't added the description of
the new FEATURE to make.conf(5).

> diff -ru old_portage/bin/ebuild-helpers/emake portage/bin/ebuild-helpers/emake
> --- old_portage/bin/ebuild-helpers/emake	2014-07-04 16:17:50.847841909 +0200
> +++ portage/bin/ebuild-helpers/emake	2014-07-04 16:30:26.020612108 +0200
> @@ -11,6 +11,14 @@
>  
>  source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh
>  
> +#${MAKEOPTS} is not transmitted to things called by make
> +has uncooperative-root ${FEATURES} && { 
> +  export TMPDIR="${EPREFIX}"/tmp

Isn't TMPDIR already set by portage? PMS says it ought to be.

> +  export BASH="${EPREFIX}"/bin/bash
> +  export CONFIG_SHELL="${EPREFIX}"/bin/sh
> +}

If you export it all anyway... what's the point of passing them via
MAKEOPTS?

> +
> +
>  if [[ $PORTAGE_QUIET != 1 ]] ; then
>  	(
>  	for arg in ${MAKE:-make} $MAKEOPTS $EXTRA_EMAKE "$@" ; do
> diff -ru old_portage/bin/ebuild.sh portage/bin/ebuild.sh
> --- old_portage/bin/ebuild.sh	2014-07-04 15:59:24.022279274 +0200
> +++ portage/bin/ebuild.sh	2014-07-04 16:01:08.613886159 +0200
> @@ -90,8 +90,13 @@
>  __qa_call() {
>  	local shopts=$(shopt) OLDIFS="$IFS"
>  	local retval
> -	"$@"
> -	retval=$?
> +        if has uncooperative-root ${FEATURES} ; then
> +		uncooperative-root-wrapper "$@" # from prefix.eclass

I don't think it is a good idea to call eclass functions from within
portage.

> +		retval=$?
> +	else
> +		"$@"
> +		retval=$?
> +	fi
>  	set +e
>  	[[ $shopts != $(shopt) ]] &&
>  		eqawarn "QA Notice: Global shell options changed and were not restored while calling '$*'"
> diff -ru old_portage/bin/misc-functions.sh portage/bin/misc-functions.sh
> --- old_portage/bin/misc-functions.sh	2014-07-04 16:03:59.416144645 +0200
> +++ portage/bin/misc-functions.sh	2014-07-04 16:57:13.327124754 +0200
> @@ -820,7 +820,7 @@
>  	rm -f "${ED}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!"
>  
>  	if has multilib-strict ${FEATURES} && \
> -	   [[ -x /usr/bin/file && -x /usr/bin/find ]] && \
> +	   [[ -x "`has uncooperative-root ${FEATURES} && echo ${EPREFIX}`/usr/bin/file" && -x "`has uncooperative-root ${FEATURES} && echo ${EPREFIX}`/usr/bin/find" ]] && \

Please use $() instead of `` because the latter is hard to read
and doesn't support nesting.

Doesn't this also need to consider ${ROOT}?

>  	   [[ -n ${MULTILIB_STRICT_DIRS} && -n ${MULTILIB_STRICT_DENY} ]]
>  	then
>  		rm -f "${T}/multilib-strict.log"
> @@ -1193,6 +1193,10 @@
>  	fi
>  	[ -n "${md5_hash}" ] && \
>  		echo ${md5_hash} > "${PORTAGE_BUILDDIR}"/build-info/BINPKGMD5
> +  	# inform about trial patching done by uncooperative-root
> +  	has uncooperative-root ${FEATURES} && bzip2 -c \
> +  	  < "${PORTAGE_BUILDDIR}"/temp/build.log \
> +  	  > "${PORTAGE_BUILDDIR}"/build-info/build.log.bz2
>  	__vecho ">>> Done."
>  
>  	# cleanup our temp tree
> diff -ru old_portage/bin/phase-helpers.sh portage/bin/phase-helpers.sh
> --- old_portage/bin/phase-helpers.sh	2014-07-04 16:27:01.655594020 +0200
> +++ portage/bin/phase-helpers.sh	2014-07-04 16:27:22.888250815 +0200
> @@ -471,6 +471,10 @@
>  	local x
>  	local pid=${BASHPID:-$(__bashpid)}
>  
> +	has uncooperative-root ${FEATURES} && { 
> +		export TMPDIR="${EPREFIX}"/tmp
> +		export CONFIG_SHELL="${EPREFIX}"/bin/sh
> +	}
>  	if ! ___eapi_has_prefix_variables; then
>  		local EPREFIX=
>  	fi
> @@ -592,6 +596,7 @@
>  
>  einstall() {
>  	# CONF_PREFIX is only set if they didn't pass in libdir above.
> +	has uncooperative-root ${FEATURES} && export TMPDIR="${EPREFIX}"/tmp
>  	local LOCAL_EXTRA_EINSTALL="${EXTRA_EINSTALL}"
>  	if ! ___eapi_has_prefix_variables; then
>  		local ED=${D}



-- 
Best regards,
Michał Górny

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 949 bytes --]