[gentoo-dev] unpacker.eclass

[gentoo-dev] unpacker.eclass

[Mike Frysinger]

[-- Attachment #1: Type: text/plain, Size: 10262 bytes --]

any feedback before merging this initial version ?
	https://bugs.gentoo.org/399019
-mike

# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: /var/cvsroot/gentoo-x86/eclass/eutils.eclass,v 1.377 2012/01/03 08:45:36 jlec Exp $

# @ECLASS: unpacker.eclass
# @MAINTAINER:
# base-system@gentoo.org
# @BLURB: helpers for extraneous file formats and consistent behavior across EAPI's
# @DESCRIPTION:
# Some extraneous file formats are not part of PMS, or are only in certain
# EAPI's.  Rather than worrying about that, support the crazy cruft here
# and for all EAPI versions.

# Possible todos:
#  - merge rpm unpacking
#  - support partial unpacks?

if [[ ${___ECLASS_ONCE_UNPACKER} != "recur -_+^+_- spank" ]] ; then
___ECLASS_ONCE_UNPACKER="recur -_+^+_- spank"

# @ECLASS-VARIABLE: UNPACKER_BZ2
# @DEFAULT_UNSET
# @DESCRIPTION:
# Utility to use to decompress bzip2 files.  Will dynamically pick between
# `pbzip2` and `bzip2`.  Make sure your choice accepts the "-c" option.
# Note: this is meant for users to set, not ebuilds.

# for internal use only (unpack_pdv and unpack_makeself)
find_unpackable_file() {
	local src=$1
	if [[ -z ${src} ]] ; then
		src=${DISTDIR}/${A}
	else
		if [[ ${src} == ./* ]] ; then
			: # already what we want
		elif [[ -e ${DISTDIR}/${src} ]] ; then
			src=${DISTDIR}/${src}
		elif [[ -e ${PWD}/${src} ]] ; then
			src=${PWD}/${src}
		elif [[ -e ${src} ]] ; then
			src=${src}
		fi
	fi
	[[ ! -e ${src} ]] && return 1
	echo "${src}"
}

unpack_banner() {
	echo ">>> Unpacking ${1##*/} to ${PWD}"
}

# @FUNCTION: unpack_pdv
# @USAGE: <file to unpack> <size of off_t>
# @DESCRIPTION:
# Unpack those pesky pdv generated files ...
# They're self-unpacking programs with the binary package stuffed in
# the middle of the archive.  Valve seems to use it a lot ... too bad
# it seems to like to segfault a lot :(.  So lets take it apart ourselves.
#
# You have to specify the off_t size ... I have no idea how to extract that
# information out of the binary executable myself.  Basically you pass in
# the size of the off_t type (in bytes) on the machine that built the pdv
# archive.
#
# One way to determine this is by running the following commands:
#
# @CODE
# 	strings <pdv archive> | grep lseek
# 	strace -elseek <pdv archive>
# @CODE
#
# Basically look for the first lseek command (we do the strings/grep because
# sometimes the function call is _llseek or something) and steal the 2nd
# parameter.  Here is an example:
#
# @CODE
# 	vapier@vapier 0 pdv_unpack # strings hldsupdatetool.bin | grep lseek
# 	lseek
# 	vapier@vapier 0 pdv_unpack # strace -elseek ./hldsupdatetool.bin
# 	lseek(3, -4, SEEK_END)					= 2981250
# @CODE
#
# Thus we would pass in the value of '4' as the second parameter.
unpack_pdv() {
	local src=$(find_unpackable_file "$1")
	local sizeoff_t=$2

	[[ -z ${src} ]] && die "Could not locate source for '$1'"
	[[ -z ${sizeoff_t} ]] && die "No idea what off_t size was used for this pdv :("

	unpack_banner "${src}"

	local metaskip=$(tail -c ${sizeoff_t} "${src}" | hexdump -e \"%i\")
	local tailskip=$(tail -c $((${sizeoff_t}*2)) "${src}" | head -c ${sizeoff_t} | hexdump -e \"%i\")

	# grab metadata for debug reasons
	local metafile=$(emktemp)
	tail -c +$((${metaskip}+1)) "${src}" > "${metafile}"

	# rip out the final file name from the metadata
	local datafile=$(tail -c +$((${metaskip}+1)) "${src}" | strings | head -n 1)
	datafile=$(basename "${datafile}")

	# now lets uncompress/untar the file if need be
	local tmpfile=$(emktemp)
	tail -c +$((${tailskip}+1)) ${src} 2>/dev/null | head -c 512 > ${tmpfile}

	local iscompressed=$(file -b "${tmpfile}")
	if [[ ${iscompressed:0:8} == "compress" ]] ; then
		iscompressed=1
		mv ${tmpfile}{,.Z}
		gunzip ${tmpfile}
	else
		iscompressed=0
	fi
	local istar=$(file -b "${tmpfile}")
	if [[ ${istar:0:9} == "POSIX tar" ]] ; then
		istar=1
	else
		istar=0
	fi

	#for some reason gzip dies with this ... dd cant provide buffer fast enough ?
	#dd if=${src} ibs=${metaskip} count=1 \
	#	| dd ibs=${tailskip} skip=1 \
	#	| gzip -dc \
	#	> ${datafile}
	if [ ${iscompressed} -eq 1 ] ; then
		if [ ${istar} -eq 1 ] ; then
			tail -c +$((${tailskip}+1)) ${src} 2>/dev/null \
				| head -c $((${metaskip}-${tailskip})) \
				| tar -xzf -
		else
			tail -c +$((${tailskip}+1)) ${src} 2>/dev/null \
				| head -c $((${metaskip}-${tailskip})) \
				| gzip -dc \
				> ${datafile}
		fi
	else
		if [ ${istar} -eq 1 ] ; then
			tail -c +$((${tailskip}+1)) ${src} 2>/dev/null \
				| head -c $((${metaskip}-${tailskip})) \
				| tar --no-same-owner -xf -
		else
			tail -c +$((${tailskip}+1)) ${src} 2>/dev/null \
				| head -c $((${metaskip}-${tailskip})) \
				> ${datafile}
		fi
	fi
	true
	#[ -s "${datafile}" ] || die "failure unpacking pdv ('${metaskip}' '${tailskip}' '${datafile}')"
	#assert "failure unpacking pdv ('${metaskip}' '${tailskip}' '${datafile}')"
}

# @FUNCTION: unpack_makeself
# @USAGE: [file to unpack] [offset] [tail|dd]
# @DESCRIPTION:
# Unpack those pesky makeself generated files ...
# They're shell scripts with the binary package tagged onto
# the end of the archive.  Loki utilized the format as does
# many other game companies.
#
# If the file is not specified, then ${A} is used.  If the
# offset is not specified then we will attempt to extract
# the proper offset from the script itself.
unpack_makeself() {
	local src_input=${1:-${A}}
	local src=$(find_unpackable_file "${src_input}")
	local skip=$2
	local exe=$3

	[[ -z ${src} ]] && die "Could not locate source for '${src_input}'"

	unpack_banner "${src}"

	if [[ -z ${skip} ]] ; then
		local ver=$(grep -m1 -a '#.*Makeself' "${src}" | awk '{print $NF}')
		local skip=0
		exe=tail
		case ${ver} in
			1.5.*|1.6.0-nv)	# tested 1.5.{3,4,5} ... guessing 1.5.x series is same
				skip=$(grep -a ^skip= "${src}" | cut -d= -f2)
				;;
			2.0|2.0.1)
				skip=$(grep -a ^$'\t'tail "${src}" | awk '{print $2}' | cut -b2-)
				;;
			2.1.1)
				skip=$(grep -a ^offset= "${src}" | awk '{print $2}' | cut -b2-)
				(( skip++ ))
				;;
			2.1.2)
				skip=$(grep -a ^offset= "${src}" | awk '{print $3}' | head -n 1)
				(( skip++ ))
				;;
			2.1.3)
				skip=`grep -a ^offset= "${src}" | awk '{print $3}'`
				(( skip++ ))
				;;
			2.1.4|2.1.5)
				skip=$(grep -a offset=.*head.*wc "${src}" | awk '{print $3}' | head -n 1)
				skip=$(head -n ${skip} "${src}" | wc -c)
				exe="dd"
				;;
			*)
				eerror "I'm sorry, but I was unable to support the Makeself file."
				eerror "The version I detected was '${ver}'."
				eerror "Please file a bug about the file ${src##*/} at"
				eerror "http://bugs.gentoo.org/ so that support can be added."
				die "makeself version '${ver}' not supported"
				;;
		esac
		debug-print "Detected Makeself version ${ver} ... using ${skip} as offset"
	fi
	case ${exe} in
		tail)	exe="tail -n +${skip} '${src}'";;
		dd)		exe="dd ibs=${skip} skip=1 if='${src}'";;
		*)		die "makeself cant handle exe '${exe}'"
	esac

	# lets grab the first few bytes of the file to figure out what kind of archive it is
	local filetype tmpfile=$(emktemp)
	eval ${exe} 2>/dev/null | head -c 512 > "${tmpfile}"
	filetype=$(file -b "${tmpfile}") || die
	case ${filetype} in
		*tar\ archive*)
			eval ${exe} | tar --no-same-owner -xf -
			;;
		bzip2*)
			eval ${exe} | bzip2 -dc | tar --no-same-owner -xf -
			;;
		gzip*)
			eval ${exe} | tar --no-same-owner -xzf -
			;;
		compress*)
			eval ${exe} | gunzip | tar --no-same-owner -xf -
			;;
		*)
			eerror "Unknown filetype \"${filetype}\" ?"
			false
			;;
	esac
	assert "failure unpacking (${filetype}) makeself ${src##*/} ('${ver}' +${skip})"
}

# @FUNCTION: unpack_deb
# @USAGE: <one deb to unpack>
# @DESCRIPTION:
# Unpack a Debian .deb archive in style.
unpack_deb() {
	[[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>"

	local deb=$(find_unpackable_file "$1")

	unpack_banner "${deb}"

	ar x "${deb}"
	unpack ./data.tar*
}

# @FUNCTION: _unpacker
# @USAGE: <one archive to unpack>
# @INTERNAL
# @DESCRIPTION:
# Unpack the specified archive.  We only operate on one archive here
# to keep down on the looping logic (that is handled by `unpacker`).
_unpacker() {
	[[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>"

	local a=$1
	local m=$(echo "${a}" | tr '[:upper:]' '[:lower:]')
	a=$(find_unpackable_file "${a}")

	# first figure out the decompression method
	case ${m} in
	*.bz2|*.tbz|*.tbz2)
		: ${PORTAGE_BZIP2_COMMAND:=$(type -P pbzip2 || bzip2)}
		: ${PORTAGE_BUNZIP2_COMMAND:=${PORTAGE_BZIP2_COMMAND} -d}
		: ${UNPACKER_BZ2:=${PORTAGE_BUNZIP2_COMMAND}}
		comp="${PORTAGE_BUNZIP2_COMMAND} -c"
		;;
	*.z|*.gz|*.tgz)
		comp="gzip -dc" ;;
	*.lzma|*.xz|*.txz)
		comp="xz -dc" ;;
	*)	comp="" ;;
	esac

	# then figure out if there are any archiving aspects
	case ${m} in
	*.tgz|*.tbz|*.tbz2|*.txz|*.tar.*|*.tar)
		arch="tar --no-same-owner -xof" ;;
	*.deb)
		arch="unpack_deb" ;;
	*.run)
		arch="unpack_makeself" ;;
	*) arch="" ;;
	esac

	# finally do the unpack
	[[ -z ${arch}${comp} ]] && return 1

	[[ ${arch} != unpack_* ]] && unpack_banner "${a}"

	if [[ -z ${arch} ]] ; then
		${comp} "${a}" > "${a%.*}"
	elif [[ -z ${comp} ]] ; then
		${arch} "${a}"
	else
		${comp} "${a}" | ${arch} -
	fi

	assert "unpacking ${a} failed (comp=${comp} arch=${arch})"
}

# @FUNCTION: unpacker
# @USAGE: [archives to unpack]
# @DESCRIPTION:
# This works in the same way that `unpack` does.  If you don't specify
# any files, it will default to ${A}.
unpacker() {
	local a
	[[ $# -eq 0 ]] && set -- ${A}
	for a ; do _unpacker "${a}" ; done
}

# @FUNCTION: unpacker_src_unpack
# @DESCRIPTION:
# Run `unpacker` to unpack all our stuff.
unpacker_src_unpack() {
	unpacker
}

# @FUNCTION: unpacker
# @USAGE: [archives that we will unpack]
# @RETURN: Dependencies needed to unpack all the archives
# @DESCRIPTION:
# Walk all the specified files (defaults to $SRC_URI) and figure out the
# dependencies that are needed to unpack things.
#
# Note: USE flags are not yet handled.
unpacker_src_uri_depends() {
	local uri deps d

	[[ $# -eq 0 ]] && set -- ${SRC_URI}

	for uri in "$@" ; do
		case ${uri} in
		*.rar|*.RAR)
			d="app-arch/unrar" ;;
		*.7z)
			d="app-arch/p7zip" ;;
		*.xz)
			d="app-arch/xz-utils" ;;
		esac
		deps+=" ${d}"
	done

	echo "${deps}"
}

EXPORT_FUNCTIONS src_unpack

fi

[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 2137 bytes --]

On Wed, 1 Feb 2012 15:05:40 -0500
Mike Frysinger <vapier@gentoo.org> wrote:

> # You have to specify the off_t size ... I have no idea how to
> extract that # information out of the binary executable myself.
> Basically you pass in # the size of the off_t type (in bytes) on the
> machine that built the pdv # archive.

Can't you use 'file' to determine the host type and just assume off_t
for it?

> # @FUNCTION: unpack_makeself
> # @USAGE: [file to unpack] [offset] [tail|dd]
> # @DESCRIPTION:
> # Unpack those pesky makeself generated files ...
> # They're shell scripts with the binary package tagged onto
> # the end of the archive.  Loki utilized the format as does
> # many other game companies.
> #
> # If the file is not specified, then ${A} is used.  If the
> # offset is not specified then we will attempt to extract
> # the proper offset from the script itself.

The third argument is not explained.

> unpack_makeself() {
> 	local src_input=${1:-${A}}

What if ${A} contains more than a single file? Well, what is this for
anyway?

> 	case ${exe} in
> 		tail)	exe="tail -n +${skip} '${src}'";;
> 		dd)		exe="dd ibs=${skip} skip=1
> if='${src}'";; *)		die "makeself cant handle exe
> '${exe}'" esac

What is the use of that?

> # @FUNCTION: unpacker

Wrong name.

> # @USAGE: [archives that we will unpack]
> # @RETURN: Dependencies needed to unpack all the archives
> # @DESCRIPTION:
> # Walk all the specified files (defaults to $SRC_URI) and figure out
> the # dependencies that are needed to unpack things.
> #
> # Note: USE flags are not yet handled.
> unpacker_src_uri_depends() {
> 	local uri deps d
> 
> 	[[ $# -eq 0 ]] && set -- ${SRC_URI}
> 
> 	for uri in "$@" ; do
> 		case ${uri} in
> 		*.rar|*.RAR)
> 			d="app-arch/unrar" ;;
> 		*.7z)
> 			d="app-arch/p7zip" ;;

Where are those file formats handled? You don't seem to fallback to
'unpack' anywhere too.

And I think you should consider using 'file --mime' rather than
relying on format descriptions not ever changing/differing due to
subtle differences.

-- 
Best regards,
Michał Górny

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 316 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Mike Frysinger]

[-- Attachment #1: Type: Text/Plain, Size: 1647 bytes --]

On Wednesday 01 February 2012 15:30:16 Michał Górny wrote:
> On Wed, 1 Feb 2012 15:05:40 -0500 Mike Frysinger wrote:
> > # You have to specify the off_t size ... I have no idea how to
> > extract that # information out of the binary executable myself.
> > Basically you pass in # the size of the off_t type (in bytes) on the
> > machine that built the pdv # archive.
> 
> Can't you use 'file' to determine the host type and just assume off_t
> for it?

i'm not looking for feedback on the unpack_{makeself,pdv} at this point in 
time.  i'll look after the eutils->unpacker migration is done.

> > # @FUNCTION: unpacker
> 
> Wrong name.

fixed

> > # @USAGE: [archives that we will unpack]
> > # @RETURN: Dependencies needed to unpack all the archives
> > # @DESCRIPTION:
> > # Walk all the specified files (defaults to $SRC_URI) and figure out
> > the # dependencies that are needed to unpack things.
> > #
> > # Note: USE flags are not yet handled.
> > unpacker_src_uri_depends() {
> > 
> > 	local uri deps d
> > 	
> > 	[[ $# -eq 0 ]] && set -- ${SRC_URI}
> > 	
> > 	for uri in "$@" ; do
> > 	
> > 		case ${uri} in
> > 		*.rar|*.RAR)
> > 		
> > 			d="app-arch/unrar" ;;
> > 		
> > 		*.7z)
> > 		
> > 			d="app-arch/p7zip" ;;
> 
> Where are those file formats handled? You don't seem to fallback to
> 'unpack' anywhere too.

eh ?  this func doesn't do unpacking, just ${SRC_URI}<->${DEPEND} matching.

> And I think you should consider using 'file --mime' rather than
> relying on format descriptions not ever changing/differing due to
> subtle differences.

probably worth looking at
-mike

[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 1105 bytes --]

On Wed, 1 Feb 2012 15:44:14 -0500
Mike Frysinger <vapier@gentoo.org> wrote:

> > > # @USAGE: [archives that we will unpack]
> > > # @RETURN: Dependencies needed to unpack all the archives
> > > # @DESCRIPTION:
> > > # Walk all the specified files (defaults to $SRC_URI) and figure
> > > out the # dependencies that are needed to unpack things.
> > > #
> > > # Note: USE flags are not yet handled.
> > > unpacker_src_uri_depends() {
> > > 
> > > 	local uri deps d
> > > 	
> > > 	[[ $# -eq 0 ]] && set -- ${SRC_URI}
> > > 	
> > > 	for uri in "$@" ; do
> > > 	
> > > 		case ${uri} in
> > > 		*.rar|*.RAR)
> > > 		
> > > 			d="app-arch/unrar" ;;
> > > 		
> > > 		*.7z)
> > > 		
> > > 			d="app-arch/p7zip" ;;
> > 
> > Where are those file formats handled? You don't seem to fallback to
> > 'unpack' anywhere too.
> 
> eh ?  this func doesn't do unpacking, just ${SRC_URI}<->${DEPEND}
> matching.

Sooo.. it's intended to generate an useless DEPEND or you have to
reset src_unpack() to default to make the archives actually extractable.

-- 
Best regards,
Michał Górny

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 316 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Mike Frysinger]

[-- Attachment #1: Type: Text/Plain, Size: 1402 bytes --]

On Wednesday 01 February 2012 15:51:52 Michał Górny wrote:
> On Wed, 1 Feb 2012 15:44:14 -0500 Mike Frysinger wrote:
> > > > # @USAGE: [archives that we will unpack]
> > > > # @RETURN: Dependencies needed to unpack all the archives
> > > > # @DESCRIPTION:
> > > > # Walk all the specified files (defaults to $SRC_URI) and figure
> > > > out the # dependencies that are needed to unpack things.
> > > > #
> > > > # Note: USE flags are not yet handled.
> > > > unpacker_src_uri_depends() {
> > > > 
> > > > 	local uri deps d
> > > > 	
> > > > 	[[ $# -eq 0 ]] && set -- ${SRC_URI}
> > > > 	
> > > > 	for uri in "$@" ; do
> > > > 	
> > > > 		case ${uri} in
> > > > 		*.rar|*.RAR)
> > > > 		
> > > > 			d="app-arch/unrar" ;;
> > > > 		
> > > > 		*.7z)
> > > > 		
> > > > 			d="app-arch/p7zip" ;;
> > > 
> > > Where are those file formats handled? You don't seem to fallback to
> > > 'unpack' anywhere too.
> > 
> > eh ?  this func doesn't do unpacking, just ${SRC_URI}<->${DEPEND}
> > matching.
> 
> Sooo.. it's intended to generate an useless DEPEND

the ebuild does:
	DEPEND+=" $(unpacker_src_uri_depends)"

> or you have to reset src_unpack() to default to make the archives actually
> extractable.

this func has nothing to do with extraction.  look at the rest of the code to 
see how the default src_unpack is handled via standard EXPORT_FUNC means.
-mike

[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Alexandre Rostovtsev]

On Wed, 2012-02-01 at 15:05 -0500, Mike Frysinger wrote:
> # @BLURB: helpers for extraneous file formats and consistent behavior across EAPI's
> # @DESCRIPTION:
> # Some extraneous file formats are not part of PMS, or are only in certain
> # EAPI's.  Rather than worrying about that, support the crazy cruft here

s/EAPI's/EAPIs/g

Plurals of acronyms are formed without using the apostrophe. E.g. see
http://grammar.about.com/od/punctuationandmechanics/tp/GuideApostrophe.htm or  http://owl.english.purdue.edu/owl/resource/621/01/

-Alexandre.

Re: [gentoo-dev] unpacker.eclass

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 1685 bytes --]

On Wed, 1 Feb 2012 15:55:46 -0500
Mike Frysinger <vapier@gentoo.org> wrote:

> On Wednesday 01 February 2012 15:51:52 Michał Górny wrote:
> > On Wed, 1 Feb 2012 15:44:14 -0500 Mike Frysinger wrote:
> > > > > # @USAGE: [archives that we will unpack]
> > > > > # @RETURN: Dependencies needed to unpack all the archives
> > > > > # @DESCRIPTION:
> > > > > # Walk all the specified files (defaults to $SRC_URI) and
> > > > > figure out the # dependencies that are needed to unpack
> > > > > things. #
> > > > > # Note: USE flags are not yet handled.
> > > > > unpacker_src_uri_depends() {
> > > > > 
> > > > > 	local uri deps d
> > > > > 	
> > > > > 	[[ $# -eq 0 ]] && set -- ${SRC_URI}
> > > > > 	
> > > > > 	for uri in "$@" ; do
> > > > > 	
> > > > > 		case ${uri} in
> > > > > 		*.rar|*.RAR)
> > > > > 		
> > > > > 			d="app-arch/unrar" ;;
> > > > > 		
> > > > > 		*.7z)
> > > > > 		
> > > > > 			d="app-arch/p7zip" ;;
> > > > 
> > > > Where are those file formats handled? You don't seem to
> > > > fallback to 'unpack' anywhere too.
> > > 
> > > eh ?  this func doesn't do unpacking, just ${SRC_URI}<->${DEPEND}
> > > matching.
> > 
> > Sooo.. it's intended to generate an useless DEPEND
> 
> the ebuild does:
> 	DEPEND+=" $(unpacker_src_uri_depends)"
> 
> > or you have to reset src_unpack() to default to make the archives
> > actually extractable.
> 
> this func has nothing to do with extraction.  look at the rest of the
> code to see how the default src_unpack is handled via standard
> EXPORT_FUNC means. -mike

Yes, and can that exported default src_unpack() extract either .rar
or .7z?

-- 
Best regards,
Michał Górny

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 316 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Mike Frysinger]

[-- Attachment #1: Type: Text/Plain, Size: 1965 bytes --]

On Wednesday 01 February 2012 18:12:02 Michał Górny wrote:
> On Wed, 1 Feb 2012 15:55:46 -0500 Mike Frysinger wrote:
> > On Wednesday 01 February 2012 15:51:52 Michał Górny wrote:
> > > On Wed, 1 Feb 2012 15:44:14 -0500 Mike Frysinger wrote:
> > > > > > # @USAGE: [archives that we will unpack]
> > > > > > # @RETURN: Dependencies needed to unpack all the archives
> > > > > > # @DESCRIPTION:
> > > > > > # Walk all the specified files (defaults to $SRC_URI) and
> > > > > > figure out the # dependencies that are needed to unpack
> > > > > > things. #
> > > > > > # Note: USE flags are not yet handled.
> > > > > > unpacker_src_uri_depends() {
> > > > > > 
> > > > > > 	local uri deps d
> > > > > > 	
> > > > > > 	[[ $# -eq 0 ]] && set -- ${SRC_URI}
> > > > > > 	
> > > > > > 	for uri in "$@" ; do
> > > > > > 	
> > > > > > 		case ${uri} in
> > > > > > 		*.rar|*.RAR)
> > > > > > 		
> > > > > > 			d="app-arch/unrar" ;;
> > > > > > 		
> > > > > > 		*.7z)
> > > > > > 		
> > > > > > 			d="app-arch/p7zip" ;;
> > > > > 
> > > > > Where are those file formats handled? You don't seem to
> > > > > fallback to 'unpack' anywhere too.
> > > > 
> > > > eh ?  this func doesn't do unpacking, just ${SRC_URI}<->${DEPEND}
> > > > matching.
> > > 
> > > Sooo.. it's intended to generate an useless DEPEND
> > 
> > the ebuild does:
> > 	DEPEND+=" $(unpacker_src_uri_depends)"
> > 	
> > > or you have to reset src_unpack() to default to make the archives
> > > actually extractable.
> > 
> > this func has nothing to do with extraction.  look at the rest of the
> > code to see how the default src_unpack is handled via standard
> > EXPORT_FUNC means.
> 
> Yes, and can that exported default src_unpack() extract either .rar
> or .7z?

there are no plans for that since portage handles it from EAPI=0 onwards.  i 
can have _unpacker() automatically tail off into `unpack` if it finds a file it 
doesn't recognize.
-mike

[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Michał Górny]

[-- Attachment #1: Type: text/plain, Size: 2351 bytes --]

On Wed, 1 Feb 2012 19:33:54 -0500
Mike Frysinger <vapier@gentoo.org> wrote:

> On Wednesday 01 February 2012 18:12:02 Michał Górny wrote:
> > On Wed, 1 Feb 2012 15:55:46 -0500 Mike Frysinger wrote:
> > > On Wednesday 01 February 2012 15:51:52 Michał Górny wrote:
> > > > On Wed, 1 Feb 2012 15:44:14 -0500 Mike Frysinger wrote:
> > > > > > > # @USAGE: [archives that we will unpack]
> > > > > > > # @RETURN: Dependencies needed to unpack all the archives
> > > > > > > # @DESCRIPTION:
> > > > > > > # Walk all the specified files (defaults to $SRC_URI) and
> > > > > > > figure out the # dependencies that are needed to unpack
> > > > > > > things. #
> > > > > > > # Note: USE flags are not yet handled.
> > > > > > > unpacker_src_uri_depends() {
> > > > > > > 
> > > > > > > 	local uri deps d
> > > > > > > 	
> > > > > > > 	[[ $# -eq 0 ]] && set -- ${SRC_URI}
> > > > > > > 	
> > > > > > > 	for uri in "$@" ; do
> > > > > > > 	
> > > > > > > 		case ${uri} in
> > > > > > > 		*.rar|*.RAR)
> > > > > > > 		
> > > > > > > 			d="app-arch/unrar" ;;
> > > > > > > 		
> > > > > > > 		*.7z)
> > > > > > > 		
> > > > > > > 			d="app-arch/p7zip" ;;
> > > > > > 
> > > > > > Where are those file formats handled? You don't seem to
> > > > > > fallback to 'unpack' anywhere too.
> > > > > 
> > > > > eh ?  this func doesn't do unpacking, just
> > > > > ${SRC_URI}<->${DEPEND} matching.
> > > > 
> > > > Sooo.. it's intended to generate an useless DEPEND
> > > 
> > > the ebuild does:
> > > 	DEPEND+=" $(unpacker_src_uri_depends)"
> > > 	
> > > > or you have to reset src_unpack() to default to make the
> > > > archives actually extractable.
> > > 
> > > this func has nothing to do with extraction.  look at the rest of
> > > the code to see how the default src_unpack is handled via standard
> > > EXPORT_FUNC means.
> > 
> > Yes, and can that exported default src_unpack() extract either .rar
> > or .7z?
> 
> there are no plans for that since portage handles it from EAPI=0
> onwards.  i can have _unpacker() automatically tail off into `unpack`
> if it finds a file it doesn't recognize.

Yes, you should do that. Otherwise, I don't see how people would
benefit from:

#v+
SRC_URI="foo-bar-baz.rar"

inherit unpacker

src_unpack() {
	default
}
#v-

-- 
Best regards,
Michał Górny

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 316 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Mike Frysinger]

[-- Attachment #1: Type: Text/Plain, Size: 159 bytes --]

On Thursday 02 February 2012 03:06:46 Michał Górny wrote:
> #v+
> SRC_URI="foo-bar-baz.rar"
> 
> inherit unpacker

inherit goes before SRC_URI
-mike

[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [gentoo-dev] unpacker.eclass

[Mike Frysinger]

[-- Attachment #1: Type: Text/Plain, Size: 10228 bytes --]

here's v2
-mike

# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: /var/cvsroot/gentoo-x86/eclass/eutils.eclass,v 1.377 2012/01/03 08:45:36 jlec Exp $

# @ECLASS: unpacker.eclass
# @MAINTAINER:
# base-system@gentoo.org
# @BLURB: helpers for extraneous file formats and consistent behavior across EAPIs
# @DESCRIPTION:
# Some extraneous file formats are not part of PMS, or are only in certain
# EAPIs.  Rather than worrying about that, support the crazy cruft here
# and for all EAPI versions.

# Possible todos:
#  - merge rpm unpacking
#  - support partial unpacks?

if [[ ${___ECLASS_ONCE_UNPACKER} != "recur -_+^+_- spank" ]] ; then
___ECLASS_ONCE_UNPACKER="recur -_+^+_- spank"

# @ECLASS-VARIABLE: UNPACKER_BZ2
# @DEFAULT_UNSET
# @DESCRIPTION:
# Utility to use to decompress bzip2 files.  Will dynamically pick between
# `pbzip2` and `bzip2`.  Make sure your choice accepts the "-c" option.
# Note: this is meant for users to set, not ebuilds.

# for internal use only (unpack_pdv and unpack_makeself)
find_unpackable_file() {
	local src=$1
	if [[ -z ${src} ]] ; then
		src=${DISTDIR}/${A}
	else
		if [[ ${src} == ./* ]] ; then
			: # already what we want
		elif [[ -e ${DISTDIR}/${src} ]] ; then
			src=${DISTDIR}/${src}
		elif [[ -e ${PWD}/${src} ]] ; then
			src=${PWD}/${src}
		elif [[ -e ${src} ]] ; then
			src=${src}
		fi
	fi
	[[ ! -e ${src} ]] && return 1
	echo "${src}"
}

unpack_banner() {
	echo ">>> Unpacking ${1##*/} to ${PWD}"
}

# @FUNCTION: unpack_pdv
# @USAGE: <file to unpack> <size of off_t>
# @DESCRIPTION:
# Unpack those pesky pdv generated files ...
# They're self-unpacking programs with the binary package stuffed in
# the middle of the archive.  Valve seems to use it a lot ... too bad
# it seems to like to segfault a lot :(.  So lets take it apart ourselves.
#
# You have to specify the off_t size ... I have no idea how to extract that
# information out of the binary executable myself.  Basically you pass in
# the size of the off_t type (in bytes) on the machine that built the pdv
# archive.
#
# One way to determine this is by running the following commands:
#
# @CODE
# 	strings <pdv archive> | grep lseek
# 	strace -elseek <pdv archive>
# @CODE
#
# Basically look for the first lseek command (we do the strings/grep because
# sometimes the function call is _llseek or something) and steal the 2nd
# parameter.  Here is an example:
#
# @CODE
# 	vapier@vapier 0 pdv_unpack # strings hldsupdatetool.bin | grep lseek
# 	lseek
# 	vapier@vapier 0 pdv_unpack # strace -elseek ./hldsupdatetool.bin
# 	lseek(3, -4, SEEK_END)					= 2981250
# @CODE
#
# Thus we would pass in the value of '4' as the second parameter.
unpack_pdv() {
	local src=$(find_unpackable_file "$1")
	local sizeoff_t=$2

	[[ -z ${src} ]] && die "Could not locate source for '$1'"
	[[ -z ${sizeoff_t} ]] && die "No idea what off_t size was used for this pdv :("

	unpack_banner "${src}"

	local metaskip=$(tail -c ${sizeoff_t} "${src}" | hexdump -e \"%i\")
	local tailskip=$(tail -c $((${sizeoff_t}*2)) "${src}" | head -c ${sizeoff_t} | hexdump -e \"%i\")

	# grab metadata for debug reasons
	local metafile=$(emktemp)
	tail -c +$((${metaskip}+1)) "${src}" > "${metafile}"

	# rip out the final file name from the metadata
	local datafile=$(tail -c +$((${metaskip}+1)) "${src}" | strings | head -n 1)
	datafile=$(basename "${datafile}")

	# now lets uncompress/untar the file if need be
	local tmpfile=$(emktemp)
	tail -c +$((${tailskip}+1)) ${src} 2>/dev/null | head -c 512 > ${tmpfile}

	local iscompressed=$(file -b "${tmpfile}")
	if [[ ${iscompressed:0:8} == "compress" ]] ; then
		iscompressed=1
		mv ${tmpfile}{,.Z}
		gunzip ${tmpfile}
	else
		iscompressed=0
	fi
	local istar=$(file -b "${tmpfile}")
	if [[ ${istar:0:9} == "POSIX tar" ]] ; then
		istar=1
	else
		istar=0
	fi

	#for some reason gzip dies with this ... dd cant provide buffer fast enough ?
	#dd if=${src} ibs=${metaskip} count=1 \
	#	| dd ibs=${tailskip} skip=1 \
	#	| gzip -dc \
	#	> ${datafile}
	if [ ${iscompressed} -eq 1 ] ; then
		if [ ${istar} -eq 1 ] ; then
			tail -c +$((${tailskip}+1)) ${src} 2>/dev/null \
				| head -c $((${metaskip}-${tailskip})) \
				| tar -xzf -
		else
			tail -c +$((${tailskip}+1)) ${src} 2>/dev/null \
				| head -c $((${metaskip}-${tailskip})) \
				| gzip -dc \
				> ${datafile}
		fi
	else
		if [ ${istar} -eq 1 ] ; then
			tail -c +$((${tailskip}+1)) ${src} 2>/dev/null \
				| head -c $((${metaskip}-${tailskip})) \
				| tar --no-same-owner -xf -
		else
			tail -c +$((${tailskip}+1)) ${src} 2>/dev/null \
				| head -c $((${metaskip}-${tailskip})) \
				> ${datafile}
		fi
	fi
	true
	#[ -s "${datafile}" ] || die "failure unpacking pdv ('${metaskip}' '${tailskip}' '${datafile}')"
	#assert "failure unpacking pdv ('${metaskip}' '${tailskip}' '${datafile}')"
}

# @FUNCTION: unpack_makeself
# @USAGE: [file to unpack] [offset] [tail|dd]
# @DESCRIPTION:
# Unpack those pesky makeself generated files ...
# They're shell scripts with the binary package tagged onto
# the end of the archive.  Loki utilized the format as does
# many other game companies.
#
# If the file is not specified, then ${A} is used.  If the
# offset is not specified then we will attempt to extract
# the proper offset from the script itself.
unpack_makeself() {
	local src_input=${1:-${A}}
	local src=$(find_unpackable_file "${src_input}")
	local skip=$2
	local exe=$3

	[[ -z ${src} ]] && die "Could not locate source for '${src_input}'"

	unpack_banner "${src}"

	if [[ -z ${skip} ]] ; then
		local ver=$(grep -m1 -a '#.*Makeself' "${src}" | awk '{print $NF}')
		local skip=0
		exe=tail
		case ${ver} in
			1.5.*|1.6.0-nv)	# tested 1.5.{3,4,5} ... guessing 1.5.x series is same
				skip=$(grep -a ^skip= "${src}" | cut -d= -f2)
				;;
			2.0|2.0.1)
				skip=$(grep -a ^$'\t'tail "${src}" | awk '{print $2}' | cut -b2-)
				;;
			2.1.1)
				skip=$(grep -a ^offset= "${src}" | awk '{print $2}' | cut -b2-)
				(( skip++ ))
				;;
			2.1.2)
				skip=$(grep -a ^offset= "${src}" | awk '{print $3}' | head -n 1)
				(( skip++ ))
				;;
			2.1.3)
				skip=`grep -a ^offset= "${src}" | awk '{print $3}'`
				(( skip++ ))
				;;
			2.1.4|2.1.5)
				skip=$(grep -a offset=.*head.*wc "${src}" | awk '{print $3}' | head -n 1)
				skip=$(head -n ${skip} "${src}" | wc -c)
				exe="dd"
				;;
			*)
				eerror "I'm sorry, but I was unable to support the Makeself file."
				eerror "The version I detected was '${ver}'."
				eerror "Please file a bug about the file ${src##*/} at"
				eerror "http://bugs.gentoo.org/ so that support can be added."
				die "makeself version '${ver}' not supported"
				;;
		esac
		debug-print "Detected Makeself version ${ver} ... using ${skip} as offset"
	fi
	case ${exe} in
		tail)	exe="tail -n +${skip} '${src}'";;
		dd)		exe="dd ibs=${skip} skip=1 if='${src}'";;
		*)		die "makeself cant handle exe '${exe}'"
	esac

	# lets grab the first few bytes of the file to figure out what kind of archive it is
	local filetype tmpfile=$(emktemp)
	eval ${exe} 2>/dev/null | head -c 512 > "${tmpfile}"
	filetype=$(file -b "${tmpfile}") || die
	case ${filetype} in
		*tar\ archive*)
			eval ${exe} | tar --no-same-owner -xf -
			;;
		bzip2*)
			eval ${exe} | bzip2 -dc | tar --no-same-owner -xf -
			;;
		gzip*)
			eval ${exe} | tar --no-same-owner -xzf -
			;;
		compress*)
			eval ${exe} | gunzip | tar --no-same-owner -xf -
			;;
		*)
			eerror "Unknown filetype \"${filetype}\" ?"
			false
			;;
	esac
	assert "failure unpacking (${filetype}) makeself ${src##*/} ('${ver}' +${skip})"
}

# @FUNCTION: unpack_deb
# @USAGE: <one deb to unpack>
# @DESCRIPTION:
# Unpack a Debian .deb archive in style.
unpack_deb() {
	[[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>"

	local deb=$(find_unpackable_file "$1")

	unpack_banner "${deb}"

	ar x "${deb}"
	unpack ./data.tar*
}

# @FUNCTION: _unpacker
# @USAGE: <one archive to unpack>
# @INTERNAL
# @DESCRIPTION:
# Unpack the specified archive.  We only operate on one archive here
# to keep down on the looping logic (that is handled by `unpacker`).
_unpacker() {
	[[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>"

	local a=$1
	local m=$(echo "${a}" | tr '[:upper:]' '[:lower:]')
	a=$(find_unpackable_file "${a}")

	# first figure out the decompression method
	case ${m} in
	*.bz2|*.tbz|*.tbz2)
		local bzcmd=${PORTAGE_BZIP2_COMMAND:-$(type -P pbzip2 || bzip2)}
		local bzuncmd=${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}
		: ${UNPACKER_BZ2:=${bzuncmd}}
		comp="${UNPACKER_BZ2} -c"
		;;
	*.z|*.gz|*.tgz)
		comp="gzip -dc" ;;
	*.lzma|*.xz|*.txz)
		comp="xz -dc" ;;
	*)	comp="" ;;
	esac

	# then figure out if there are any archiving aspects
	case ${m} in
	*.tgz|*.tbz|*.tbz2|*.txz|*.tar.*|*.tar)
		arch="tar --no-same-owner -xof" ;;
	*.deb)
		arch="unpack_deb" ;;
	*.run)
		arch="unpack_makeself" ;;
	*) arch="" ;;
	esac

	# finally do the unpack
	if [[ -z ${arch}${comp} ]] ; then
		unpack "${a}"
		return $?
	fi

	[[ ${arch} != unpack_* ]] && unpack_banner "${a}"

	if [[ -z ${arch} ]] ; then
		${comp} "${a}" > "${a%.*}"
	elif [[ -z ${comp} ]] ; then
		${arch} "${a}"
	else
		${comp} "${a}" | ${arch} -
	fi

	assert "unpacking ${a} failed (comp=${comp} arch=${arch})"
}

# @FUNCTION: unpacker
# @USAGE: [archives to unpack]
# @DESCRIPTION:
# This works in the same way that `unpack` does.  If you don't specify
# any files, it will default to ${A}.
unpacker() {
	local a
	[[ $# -eq 0 ]] && set -- ${A}
	for a ; do _unpacker "${a}" ; done
}

# @FUNCTION: unpacker_src_unpack
# @DESCRIPTION:
# Run `unpacker` to unpack all our stuff.
unpacker_src_unpack() {
	unpacker
}

# @FUNCTION: unpacker_src_uri_depends
# @USAGE: [archives that we will unpack]
# @RETURN: Dependencies needed to unpack all the archives
# @DESCRIPTION:
# Walk all the specified files (defaults to $SRC_URI) and figure out the
# dependencies that are needed to unpack things.
#
# Note: USE flags are not yet handled.
unpacker_src_uri_depends() {
	local uri deps d

	[[ $# -eq 0 ]] && set -- ${SRC_URI}

	for uri in "$@" ; do
		case ${uri} in
		*.rar|*.RAR)
			d="app-arch/unrar" ;;
		*.7z)
			d="app-arch/p7zip" ;;
		*.xz)
			d="app-arch/xz-utils" ;;
		esac
		deps+=" ${d}"
	done

	echo "${deps}"
}

EXPORT_FUNCTIONS src_unpack

fi

[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

[gentoo-dev] unpacker.eclass and base.eclass integration

[Mike Frysinger]

[-- Attachment #1: Type: Text/Plain, Size: 503 bytes --]

follow up: have base.eclass leverage unpacker.eclass when it's available
-mike

--- base.eclass	14 Dec 2011 23:38:09 -0000	1.55
+++ base.eclass	5 Feb 2012 05:19:28 -0000
@@ -59,7 +59,11 @@ base_src_unpack() {
 
 	pushd "${WORKDIR}" > /dev/null
 
-	[[ -n "${A}" ]] && unpack ${A}
+	if [[ $(type -t unpacker_src_unpack) == "function" ]] ; then
+		unpacker_src_unpack
+	elif [[ -n ${A} ]] ; then
+		unpack ${A}
+	fi
 	has src_prepare ${BASE_EXPF} || base_src_prepare
 
 	popd > /dev/null

[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]