* [gentoo-portage-dev] [PATCH] prepman: do not compress files <=128 bytes
@ 2013-03-20 8:37 Mike Frysinger
2013-03-20 8:41 ` [gentoo-portage-dev] [PATCH v2] " Mike Frysinger
0 siblings, 1 reply; 10+ messages in thread
From: Mike Frysinger @ 2013-03-20 8:37 UTC (permalink / raw
To: gentoo-portage-dev
The vast majority of these small files do not compress better than
their inputs, and they're just .so redirection. Omit compression
on them to save disk and cpu and speed things up.
URL: http://bugs.gentoo.org/169260
Signed-off-by: Mike Frysinger <vapier@gentoo.org>
---
bin/ebuild-helpers/ecompressdir | 33 ++++++++++++++++++++++++++-------
bin/ebuild-helpers/prepman | 5 ++++-
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/bin/ebuild-helpers/ecompressdir b/bin/ebuild-helpers/ecompressdir
index 75f3e3a..bde2cca 100755
--- a/bin/ebuild-helpers/ecompressdir
+++ b/bin/ebuild-helpers/ecompressdir
@@ -13,7 +13,9 @@ if ! ___eapi_has_prefix_variables; then
ED=${D} EPREFIX=
fi
-case $1 in
+SIZE_LIMIT=''
+while [[ $# -gt 0 ]] ; do
+ case $1 in
--ignore)
shift
for skip in "$@" ; do
@@ -28,7 +30,8 @@ case $1 in
set -- "${@/#/${ED}}"
ret=0
for x in "$@" ; do
- >> "$x"
+ # Stash the limit in the .dir file so we can reload it later.
+ printf ${SIZE_LIMIT} > "${x}"
((ret|=$?))
done
[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
@@ -42,29 +45,44 @@ case $1 in
find "${ED}" -name '*.ecompress.skip' -print0 | ${XARGS} -0 rm -f
exit 0
;;
+ --limit)
+ SIZE_LIMIT=$2
+ shift
+ ;;
--*)
__helpers_die "${0##*/}: unknown arguments '$*'"
exit 1
;;
-esac
+ *)
+ break
+ ;;
+ esac
+ shift
+done
# figure out the new suffix
suffix=$(ecompress --suffix)
-# funk_up_dir(action, suffix, binary)
+# funk_up_dir(action, suffix, binary, [size_limit])
# - action: compress or decompress
# - suffix: the compression suffix to work with
# - binary: the program to execute that'll compress/decompress
+# - size_limit: if compressing, skip files smaller than this
# The directory we act on is implied in the ${dir} variable
funk_up_dir() {
- local act=$1 suffix=$2 binary=$3
+ local act=$1 suffix=$2 binary=$3 size_limit=$4
local negate=""
[[ ${act} == "compress" ]] && negate="!"
local ret=0
# first we act on all the files
- find "${dir}" -type f ${negate} -iname '*'${suffix} -print0 | ${XARGS} -0 ${binary}
+ local args=(
+ -type f
+ ${negate} -iname "*${suffix}"
+ )
+ [[ -n ${size_limit} ]] && args+=( -size "+${size_limit}c" )
+ find "${dir}" "${args[@]}" -print0 | ${XARGS} -0 ${binary}
((ret|=$?))
while read -r -d $'\0' brokenlink ; do
@@ -152,6 +170,7 @@ for dir in "$@" ; do
# since we've been requested to compress the whole dir,
# delete any individual queued requests
+ size_limit=${SIZE_LIMIT:-$(<"${actual_dir}.ecompress.dir")}
rm -f "${actual_dir}.ecompress.dir"
find "${dir}" -type f -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f
@@ -179,7 +198,7 @@ for dir in "$@" ; do
# now lets do our work
if [[ -n ${suffix} ]] ; then
__vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED}}"
- funk_up_dir "compress" "${suffix}" "ecompress"
+ funk_up_dir "compress" "${suffix}" "ecompress" "${size_limit}"
: $(( ret |= $? ))
fi
diff --git a/bin/ebuild-helpers/prepman b/bin/ebuild-helpers/prepman
index 55a9483..fb5dcb4 100755
--- a/bin/ebuild-helpers/prepman
+++ b/bin/ebuild-helpers/prepman
@@ -2,6 +2,9 @@
# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+# Do not compress man pages which are smaller than this (in bytes). #169260
+SIZE_LIMIT='128'
+
source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh
if ! ___eapi_has_prefix_variables; then
@@ -31,6 +34,6 @@ for subdir in "${mandir}"/man* "${mandir}"/*/man* ; do
[[ -d ${subdir} ]] && really_is_mandir=1 && break
done
-[[ ${really_is_mandir} == 1 ]] && exec ecompressdir --queue "${mandir#${ED}}"
+[[ ${really_is_mandir} == 1 ]] && exec ecompressdir --limit ${SIZE_LIMIT} --queue "${mandir#${ED}}"
exit 0
--
1.8.1.2
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-20 8:37 [gentoo-portage-dev] [PATCH] prepman: do not compress files <=128 bytes Mike Frysinger
@ 2013-03-20 8:41 ` Mike Frysinger
2013-03-20 18:32 ` Zac Medico
2013-03-21 11:51 ` James Cloos
0 siblings, 2 replies; 10+ messages in thread
From: Mike Frysinger @ 2013-03-20 8:41 UTC (permalink / raw
To: gentoo-portage-dev
The vast majority of these small files do not compress better than
their inputs, and they're just .so redirection. Omit compression
on them to save disk and cpu and speed things up.
URL: http://bugs.gentoo.org/169260
Signed-off-by: Mike Frysinger <vapier@gentoo.org>
---
v2
- fix printf thinko
bin/ebuild-helpers/ecompressdir | 33 ++++++++++++++++++++++++++-------
bin/ebuild-helpers/prepman | 5 ++++-
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/bin/ebuild-helpers/ecompressdir b/bin/ebuild-helpers/ecompressdir
index 75f3e3a..0f05c27 100755
--- a/bin/ebuild-helpers/ecompressdir
+++ b/bin/ebuild-helpers/ecompressdir
@@ -13,7 +13,9 @@ if ! ___eapi_has_prefix_variables; then
ED=${D} EPREFIX=
fi
-case $1 in
+SIZE_LIMIT=''
+while [[ $# -gt 0 ]] ; do
+ case $1 in
--ignore)
shift
for skip in "$@" ; do
@@ -28,7 +30,8 @@ case $1 in
set -- "${@/#/${ED}}"
ret=0
for x in "$@" ; do
- >> "$x"
+ # Stash the limit in the .dir file so we can reload it later.
+ printf "${SIZE_LIMIT}" > "${x}"
((ret|=$?))
done
[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
@@ -42,29 +45,44 @@ case $1 in
find "${ED}" -name '*.ecompress.skip' -print0 | ${XARGS} -0 rm -f
exit 0
;;
+ --limit)
+ SIZE_LIMIT=$2
+ shift
+ ;;
--*)
__helpers_die "${0##*/}: unknown arguments '$*'"
exit 1
;;
-esac
+ *)
+ break
+ ;;
+ esac
+ shift
+done
# figure out the new suffix
suffix=$(ecompress --suffix)
-# funk_up_dir(action, suffix, binary)
+# funk_up_dir(action, suffix, binary, [size_limit])
# - action: compress or decompress
# - suffix: the compression suffix to work with
# - binary: the program to execute that'll compress/decompress
+# - size_limit: if compressing, skip files smaller than this
# The directory we act on is implied in the ${dir} variable
funk_up_dir() {
- local act=$1 suffix=$2 binary=$3
+ local act=$1 suffix=$2 binary=$3 size_limit=$4
local negate=""
[[ ${act} == "compress" ]] && negate="!"
local ret=0
# first we act on all the files
- find "${dir}" -type f ${negate} -iname '*'${suffix} -print0 | ${XARGS} -0 ${binary}
+ local args=(
+ -type f
+ ${negate} -iname "*${suffix}"
+ )
+ [[ -n ${size_limit} ]] && args+=( -size "+${size_limit}c" )
+ find "${dir}" "${args[@]}" -print0 | ${XARGS} -0 ${binary}
((ret|=$?))
while read -r -d $'\0' brokenlink ; do
@@ -152,6 +170,7 @@ for dir in "$@" ; do
# since we've been requested to compress the whole dir,
# delete any individual queued requests
+ size_limit=${SIZE_LIMIT:-$(<"${actual_dir}.ecompress.dir")}
rm -f "${actual_dir}.ecompress.dir"
find "${dir}" -type f -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f
@@ -179,7 +198,7 @@ for dir in "$@" ; do
# now lets do our work
if [[ -n ${suffix} ]] ; then
__vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED}}"
- funk_up_dir "compress" "${suffix}" "ecompress"
+ funk_up_dir "compress" "${suffix}" "ecompress" "${size_limit}"
: $(( ret |= $? ))
fi
diff --git a/bin/ebuild-helpers/prepman b/bin/ebuild-helpers/prepman
index 55a9483..fb5dcb4 100755
--- a/bin/ebuild-helpers/prepman
+++ b/bin/ebuild-helpers/prepman
@@ -2,6 +2,9 @@
# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+# Do not compress man pages which are smaller than this (in bytes). #169260
+SIZE_LIMIT='128'
+
source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh
if ! ___eapi_has_prefix_variables; then
@@ -31,6 +34,6 @@ for subdir in "${mandir}"/man* "${mandir}"/*/man* ; do
[[ -d ${subdir} ]] && really_is_mandir=1 && break
done
-[[ ${really_is_mandir} == 1 ]] && exec ecompressdir --queue "${mandir#${ED}}"
+[[ ${really_is_mandir} == 1 ]] && exec ecompressdir --limit ${SIZE_LIMIT} --queue "${mandir#${ED}}"
exit 0
--
1.8.1.2
^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-20 8:41 ` [gentoo-portage-dev] [PATCH v2] " Mike Frysinger
@ 2013-03-20 18:32 ` Zac Medico
2013-03-21 11:51 ` James Cloos
1 sibling, 0 replies; 10+ messages in thread
From: Zac Medico @ 2013-03-20 18:32 UTC (permalink / raw
To: gentoo-portage-dev
On 03/20/2013 01:41 AM, Mike Frysinger wrote:
> The vast majority of these small files do not compress better than
> their inputs, and they're just .so redirection. Omit compression
> on them to save disk and cpu and speed things up.
>
> URL: http://bugs.gentoo.org/169260
> Signed-off-by: Mike Frysinger <vapier@gentoo.org>
> ---
> v2
> - fix printf thinko
>
> bin/ebuild-helpers/ecompressdir | 33 ++++++++++++++++++++++++++-------
> bin/ebuild-helpers/prepman | 5 ++++-
> 2 files changed, 30 insertions(+), 8 deletions(-)
Looks good to me.
--
Thanks,
Zac
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-20 8:41 ` [gentoo-portage-dev] [PATCH v2] " Mike Frysinger
2013-03-20 18:32 ` Zac Medico
@ 2013-03-21 11:51 ` James Cloos
2013-03-21 16:43 ` Sergei Trofimovich
2013-03-21 18:43 ` Mike Frysinger
1 sibling, 2 replies; 10+ messages in thread
From: James Cloos @ 2013-03-21 11:51 UTC (permalink / raw
To: Mike Frysinger; +Cc: gentoo-portage-dev
>>>>> "MF" == Mike Frysinger <vapier@gentoo.org> writes:
MF> +SIZE_LIMIT='128'
The limit probably should be larger, given that nothing which fits into
one filesystem block will use less space compressed.
In many cases a limit of 4096 would be appropriate. But a limit of 1024
should cover the lowest common denominator.
-JimC
--
James Cloos <cloos@jhcloos.com> OpenPGP: 1024D/ED7DAEA6
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-21 11:51 ` James Cloos
@ 2013-03-21 16:43 ` Sergei Trofimovich
2013-03-21 18:43 ` Mike Frysinger
1 sibling, 0 replies; 10+ messages in thread
From: Sergei Trofimovich @ 2013-03-21 16:43 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: cloos, Mike Frysinger
[-- Attachment #1: Type: text/plain, Size: 536 bytes --]
On Thu, 21 Mar 2013 07:51:49 -0400
James Cloos <cloos@jhcloos.com> wrote:
> >>>>> "MF" == Mike Frysinger <vapier@gentoo.org> writes:
>
> MF> +SIZE_LIMIT='128'
>
> The limit probably should be larger, given that nothing which fits into
> one filesystem block will use less space compressed.
While it's usually true btrfs does not store things in blocks and
allocates space in byte ranges. All filesystem (data or meta)
modifications are copy-on-write thus it does not needs "growth holes"
by design.
--
Sergei
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-21 11:51 ` James Cloos
2013-03-21 16:43 ` Sergei Trofimovich
@ 2013-03-21 18:43 ` Mike Frysinger
2013-03-21 23:25 ` James Cloos
1 sibling, 1 reply; 10+ messages in thread
From: Mike Frysinger @ 2013-03-21 18:43 UTC (permalink / raw
To: James Cloos; +Cc: gentoo-portage-dev
[-- Attachment #1: Type: Text/Plain, Size: 661 bytes --]
On Thursday 21 March 2013 07:51:49 James Cloos wrote:
> >>>>> "MF" == Mike Frysinger <vapier@gentoo.org> writes:
> MF> +SIZE_LIMIT='128'
>
> The limit probably should be larger, given that nothing which fits into
> one filesystem block will use less space compressed.
>
> In many cases a limit of 4096 would be appropriate. But a limit of 1024
> should cover the lowest common denominator.
realistically, there are no man pages that are under 4k. look at the
referenced bug for more details.
you're also ignoring things like tail packing. i really don't want to start
customizing code for FS idiosyncrasies that are not reliable.
-mike
[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-21 18:43 ` Mike Frysinger
@ 2013-03-21 23:25 ` James Cloos
2013-03-22 0:17 ` Mike Frysinger
0 siblings, 1 reply; 10+ messages in thread
From: James Cloos @ 2013-03-21 23:25 UTC (permalink / raw
To: Mike Frysinger; +Cc: gentoo-portage-dev
>>>>> "MF" == Mike Frysinger <vapier@gentoo.org> writes:
MF> realistically, there are no man pages that are under 4k. look at the
MF> referenced bug for more details.
Oh. I didn't look closely enough to notice that it was only for man; I
presumed that it would apply to /usr/share/doc/ as well....
-JimC
--
James Cloos <cloos@jhcloos.com> OpenPGP: 1024D/ED7DAEA6
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-21 23:25 ` James Cloos
@ 2013-03-22 0:17 ` Mike Frysinger
2013-03-23 19:55 ` James Cloos
2014-01-22 5:30 ` Mike Frysinger
0 siblings, 2 replies; 10+ messages in thread
From: Mike Frysinger @ 2013-03-22 0:17 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: James Cloos
[-- Attachment #1: Type: Text/Plain, Size: 484 bytes --]
On Thursday 21 March 2013 19:25:08 James Cloos wrote:
> >>>>> "MF" == Mike Frysinger <vapier@gentoo.org> writes:
> MF> realistically, there are no man pages that are under 4k. look at the
> MF> referenced bug for more details.
>
> Oh. I didn't look closely enough to notice that it was only for man; I
> presumed that it would apply to /usr/share/doc/ as well....
the subject does say "prepman" ;)
we'll probably do something for doc too. my focus was on man.
-mike
[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-22 0:17 ` Mike Frysinger
@ 2013-03-23 19:55 ` James Cloos
2014-01-22 5:30 ` Mike Frysinger
1 sibling, 0 replies; 10+ messages in thread
From: James Cloos @ 2013-03-23 19:55 UTC (permalink / raw
To: Mike Frysinger; +Cc: gentoo-portage-dev
>>>>> "MF" == Mike Frysinger <vapier@gentoo.org> writes:
MF> the subject does say "prepman" ;)
I guess that blended in with the [] sections; all I remember noticing in
the subject was the text after the last colon....
-JimC
--
James Cloos <cloos@jhcloos.com> OpenPGP: 1024D/ED7DAEA6
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v2] prepman: do not compress files <=128 bytes
2013-03-22 0:17 ` Mike Frysinger
2013-03-23 19:55 ` James Cloos
@ 2014-01-22 5:30 ` Mike Frysinger
1 sibling, 0 replies; 10+ messages in thread
From: Mike Frysinger @ 2014-01-22 5:30 UTC (permalink / raw
To: gentoo-portage-dev; +Cc: James Cloos
[-- Attachment #1: Type: Text/Plain, Size: 641 bytes --]
On Thursday 21 March 2013 20:17:30 Mike Frysinger wrote:
> On Thursday 21 March 2013 19:25:08 James Cloos wrote:
> > >>>>> "MF" == Mike Frysinger <vapier@gentoo.org> writes:
> > MF> realistically, there are no man pages that are under 4k. look at the
> > MF> referenced bug for more details.
> >
> > Oh. I didn't look closely enough to notice that it was only for man; I
> > presumed that it would apply to /usr/share/doc/ as well....
>
> the subject does say "prepman" ;)
>
> we'll probably do something for doc too. my focus was on man.
portage-2.2.8 now implements this policy for all files given to ecompress
-mike
[-- Attachment #2: This is a digitally signed message part. --]
[-- Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 10+ messages in thread
end of thread, other threads:[~2014-01-22 5:30 UTC | newest]
Thread overview: 10+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-03-20 8:37 [gentoo-portage-dev] [PATCH] prepman: do not compress files <=128 bytes Mike Frysinger
2013-03-20 8:41 ` [gentoo-portage-dev] [PATCH v2] " Mike Frysinger
2013-03-20 18:32 ` Zac Medico
2013-03-21 11:51 ` James Cloos
2013-03-21 16:43 ` Sergei Trofimovich
2013-03-21 18:43 ` Mike Frysinger
2013-03-21 23:25 ` James Cloos
2013-03-22 0:17 ` Mike Frysinger
2013-03-23 19:55 ` James Cloos
2014-01-22 5:30 ` Mike Frysinger
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox