* [gentoo-commits] proj/portage:master commit in: man/, pym/portage/dbapi/, pym/portage/
@ 2012-08-08 20:20 Zac Medico
0 siblings, 0 replies; 2+ messages in thread
From: Zac Medico @ 2012-08-08 20:20 UTC (permalink / raw
To: gentoo-commits
commit: 11c0619c63b54346ee5c67cd67ab1ccb24f5f947
Author: W-Mark Kubacki <wmark <AT> hurrikane <DOT> de>
AuthorDate: Wed Aug 8 16:49:36 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Aug 8 20:18:43 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=11c0619c
Portage writes a compressed copy of 'Packages' index file.
This behaviour is enabled by FEATURES="compress-index". The
resulting file is 'Packages.gz' and its modification time will
match that of 'Packages'.
Web-servers use that copy to avoid repeated on-the-fly compression.
In order to re-use 'atomic_ofstream' usage of 'codecs.zlib_codec'
has been considered and discarded, because 'GzipFile' yields
smaller files. (According to Mark's tests 62% smaller.)
Example usage, Nginx:
location =/Packages {
gzip_static on;
default_type text/plain;
}
Apache httpd (use with caution):
RewriteRule ^(.*)/Packages$ $1/Packages.gz [T=text/plain,E=GZIP:gzip,L]
<FilesMatch "Packages\.gz$">
Header set Content-Encoding gzip
</FilesMatch>
---
man/make.conf.5 | 7 +++++++
pym/portage/const.py | 2 +-
pym/portage/dbapi/bintree.py | 21 +++++++++++++++------
3 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/man/make.conf.5 b/man/make.conf.5
index 876a8a3..02cc809 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -268,6 +268,13 @@ space. Make sure you have built both binutils and gdb with USE=zlib
support for this to work. See \fBsplitdebug\fR for general split debug
information (upon which this feature depends).
.TP
+.B compress\-index
+If set then a compressed copy of 'Packages' index file will be written.
+This feature is intended for Gentoo binhosts using certain webservers
+(such as, but not limited to, Nginx with gzip_static module) to avoid
+redundant on\-the\-fly compression. The resulting file will be called
+'Packages.gz' and its modification time will match that of 'Packages'.
+.TP
.B config\-protect\-if\-modified
This causes the \fBCONFIG_PROTECT\fR behavior to be skipped for files
that have not been modified since they were installed. This feature is
diff --git a/pym/portage/const.py b/pym/portage/const.py
index ceef5c5..c2049f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -89,7 +89,7 @@ SUPPORTED_FEATURES = frozenset([
"assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
"ccache", "chflags", "clean-logs",
"collision-protect", "compress-build-logs", "compressdebug",
- "config-protect-if-modified",
+ "compress-index", "config-protect-if-modified",
"digest", "distcc", "distcc-pump", "distlocks",
"downgrade-backup", "ebuild-locks", "fakeroot",
"fail-clean", "force-mirror", "force-prefix", "getbinpkg",
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 0367503..77fc0c4 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -41,6 +41,7 @@ import sys
import tempfile
import textwrap
import warnings
+from gzip import GzipFile
from itertools import chain
try:
from urllib.parse import urlparse
@@ -1186,13 +1187,21 @@ class binarytree(object):
pkgindex.packages.append(d)
self._update_pkgindex_header(pkgindex.header)
- pkgindex_filename = os.path.join(self.pkgdir, "Packages")
- f = atomic_ofstream(pkgindex_filename)
- pkgindex.write(f)
- f.close()
- # some seconds might have elapsed since TIMESTAMP
+ contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO())
+ pkgindex.write(contents)
+ contents = contents.getvalue()
atime = mtime = long(pkgindex.header["TIMESTAMP"])
- os.utime(pkgindex_filename, (atime, mtime))
+
+ pkgindex_filename = os.path.join(self.pkgdir, "Packages")
+ output_files = [(atomic_ofstream(pkgindex_filename, mode="wb"), pkgindex_filename)]
+ if "compress-index" in self.settings.features:
+ gz_fname = pkgindex_filename + ".gz"
+ output_files.append((GzipFile(gz_fname, mode="wb"), gz_fname))
+ for f, fname in output_files:
+ f.write(contents)
+ f.close()
+ # some seconds might have elapsed since TIMESTAMP
+ os.utime(fname, (atime, mtime))
finally:
if pkgindex_lock:
unlockfile(pkgindex_lock)
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [gentoo-commits] proj/portage:master commit in: man/, pym/portage/dbapi/, pym/portage/
@ 2011-12-24 11:17 Zac Medico
0 siblings, 0 replies; 2+ messages in thread
From: Zac Medico @ 2011-12-24 11:17 UTC (permalink / raw
To: gentoo-commits
commit: 770bb7ae204643be1968f0fd0379706b6ce017c0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 24 11:17:26 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Dec 24 11:17:26 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=770bb7ae
Add FEATURES=config-protect-if-modified support.
This causes the CONFIG_PROTECT behavior to be skipped for files
that have not been modified since they were installed.
---
man/make.conf.5 | 4 ++++
pym/portage/const.py | 1 +
pym/portage/dbapi/vartree.py | 15 ++++++++++++++-
3 files changed, 19 insertions(+), 1 deletions(-)
diff --git a/man/make.conf.5 b/man/make.conf.5
index 8a66c21..33f4e45 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -260,6 +260,10 @@ Log file names have an extension that is appropriate for the compression
type. Currently, only \fBgzip\fR(1) compression is supported, so build
logs will have a '.gz' extension when this feature is enabled.
.TP
+.B config\-protect\-if\-modified
+This causes the \fBCONFIG_PROTECT\fR behavior to be skipped for files
+that have not been modified since they were installed.
+.TP
.B digest
Autogenerate digests for packages when running the
\fBemerge\fR(1), \fBebuild\fR(1), or \fBrepoman\fR(1) commands. If
diff --git a/pym/portage/const.py b/pym/portage/const.py
index 77c68eb..5fcb24f 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -89,6 +89,7 @@ SUPPORTED_FEATURES = frozenset([
"assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
"ccache", "chflags", "clean-logs",
"collision-protect", "compress-build-logs",
+ "config-protect-if-modified",
"digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
"fail-clean", "force-mirror", "force-prefix", "getbinpkg",
"installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index b9ef583..a66316b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4056,6 +4056,10 @@ class dblink(object):
destroot = normalize_path(destroot).rstrip(sep) + sep
calc_prelink = "prelink-checksums" in self.settings.features
+ protect_if_modified = \
+ "config-protect-if-modified" in self.settings.features and \
+ self._installed_instance is not None
+
# this is supposed to merge a list of files. There will be 2 forms of argument passing.
if isinstance(stufftomerge, basestring):
#A directory is specified. Figure out protection paths, listdir() it and process it.
@@ -4297,9 +4301,18 @@ class dblink(object):
# now, config file management may come into play.
# we only need to tweak mydest if cfg file management is in play.
if protected:
+ destmd5 = perform_md5(mydest, calc_prelink=calc_prelink)
+ if protect_if_modified:
+ contents_key = \
+ self._installed_instance._match_contents(myrealdest)
+ if contents_key:
+ inst_info = self._installed_instance.getcontents()[contents_key]
+ if inst_info[0] == "obj" and inst_info[2] == destmd5:
+ protected = False
+
+ if protected:
# we have a protection path; enable config file management.
cfgprot = 0
- destmd5 = perform_md5(mydest, calc_prelink=calc_prelink)
if mymd5 == destmd5:
#file already in place; simply update mtimes of destination
moveme = 1
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2012-08-08 20:20 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2012-08-08 20:20 [gentoo-commits] proj/portage:master commit in: man/, pym/portage/dbapi/, pym/portage/ Zac Medico
-- strict thread matches above, loose matches on Subject: below --
2011-12-24 11:17 Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox