[gentoo-commits] proj/portage:master commit in: man/, pym/portage/dbapi/, pym/portage/

[gentoo-commits] proj/portage:master commit in: man/, pym/portage/dbapi/, pym/portage/

[Zac Medico]

commit:     770bb7ae204643be1968f0fd0379706b6ce017c0
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 24 11:17:26 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Dec 24 11:17:26 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=770bb7ae

Add FEATURES=config-protect-if-modified support.

This causes the CONFIG_PROTECT behavior to be skipped for files
that have not been modified since they were installed.

---
 man/make.conf.5              |    4 ++++
 pym/portage/const.py         |    1 +
 pym/portage/dbapi/vartree.py |   15 ++++++++++++++-
 3 files changed, 19 insertions(+), 1 deletions(-)

diff --git a/man/make.conf.5 b/man/make.conf.5
index 8a66c21..33f4e45 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -260,6 +260,10 @@ Log file names have an extension that is appropriate for the compression
 type. Currently, only \fBgzip\fR(1) compression is supported, so build
 logs will have a '.gz' extension when this feature is enabled.
 .TP
+.B config\-protect\-if\-modified
+This causes the \fBCONFIG_PROTECT\fR behavior to be skipped for files
+that have not been modified since they were installed.
+.TP
 .B digest
 Autogenerate digests for packages when running the
 \fBemerge\fR(1), \fBebuild\fR(1), or \fBrepoman\fR(1) commands. If

diff --git a/pym/portage/const.py b/pym/portage/const.py
index 77c68eb..5fcb24f 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -89,6 +89,7 @@ SUPPORTED_FEATURES       = frozenset([
                            "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
                            "ccache", "chflags", "clean-logs",
                            "collision-protect", "compress-build-logs",
+                           "config-protect-if-modified",
                            "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
                            "fail-clean", "force-mirror", "force-prefix", "getbinpkg",
                            "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index b9ef583..a66316b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4056,6 +4056,10 @@ class dblink(object):
 		destroot = normalize_path(destroot).rstrip(sep) + sep
 		calc_prelink = "prelink-checksums" in self.settings.features
 
+		protect_if_modified = \
+			"config-protect-if-modified" in self.settings.features and \
+			self._installed_instance is not None
+
 		# this is supposed to merge a list of files.  There will be 2 forms of argument passing.
 		if isinstance(stufftomerge, basestring):
 			#A directory is specified.  Figure out protection paths, listdir() it and process it.
@@ -4297,9 +4301,18 @@ class dblink(object):
 						# now, config file management may come into play.
 						# we only need to tweak mydest if cfg file management is in play.
 						if protected:
+							destmd5 = perform_md5(mydest, calc_prelink=calc_prelink)
+							if protect_if_modified:
+								contents_key = \
+									self._installed_instance._match_contents(myrealdest)
+								if contents_key:
+									inst_info = self._installed_instance.getcontents()[contents_key]
+									if inst_info[0] == "obj" and inst_info[2] == destmd5:
+										protected = False
+
+						if protected:
 							# we have a protection path; enable config file management.
 							cfgprot = 0
-							destmd5 = perform_md5(mydest, calc_prelink=calc_prelink)
 							if mymd5 == destmd5:
 								#file already in place; simply update mtimes of destination
 								moveme = 1

[gentoo-commits] proj/portage:master commit in: man/, pym/portage/dbapi/, pym/portage/

[Zac Medico]

commit:     11c0619c63b54346ee5c67cd67ab1ccb24f5f947
Author:     W-Mark Kubacki <wmark <AT> hurrikane <DOT> de>
AuthorDate: Wed Aug  8 16:49:36 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Aug  8 20:18:43 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=11c0619c

Portage writes a compressed copy of 'Packages' index file.

This behaviour is enabled by FEATURES="compress-index". The
resulting file is 'Packages.gz' and its modification time will
match that of 'Packages'.

Web-servers use that copy to avoid repeated on-the-fly compression.

In order to re-use 'atomic_ofstream' usage of 'codecs.zlib_codec'
has been considered and discarded, because 'GzipFile' yields
smaller files. (According to Mark's tests 62% smaller.)

Example usage, Nginx:

  location =/Packages {
    gzip_static on;
    default_type text/plain;
  }

Apache httpd (use with caution):

  RewriteRule ^(.*)/Packages$ $1/Packages.gz [T=text/plain,E=GZIP:gzip,L]
  <FilesMatch "Packages\.gz$">
    Header set Content-Encoding gzip
  </FilesMatch>

---
 man/make.conf.5              |    7 +++++++
 pym/portage/const.py         |    2 +-
 pym/portage/dbapi/bintree.py |   21 +++++++++++++++------
 3 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/man/make.conf.5 b/man/make.conf.5
index 876a8a3..02cc809 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -268,6 +268,13 @@ space.  Make sure you have built both binutils and gdb with USE=zlib
 support for this to work.  See \fBsplitdebug\fR for general split debug
 information (upon which this feature depends).
 .TP
+.B compress\-index
+If set then a compressed copy of 'Packages' index file will be written.
+This feature is intended for Gentoo binhosts using certain webservers
+(such as, but not limited to, Nginx with gzip_static module) to avoid
+redundant on\-the\-fly compression.  The resulting file will be called
+'Packages.gz' and its modification time will match that of 'Packages'.
+.TP
 .B config\-protect\-if\-modified
 This causes the \fBCONFIG_PROTECT\fR behavior to be skipped for files
 that have not been modified since they were installed. This feature is

diff --git a/pym/portage/const.py b/pym/portage/const.py
index ceef5c5..c2049f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -89,7 +89,7 @@ SUPPORTED_FEATURES       = frozenset([
                            "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
                            "ccache", "chflags", "clean-logs",
                            "collision-protect", "compress-build-logs", "compressdebug",
-                           "config-protect-if-modified",
+                           "compress-index", "config-protect-if-modified",
                            "digest", "distcc", "distcc-pump", "distlocks",
                            "downgrade-backup", "ebuild-locks", "fakeroot",
                            "fail-clean", "force-mirror", "force-prefix", "getbinpkg",

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 0367503..77fc0c4 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -41,6 +41,7 @@ import sys
 import tempfile
 import textwrap
 import warnings
+from gzip import GzipFile
 from itertools import chain
 try:
 	from urllib.parse import urlparse
@@ -1186,13 +1187,21 @@ class binarytree(object):
 			pkgindex.packages.append(d)
 
 			self._update_pkgindex_header(pkgindex.header)
-			pkgindex_filename = os.path.join(self.pkgdir, "Packages")
-			f = atomic_ofstream(pkgindex_filename)
-			pkgindex.write(f)
-			f.close()
-			# some seconds might have elapsed since TIMESTAMP
+			contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO())
+			pkgindex.write(contents)
+			contents = contents.getvalue()
 			atime = mtime = long(pkgindex.header["TIMESTAMP"])
-			os.utime(pkgindex_filename, (atime, mtime))
+
+			pkgindex_filename = os.path.join(self.pkgdir, "Packages")
+			output_files = [(atomic_ofstream(pkgindex_filename, mode="wb"), pkgindex_filename)]
+			if "compress-index" in self.settings.features:
+				gz_fname = pkgindex_filename + ".gz"
+				output_files.append((GzipFile(gz_fname, mode="wb"), gz_fname))
+			for f, fname in output_files:
+				f.write(contents)
+				f.close()
+				# some seconds might have elapsed since TIMESTAMP
+				os.utime(fname, (atime, mtime))
 		finally:
 			if pkgindex_lock:
 				unlockfile(pkgindex_lock)