[gentoo-portage-dev] [PATCH] Manifest.write: stable/predictable Manifest mtime for rsync (bug 557962)

[gentoo-portage-dev] [PATCH] Manifest.write: stable/predictable Manifest mtime for rsync (bug 557962)

[Zac Medico]

Use the max mtime of the existing Manifest and the files that the updated
Manifest contains.

X-Gentoo-Bug: 557962
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=557962
---
 pym/portage/manifest.py | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 3936b9a..f5cf0f5 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 import errno
 import io
 import re
+import stat
 import sys
 import warnings
 
@@ -281,6 +282,7 @@ class Manifest(object):
 		try:
 			myentries = list(self._createManifestEntries())
 			update_manifest = True
+			existing_st = None
 			if myentries and not force:
 				try:
 					f = io.open(_unicode_encode(self.getFullname(),
@@ -288,6 +290,7 @@ class Manifest(object):
 						mode='r', encoding=_encodings['repo.content'],
 						errors='replace')
 					oldentries = list(self._parseManifestLines(f))
+					existing_st = os.fstat(f.fileno())
 					f.close()
 					if len(oldentries) == len(myentries):
 						update_manifest = False
@@ -309,6 +312,7 @@ class Manifest(object):
 					# non-empty for all currently known use cases.
 					write_atomic(self.getFullname(), "".join("%s\n" %
 						_unicode(myentry) for myentry in myentries))
+					self._apply_max_mtime(existing_st, myentries)
 					rval = True
 				else:
 					# With thin manifest, there's no need to have
@@ -328,6 +332,37 @@ class Manifest(object):
 			raise
 		return rval
 
+	def _apply_max_mtime(self, existing_st, entries):
+		"""
+		Set the Manifest mtime to the max mtime of all relevant files
+		(the existing Manifest mtime is included in order to account for
+		eclass modifications that change DIST entries). This results in a
+		stable/predictable mtime, which is useful when converting thin
+		manifests to thick manifests for distribution via rsync. For
+		portability, the mtime is set with 1 second resolution.
+
+		@param existing_st: stat result for existing Manifest
+		@type existing_st: posix.stat_result
+		@param entries: list of current Manifest2Entry instances
+		@type entries: list
+		"""
+		# Use stat_result[stat.ST_MTIME] for 1 second resolution, since
+		# it always rounds down. Note that stat_result.st_mtime will round
+		# up from 0.999999999 to 1.0 when precision is lost during conversion
+		# from nanosecond resolution to float.
+		max_mtime = None if existing_st is None else existing_st[stat.ST_MTIME]
+		for entry in entries:
+			if entry.type == 'DIST':
+				continue
+			abs_path = (os.path.join(self.pkgdir, 'files', entry.name) if
+				entry.type == 'AUX' else os.path.join(self.pkgdir, entry.name))
+			mtime = os.stat(abs_path)[stat.ST_MTIME]
+			if max_mtime is None or mtime > max_mtime:
+				max_mtime = mtime
+
+		if max_mtime is not None:
+			os.utime(self.getFullname(), (max_mtime, max_mtime))
+
 	def sign(self):
 		""" Sign the Manifest """
 		raise NotImplementedError()
-- 
2.4.6

Re: [gentoo-portage-dev] [PATCH] Manifest.write: stable/predictable Manifest mtime for rsync (bug 557962)

[Brian Dolbec]

On Sun, 16 Aug 2015 20:08:10 -0700
Zac Medico <zmedico@gentoo.org> wrote:

> Use the max mtime of the existing Manifest and the files that the
> updated Manifest contains.
> 
> X-Gentoo-Bug: 557962
> X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=557962
> ---
>  pym/portage/manifest.py | 35 +++++++++++++++++++++++++++++++++++
>  1 file changed, 35 insertions(+)
> 
> diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
> index 3936b9a..f5cf0f5 100644
> --- a/pym/portage/manifest.py
> +++ b/pym/portage/manifest.py
> @@ -6,6 +6,7 @@ from __future__ import unicode_literals
>  import errno
>  import io
>  import re
> +import stat
>  import sys
>  import warnings
>  
> @@ -281,6 +282,7 @@ class Manifest(object):
>  		try:
>  			myentries =
> list(self._createManifestEntries()) update_manifest = True
> +			existing_st = None
>  			if myentries and not force:
>  				try:
>  					f =
> io.open(_unicode_encode(self.getFullname(), @@ -288,6 +290,7 @@ class
> Manifest(object): mode='r', encoding=_encodings['repo.content'],
>  						errors='replace')
>  					oldentries =
> list(self._parseManifestLines(f))
> +					existing_st =
> os.fstat(f.fileno()) f.close()
>  					if len(oldentries) ==
> len(myentries): update_manifest = False
> @@ -309,6 +312,7 @@ class Manifest(object):
>  					# non-empty for all
> currently known use cases. write_atomic(self.getFullname(),
> "".join("%s\n" % _unicode(myentry) for myentry in myentries))
> +
> self._apply_max_mtime(existing_st, myentries) rval = True
>  				else:
>  					# With thin manifest,
> there's no need to have @@ -328,6 +332,37 @@ class Manifest(object):
>  			raise
>  		return rval
>  
> +	def _apply_max_mtime(self, existing_st, entries):
> +		"""
> +		Set the Manifest mtime to the max mtime of all
> relevant files
> +		(the existing Manifest mtime is included in order to
> account for
> +		eclass modifications that change DIST entries). This
> results in a
> +		stable/predictable mtime, which is useful when
> converting thin
> +		manifests to thick manifests for distribution via
> rsync. For
> +		portability, the mtime is set with 1 second
> resolution. +
> +		@param existing_st: stat result for existing Manifest
> +		@type existing_st: posix.stat_result
> +		@param entries: list of current Manifest2Entry
> instances
> +		@type entries: list
> +		"""
> +		# Use stat_result[stat.ST_MTIME] for 1 second
> resolution, since
> +		# it always rounds down. Note that
> stat_result.st_mtime will round
> +		# up from 0.999999999 to 1.0 when precision is lost
> during conversion
> +		# from nanosecond resolution to float.
> +		max_mtime = None if existing_st is None else
> existing_st[stat.ST_MTIME]
> +		for entry in entries:
> +			if entry.type == 'DIST':
> +				continue
> +			abs_path = (os.path.join(self.pkgdir,
> 'files', entry.name) if
> +				entry.type == 'AUX' else
> os.path.join(self.pkgdir, entry.name))
> +			mtime = os.stat(abs_path)[stat.ST_MTIME]
> +			if max_mtime is None or mtime > max_mtime:
> +				max_mtime = mtime
> +
> +		if max_mtime is not None:
> +			os.utime(self.getFullname(), (max_mtime,
> max_mtime)) +
>  	def sign(self):
>  		""" Sign the Manifest """
>  		raise NotImplementedError()

looks fine :)
-- 
Brian Dolbec <dolsen>