[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Michał Górny]

commit:     8c92c14269dfadc87413ebd952397021b1c4dcc7
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 11 21:24:12 2019 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sat Oct 12 06:40:29 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8c92c142

emirrordist: Support specifying custom --layout-conf

Support specifying custom layout.conf file to use.  This makes it
possible to start propagating new mirror layout without exposing it
to users.

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Closes: https://github.com/gentoo/portage/pull/465
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/_emirrordist/Config.py | 6 ++++--
 lib/portage/_emirrordist/main.py   | 6 ++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py
index ace3b0a6a..c1f59f725 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -74,8 +74,10 @@ class Config(object):
 				options.deletion_db, 'deletion')
 
 		self.layout_conf = MirrorLayoutConfig()
-		self.layout_conf.read_from_file(
-				os.path.join(self.distfiles, 'layout.conf'))
+		if options.layout_conf is None:
+			options.layout_conf = os.path.join(self.distfiles,
+					'layout.conf')
+		self.layout_conf.read_from_file(options.layout_conf)
 		self.layouts = self.layout_conf.get_all_layouts()
 
 	def _open_log(self, log_desc, log_path, mode):

diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py
index 0ae45ab6f..ce0c2929f 100644
--- a/lib/portage/_emirrordist/main.py
+++ b/lib/portage/_emirrordist/main.py
@@ -193,6 +193,12 @@ common_options = (
 			"distfiles between layouts",
 		"action"   : "store_true"
 	},
+	{
+		"longopt"  : "--layout-conf",
+		"help"     : "specifies layout.conf file to use instead of "
+			"the one present in the distfiles directory",
+		"metavar"  : "FILE"
+	},
 )
 
 def parse_args(args):

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Michał Górny]

commit:     87c1f0d53eebfeda28f17400b072d8ffa76732dc
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 12 11:04:29 2019 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sun Oct 13 19:38:52 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=87c1f0d5

emirrordist: Refetch only if none of the layouts have the file

Check all layouts for a file before refetching it.  This is helpful
when combined with --layout-conf that specifies the new layout
as primary to cover new files but user-oriented layout.conf still uses
old layout.

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Closes: https://github.com/gentoo/portage/pull/466
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/_emirrordist/FetchTask.py | 54 ++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
index 7b68b7d3e..6f547d397 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -61,23 +61,24 @@ class FetchTask(CompositeTask):
 			self._async_wait()
 			return
 
-		distfile_path = os.path.join(
-			self.config.options.distfiles,
-			self.config.layouts[0].get_path(self.distfile))
-
 		st = None
-		size_ok = False
-		try:
-			st = os.stat(distfile_path)
-		except OSError as e:
-			if e.errno not in (errno.ENOENT, errno.ESTALE):
-				msg = "%s stat failed in %s: %s" % \
-					(self.distfile, "distfiles", e)
-				self.scheduler.output(msg + '\n', background=True,
-					log_path=self._log_path)
-				logging.error(msg)
-		else:
-			size_ok = st.st_size == self.digests["size"]
+		for layout in self.config.layouts:
+			distfile_path = os.path.join(
+				self.config.options.distfiles,
+				layout.get_path(self.distfile))
+			try:
+				st = os.stat(distfile_path)
+			except OSError as e:
+				if e.errno not in (errno.ENOENT, errno.ESTALE):
+					msg = "%s stat failed in %s: %s" % \
+						(self.distfile, "distfiles", e)
+					self.scheduler.output(msg + '\n', background=True,
+						log_path=self._log_path)
+					logging.error(msg)
+			else:
+				break
+
+		size_ok = st is not None and st.st_size == self.digests["size"]
 
 		if not size_ok:
 			if self.config.options.dry_run:
@@ -88,13 +89,20 @@ class FetchTask(CompositeTask):
 				# Do the unlink in order to ensure that the path is clear,
 				# even if stat raised ENOENT, since a broken symlink can
 				# trigger ENOENT.
-				if self._unlink_file(distfile_path, "distfiles"):
-					if st is not None:
-						logging.debug(("delete '%s' with "
-							"wrong size from distfiles") % (self.distfile,))
-				else:
-					self.config.log_failure("%s\t%s\t%s" %
-						(self.cpv, self.distfile, "unlink failed in distfiles"))
+				unlink_success = True
+				for layout in self.config.layouts:
+					unlink_path = os.path.join(
+						self.config.options.distfiles,
+						layout.get_path(self.distfile))
+					if self._unlink_file(unlink_path, "distfiles"):
+						if st is not None:
+							logging.debug(("delete '%s' with "
+								"wrong size from distfiles") % (self.distfile,))
+					else:
+						self.config.log_failure("%s\t%s\t%s" %
+							(self.cpv, self.distfile, "unlink failed in distfiles"))
+						unlink_success = False
+				if not unlink_success:
 					self.returncode = os.EX_OK
 					self._async_wait()
 					return

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Michał Górny]

commit:     4e9f04a1e11e84a8c513ee334cf2bc1c013d8c11
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Thu Oct 17 12:05:30 2019 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Thu Oct 17 18:08:58 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=4e9f04a1

emirrordist: Delete potential symlinks for all layouts

Unlink for all distfile layouts rather than for all but primary.  This
wrongly assumed that the primary layout will always be removed
as a result of earlier code.  However, the rename() call correctly
assumes that the primary layout may be an old symlink, and operates
on realpath().  Therefore, the real file from a secondary layout may
be renamed, leaving dangling symlink.

Bug: https://bugs.gentoo.org/697906
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/_emirrordist/DeletionTask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py
index db5ac5ffb..a4bb29419 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -102,7 +102,7 @@ class DeletionTask(CompositeTask):
 
 	def _delete_links(self):
 		success = True
-		for layout in self.config.layouts[1:]:
+		for layout in self.config.layouts:
 			distfile_path = os.path.join(
 				self.config.options.distfiles,
 				layout.get_path(self.distfile))

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Michał Górny]

commit:     b4bb946e65660db34be7ce67f4e1ed00df47fd53
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 09:53:30 2019 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Oct 21 07:39:38 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b4bb946e

emirrordist: Report all stat() errors instead of first one

When DeletionIterator fails, report all stat() errors.  Reporting
just the first one results in confusing logs, suggesting that one
of the location did not exist but the other existed and was removed.

Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/_emirrordist/DeletionIterator.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py
index cd773b3c8..dab6eaea2 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -25,20 +25,20 @@ class DeletionIterator(object):
 			distfiles_set.update(layout.get_filenames(distdir))
 		for filename in distfiles_set:
 			# require at least one successful stat()
-			first_exception = None
+			exceptions = []
 			for layout in reversed(self._config.layouts):
 				try:
 					st = os.stat(
 							os.path.join(distdir, layout.get_path(filename)))
 				except OSError as e:
-					first_exception = e
+					exceptions.append(e)
 				else:
 					if stat.S_ISREG(st.st_mode):
 						break
 			else:
-				if first_exception is not None:
+				if exceptions:
 					logging.error("stat failed on '%s' in distfiles: %s\n" %
-						(filename, first_exception))
+						(filename, '; '.join(str(x) for x in exceptions)))
 				continue
 
 			if filename in file_owners:

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Michał Górny]

commit:     dc095fc06d323849a6fe47ae3391bc0356439b65
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 20 10:22:11 2019 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Oct 21 08:09:31 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=dc095fc0

emirrordist: Clean dangling symlinks up

Bug: https://bugs.gentoo.org/697906
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/_emirrordist/DeletionIterator.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py
index dab6eaea2..5c193911a 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -27,11 +27,16 @@ class DeletionIterator(object):
 			# require at least one successful stat()
 			exceptions = []
 			for layout in reversed(self._config.layouts):
+				path = os.path.join(distdir, layout.get_path(filename))
 				try:
-					st = os.stat(
-							os.path.join(distdir, layout.get_path(filename)))
+					st = os.stat(path)
 				except OSError as e:
-					exceptions.append(e)
+					# is it a dangling symlink?
+					try:
+						if os.path.islink(path):
+							os.unlink(path)
+					except OSError as e:
+						exceptions.append(e)
 				else:
 					if stat.S_ISREG(st.st_mode):
 						break

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Michał Górny]

commit:     af9c77bd037694164755d06034a74456ec45e3ab
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 21 08:41:54 2019 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Mon Oct 21 13:38:26 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=af9c77bd

emirrordist: Pass path from DeletionIterator to DeletionTask

Since DeletionIterator needs to stat the distfile and therefore find
one working path for it, pass it to DeletionTask instead of recomputing
it there.  This also fixes wrongly assuming that first layout will
always be correct.

Bug: https://bugs.gentoo.org/697890
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 lib/portage/_emirrordist/DeletionIterator.py |  2 ++
 lib/portage/_emirrordist/DeletionTask.py     | 14 +++++---------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py
index 5c193911a..3cbff2c3a 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -72,6 +72,7 @@ class DeletionIterator(object):
 
 					yield DeletionTask(background=True,
 						distfile=filename,
+						distfile_path=path,
 						config=self._config)
 
 				else:
@@ -85,6 +86,7 @@ class DeletionIterator(object):
 
 						yield DeletionTask(background=True,
 							distfile=filename,
+							distfile_path=path,
 							config=self._config)
 
 		if deletion_db is not None:

diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py
index a4bb29419..4e9c26ca2 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -10,14 +10,9 @@ from _emerge.CompositeTask import CompositeTask
 
 class DeletionTask(CompositeTask):
 
-	__slots__ = ('distfile', 'config')
+	__slots__ = ('distfile', 'distfile_path', 'config')
 
 	def _start(self):
-
-		distfile_path = os.path.join(
-			self.config.options.distfiles,
-			self.config.layouts[0].get_path(self.distfile))
-
 		if self.config.options.recycle_dir is not None:
 			recycle_path = os.path.join(
 				self.config.options.recycle_dir, self.distfile)
@@ -29,7 +24,8 @@ class DeletionTask(CompositeTask):
 					"distfiles to recycle") % self.distfile)
 				try:
 					# note: distfile_path can be a symlink here
-					os.rename(os.path.realpath(distfile_path), recycle_path)
+					os.rename(os.path.realpath(self.distfile_path),
+							recycle_path)
 				except OSError as e:
 					if e.errno != errno.EXDEV:
 						logging.error(("rename %s from distfiles to "
@@ -40,7 +36,7 @@ class DeletionTask(CompositeTask):
 					return
 
 				self._start_task(
-					FileCopier(src_path=distfile_path,
+					FileCopier(src_path=self.distfile_path,
 						dest_path=recycle_path,
 						background=False),
 					self._recycle_copier_exit)
@@ -55,7 +51,7 @@ class DeletionTask(CompositeTask):
 			logging.debug(("delete '%s' from "
 				"distfiles") % self.distfile)
 			try:
-				os.unlink(distfile_path)
+				os.unlink(self.distfile_path)
 			except OSError as e:
 				if e.errno not in (errno.ENOENT, errno.ESTALE):
 					logging.error("%s unlink failed in distfiles: %s" %

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Zac Medico]

commit:     e6190145c18a275f9e44618deadb9763631d6c19
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Nov  5 19:01:20 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Nov  5 19:03:21 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e6190145

emirrordist: _recycle_copier_exit UnboundLocalError (bug 699392)

Bug: https://bugs.gentoo.org/699392
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_emirrordist/DeletionTask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py
index 4e9c26ca2..2fdafd59e 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -85,7 +85,7 @@ class DeletionTask(CompositeTask):
 
 		else:
 			logging.error(("%s copy from distfiles "
-				"to recycle failed: %s") % (self.distfile, e))
+				"to recycle failed") % (self.distfile,))
 			success = False
 
 		if success:

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Zac Medico]

commit:     47e0d5b5390873edd0020245307b0febad84a5b5
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Nov  6 19:56:52 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Nov  6 20:03:42 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=47e0d5b5

emirrordist: _make_layout_links msg UnboundLocalError

Fixes: 0d34d89d5028 ("emirrordist: Implement mirror layout.conf support")
Bug: https://bugs.gentoo.org/699392
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_emirrordist/FetchTask.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
index 6f547d397..ca573fa49 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -585,6 +585,8 @@ class FetchTask(CompositeTask):
 			self._success()
 			self.returncode = os.EX_OK
 		else:
+			msg = "failed to create distfiles layout {}".format(
+				"symlink" if self.config.options.symlinks else "hardlink")
 			self.config.log_failure("%s\t%s\t%s" %
 				(self.cpv, self.distfile, msg))
 			self.config.file_failures[self.distfile] = self.cpv

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Zac Medico]

commit:     e5c7b9e292c81c74c6b016aa03ef76214515f0bd
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Nov  6 08:03:48 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Nov  6 20:05:27 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=e5c7b9e2

emaint: log FileCopier exceptions

Since FileCopier now captures exceptions instead of showing
a traceback, it's necessary to explicitly log them.

Bug: https://bugs.gentoo.org/699400
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_emirrordist/DeletionTask.py | 2 +-
 lib/portage/_emirrordist/FetchTask.py    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py
index 2fdafd59e..5eb01d840 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -85,7 +85,7 @@ class DeletionTask(CompositeTask):
 
 		else:
 			logging.error(("%s copy from distfiles "
-				"to recycle failed") % (self.distfile,))
+				"to recycle failed: %s") % (self.distfile, copier.future.exception()))
 			success = False
 
 		if success:

diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
index ca573fa49..0441fc677 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -372,8 +372,8 @@ class FetchTask(CompositeTask):
 
 		current_mirror = self._current_mirror
 		if copier.returncode != os.EX_OK:
-			msg = "%s %s copy failed unexpectedly" % \
-				(self.distfile, current_mirror.name)
+			msg = "%s %s copy failed unexpectedly: %s" % \
+				(self.distfile, current_mirror.name, copier.future.exception())
 			self.scheduler.output(msg + '\n', background=True,
 				log_path=self._log_path)
 			logging.error(msg)
@@ -549,8 +549,8 @@ class FetchTask(CompositeTask):
 			self._make_layout_links()
 		else:
 			# out of space?
-			msg = "%s %s copy failed unexpectedly" % \
-				(self.distfile, self._fetch_tmp_dir_info)
+			msg = "%s %s copy failed unexpectedly: %s" % \
+				(self.distfile, self._fetch_tmp_dir_info, copier.future.exception())
 			self.scheduler.output(msg + '\n', background=True,
 				log_path=self._log_path)
 			logging.error(msg)

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Zac Medico]

commit:     1db30c8a1dc2215425a2d1036effc749c876c1b4
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  3 19:05:54 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  3 19:22:06 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1db30c8a

lib/portage/_emirrordist/main.py: drop unused-import

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_emirrordist/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py
index 23f6468c5..8d00a05f5 100644
--- a/lib/portage/_emirrordist/main.py
+++ b/lib/portage/_emirrordist/main.py
@@ -7,7 +7,7 @@ import sys
 
 import portage
 from portage import os
-from portage.util import normalize_path, writemsg_level, _recursive_file_list
+from portage.util import normalize_path, _recursive_file_list
 from portage.util._async.run_main_scheduler import run_main_scheduler
 from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.util._eventloop.global_event_loop import global_event_loop

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Zac Medico]

commit:     1f43925766b641158aa98e08a58385e65d1ac8bb
Author:     Aaron Bauman <bman <AT> gentoo <DOT> org>
AuthorDate: Mon Aug  3 20:20:24 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug  3 21:25:52 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1f439257

lib/portage/_emirrordist/Config.py: drop unused-import

Signed-off-by: Aaron Bauman <bman <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_emirrordist/Config.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py
index 56732089e..4bee4f45e 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -7,7 +7,6 @@ import logging
 import shelve
 import time
 
-import portage
 from portage import os
 from portage.package.ebuild.fetch import MirrorLayoutConfig
 from portage.util import grabdict, grablines

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Zac Medico]

commit:     8b5f9678200964c3cbb2eb06b58ebfa138fbac45
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jan 22 01:00:16 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jan 22 01:11:35 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b5f9678

emirrordist: prevent distfiles_db _pkg_str pickle problems

When storing a cpv as a shelve value, convert _pkg_str to str
in order to prevent pickle problems.

Bug: https://bugs.gentoo.org/766459
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_emirrordist/FetchTask.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
index 41f96b962..997762082 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -44,7 +44,8 @@ class FetchTask(CompositeTask):
 		if self.config.distfiles_db is not None and \
 			self.distfile not in self.config.distfiles_db:
 			self._previously_added = False
-			self.config.distfiles_db[self.distfile] = self.cpv
+			# Convert _pkg_str to str in order to prevent pickle problems.
+			self.config.distfiles_db[self.distfile] = str(self.cpv)
 
 		if not self._have_needed_digests():
 			msg = "incomplete digests: %s" % " ".join(self.digests)

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Zac Medico]

commit:     0f3070198c56a8bc3b23e3965ab61136d3de76ae
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 27 03:49:29 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Feb 27 03:54:39 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0f307019

emirrordist: support minimal object as options for use in unit tests

Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_emirrordist/Config.py | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..1c7a27d66 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import copy
@@ -25,24 +25,24 @@ class Config:
 		self.start_time = time.time()
 		self._open_files = []
 
-		self.log_success = self._open_log('success', options.success_log, 'a')
-		self.log_failure = self._open_log('failure', options.failure_log, 'a')
+		self.log_success = self._open_log('success', getattr(options, 'success_log', None), 'a')
+		self.log_failure = self._open_log('failure', getattr(options, 'failure_log', None), 'a')
 
 		self.distfiles = None
-		if options.distfiles is not None:
+		if getattr(options, 'distfiles', None) is not None:
 			self.distfiles = options.distfiles
 
 		self.mirrors = copy.copy(portdb.settings.thirdpartymirrors())
 
-		if options.mirror_overrides is not None:
+		if getattr(options, 'mirror_overrides', None) is not None:
 			self.mirrors.update(grabdict(options.mirror_overrides))
 
-		if options.mirror_skip is not None:
+		if getattr(options, 'mirror_skip', None) is not None:
 			for x in options.mirror_skip.split(","):
 				self.mirrors[x] = []
 
 		self.whitelist = None
-		if options.whitelist_from is not None:
+		if getattr(options, 'whitelist_from', None) is not None:
 			self.whitelist = set()
 			for filename in options.whitelist_from:
 				for line in grablines(filename):
@@ -51,27 +51,27 @@ class Config:
 						self.whitelist.add(line)
 
 		self.restrict_mirror_exemptions = None
-		if options.restrict_mirror_exemptions is not None:
+		if getattr(options, 'restrict_mirror_exemptions', None) is not None:
 			self.restrict_mirror_exemptions = frozenset(
 				options.restrict_mirror_exemptions.split(","))
 
 		self.recycle_db = None
-		if options.recycle_db is not None:
+		if getattr(options, 'recycle_db', None) is not None:
 			self.recycle_db = self._open_shelve(
 				options.recycle_db, 'recycle')
 
 		self.distfiles_db = None
-		if options.distfiles_db is not None:
+		if getattr(options, 'distfiles_db', None) is not None:
 			self.distfiles_db = self._open_shelve(
 				options.distfiles_db, 'distfiles')
 
 		self.deletion_db = None
-		if options.deletion_db is not None:
+		if getattr(options, 'deletion_db', None) is not None:
 			self.deletion_db = self._open_shelve(
 				options.deletion_db, 'deletion')
 
 		self.layout_conf = MirrorLayoutConfig()
-		if options.layout_conf is None:
+		if getattr(options, 'layout_conf', None) is None:
 			options.layout_conf = os.path.join(self.distfiles,
 					'layout.conf')
 		self.layout_conf.read_from_file(options.layout_conf)
@@ -79,7 +79,7 @@ class Config:
 
 	def _open_log(self, log_desc, log_path, mode):
 
-		if log_path is None or self.options.dry_run:
+		if log_path is None or getattr(self.options, 'dry_run', False):
 			log_func = logging.info
 			line_format = "%s: %%s" % log_desc
 			add_newline = False
@@ -106,12 +106,13 @@ class Config:
 			self._log_func(self._line_format % (msg,))
 
 	def _open_shelve(self, db_file, db_desc):
-		if self.options.dry_run:
+		dry_run = getattr(self.options, 'dry_run', False)
+		if dry_run:
 			open_flag = "r"
 		else:
 			open_flag = "c"
 
-		if self.options.dry_run and not os.path.exists(db_file):
+		if dry_run and not os.path.exists(db_file):
 			db = {}
 		else:
 			try:
@@ -123,7 +124,7 @@ class Config:
 					from bsddb3 import dbshelve
 					db = dbshelve.open(db_file, flags=open_flag)
 
-		if self.options.dry_run:
+		if dry_run:
 			logging.warning("dry-run: %s db opened in readonly mode" % db_desc)
 			if not isinstance(db, dict):
 				volatile_db = dict((k, db[k]) for k in db)

[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/

[Zac Medico]

commit:     3c6ea3c726ebcac71912c3f8336af49d3e9abd38
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Thu May 13 10:21:41 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 24 05:21:58 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3c6ea3c7

Support selective fetch restriction in emirrordist

Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/_emirrordist/FetchIterator.py | 68 ++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 29 deletions(-)

diff --git a/lib/portage/_emirrordist/FetchIterator.py b/lib/portage/_emirrordist/FetchIterator.py
index 8ca6a2eac..e23d742f3 100644
--- a/lib/portage/_emirrordist/FetchIterator.py
+++ b/lib/portage/_emirrordist/FetchIterator.py
@@ -171,10 +171,6 @@ def _async_fetch_tasks(config, hash_filter, repo_config, digests_future, cpv,
 			result.set_result(fetch_tasks)
 			return
 
-		if "fetch" in restrict:
-			result.set_result(fetch_tasks)
-			return
-
 		try:
 			uri_map = fetch_map_result.result()
 		except PortageException as e:
@@ -187,28 +183,42 @@ def _async_fetch_tasks(config, hash_filter, repo_config, digests_future, cpv,
 			result.set_result(fetch_tasks)
 			return
 
-		if "mirror" in restrict:
-			skip = False
-			if config.restrict_mirror_exemptions is not None:
-				new_uri_map = {}
-				for filename, uri_tuple in uri_map.items():
-					for uri in uri_tuple:
-						if uri[:9] == "mirror://":
-							i = uri.find("/", 9)
-							if i != -1 and uri[9:i].strip("/") in \
-								config.restrict_mirror_exemptions:
-								new_uri_map[filename] = uri_tuple
-								break
-				if new_uri_map:
-					uri_map = new_uri_map
-				else:
-					skip = True
-			else:
-				skip = True
-
-			if skip:
-				result.set_result(fetch_tasks)
-				return
+		new_uri_map = {}
+		restrict_fetch = "fetch" in restrict
+		restrict_mirror = restrict_fetch or "mirror" in restrict
+		for filename, uri_tuple in uri_map.items():
+			new_uris = []
+			for uri in uri_tuple:
+				override_mirror = uri.startswith("mirror+")
+				override_fetch = override_mirror or uri.startswith("fetch+")
+				if override_fetch:
+					uri = uri.partition("+")[2]
+
+				# skip fetch-restricted files unless overriden via fetch+
+				# or mirror+
+				if restrict_fetch and not override_fetch:
+					continue
+				# skip mirror-restricted files unless override via mirror+
+				# or in config_mirror_exemptions
+				if restrict_mirror and not override_mirror:
+					if (config.restrict_mirror_exemptions is None or
+							not uri.startswith("mirror://")):
+						continue
+					mirror_name = uri.split('/', 3)[2]
+					if mirror_name not in config.restrict_mirror_exemptions:
+						continue
+				# if neither fetch or mirror restriction applies to the URI
+				# or it is exempted from them, readd it (with fetch+/mirror+
+				# prefix stripped)
+				new_uris.append(uri)
+
+			# if we've gotten any new URIs, then we readd the file
+			if new_uris:
+				new_uri_map[filename] = new_uris
+
+		if not new_uri_map:
+			result.set_result(fetch_tasks)
+			return
 
 		# Parse Manifest for this cp if we haven't yet.
 		try:
@@ -221,7 +231,7 @@ def _async_fetch_tasks(config, hash_filter, repo_config, digests_future, cpv,
 					getTypeDigests("DIST")
 		except (EnvironmentError, PortageException) as e:
 			digests_future.done() or digests_future.set_exception(e)
-			for filename in uri_map:
+			for filename in new_uri_map:
 				config.log_failure(
 					"%s\t%s\tManifest exception %s" %
 					(cpv, filename, e))
@@ -232,14 +242,14 @@ def _async_fetch_tasks(config, hash_filter, repo_config, digests_future, cpv,
 			digests_future.done() or digests_future.set_result(digests)
 
 		if not digests:
-			for filename in uri_map:
+			for filename in new_uri_map:
 				config.log_failure("%s\t%s\tdigest entry missing" %
 					(cpv, filename))
 				config.file_failures[filename] = cpv
 			result.set_result(fetch_tasks)
 			return
 
-		for filename, uri_tuple in uri_map.items():
+		for filename, uri_tuple in new_uri_map.items():
 			file_digests = digests.get(filename)
 			if file_digests is None:
 				config.log_failure("%s\t%s\tdigest entry missing" %