[gentoo-commits] proj/portage:master commit in: pym/_emerge/resolver/, pym/portage/package/ebuild/, pym/portage/dbapi/, ...

[gentoo-commits] proj/portage:master commit in: pym/_emerge/resolver/, pym/portage/package/ebuild/, pym/portage/dbapi/, ...

[Zac Medico]

commit:     d3f704a425a50b5cfa997a25866929b30f1b7d0f
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Nov 17 23:10:13 2011 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Nov 17 23:10:13 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d3f704a4

Skip the "resume after portage update" routine.

Instead, finish the whole job using a copy of the currently running
instance. This allows us to avoid the complexities of emerge --resume,
such as the differences in option handling between different portage
versions, as reported in bug #390819.

---
 pym/_emerge/Scheduler.py               |  143 +++++++++-----------------------
 pym/_emerge/depgraph.py                |    2 -
 pym/_emerge/resolver/output.py         |   21 -----
 pym/_emerge/resolver/output_helpers.py |    1 -
 pym/portage/dbapi/_MergeProcess.py     |   58 -------------
 pym/portage/package/ebuild/config.py   |   17 +++--
 pym/portage/package/ebuild/doebuild.py |   44 ++++++++++-
 7 files changed, 92 insertions(+), 194 deletions(-)

diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index 3b53a37..393eeb6 100644
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -29,7 +29,8 @@ from portage._sets.base import InternalPackageSet
 from portage.util import ensure_dirs, writemsg, writemsg_level
 from portage.package.ebuild.digestcheck import digestcheck
 from portage.package.ebuild.digestgen import digestgen
-from portage.package.ebuild.doebuild import _check_temp_dir
+from portage.package.ebuild.doebuild import (_check_temp_dir,
+	_prepare_self_update)
 from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs
 
 import _emerge
@@ -75,12 +76,9 @@ class Scheduler(PollScheduler):
 		frozenset(["--pretend",
 		"--fetchonly", "--fetch-all-uri"])
 
-	_opts_no_restart = frozenset(["--buildpkgonly",
+	_opts_no_self_reinstall = frozenset(["--buildpkgonly",
 		"--fetchonly", "--fetch-all-uri", "--pretend"])
 
-	_bad_resume_opts = set(["--ask", "--changelog",
-		"--resume", "--skipfirst"])
-
 	class _iface_class(SlotObject):
 		__slots__ = ("fetch",
 			"output", "register", "schedule",
@@ -289,6 +287,38 @@ class Scheduler(PollScheduler):
 			self._running_portage = self._pkg(cpv, "installed",
 				self._running_root, installed=True)
 
+	def _handle_self_update(self):
+		"""
+		If portage is updating itself, create temporary
+		copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH in order
+		to avoid relying on the new versions which may be
+		incompatible. Register an atexit hook to clean up the
+		temporary directories. Pre-load elog modules here since
+		we won't be able to later if they get unmerged (happens
+		when namespace changes).
+		"""
+
+		if self._opts_no_self_reinstall.intersection(self.myopts):
+			return
+
+		for x in self._mergelist:
+			if not isinstance(x, Package):
+				continue
+			if x.operation != "merge":
+				continue
+			if x.root != self._running_root.root:
+				continue
+			if not portage.dep.match_from_list(
+				portage.const.PORTAGE_PACKAGE_ATOM, [x]):
+				continue
+			if self._running_portage is None or \
+				self._running_portage.cpv != x.cpv or \
+				'9999' in x.cpv or \
+				'git' in x.inherited or \
+				'git-2' in x.inherited:
+				_prepare_self_update(self.settings)
+			break
+
 	def _terminate_tasks(self):
 		self._status_display.quiet = True
 		while self._running_tasks:
@@ -785,100 +815,6 @@ class Scheduler(PollScheduler):
 
 		return prefetcher
 
-	def _is_restart_scheduled(self):
-		"""
-		Check if the merge list contains a replacement
-		for the current running instance, that will result
-		in restart after merge.
-		@rtype: bool
-		@returns: True if a restart is scheduled, False otherwise.
-		"""
-		if self._opts_no_restart.intersection(self.myopts):
-			return False
-
-		mergelist = self._mergelist
-
-		for i, pkg in enumerate(mergelist):
-			if self._is_restart_necessary(pkg) and \
-				i != len(mergelist) - 1:
-				return True
-
-		return False
-
-	def _is_restart_necessary(self, pkg):
-		"""
-		@return: True if merging the given package
-			requires restart, False otherwise.
-		"""
-
-		# Figure out if we need a restart.
-		if pkg.root == self._running_root.root and \
-			portage.match_from_list(
-			portage.const.PORTAGE_PACKAGE_ATOM, [pkg]):
-			if self._running_portage is None:
-				return True
-			elif pkg.cpv != self._running_portage.cpv or \
-				'9999' in pkg.cpv or \
-				'git' in pkg.inherited or \
-				'git-2' in pkg.inherited:
-				return True
-		return False
-
-	def _restart_if_necessary(self, pkg):
-		"""
-		Use execv() to restart emerge. This happens
-		if portage upgrades itself and there are
-		remaining packages in the list.
-		"""
-
-		if self._opts_no_restart.intersection(self.myopts):
-			return
-
-		if not self._is_restart_necessary(pkg):
-			return
-
-		if pkg == self._mergelist[-1]:
-			return
-
-		self._main_loop_cleanup()
-
-		logger = self._logger
-		pkg_count = self._pkg_count
-		mtimedb = self._mtimedb
-		bad_resume_opts = self._bad_resume_opts
-
-		logger.log(" ::: completed emerge (%s of %s) %s to %s" % \
-			(pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg.root))
-
-		logger.log(" *** RESTARTING " + \
-			"emerge via exec() after change of " + \
-			"portage version.")
-
-		mtimedb["resume"]["mergelist"].remove(list(pkg))
-		mtimedb.commit()
-		portage.run_exitfuncs()
-		# Don't trust sys.argv[0] here because eselect-python may modify it.
-		emerge_binary = os.path.join(portage.const.PORTAGE_BIN_PATH, 'emerge')
-		mynewargv = [emerge_binary, "--resume"]
-		resume_opts = self.myopts.copy()
-		# For automatic resume, we need to prevent
-		# any of bad_resume_opts from leaking in
-		# via EMERGE_DEFAULT_OPTS.
-		resume_opts["--ignore-default-opts"] = True
-		for myopt, myarg in resume_opts.items():
-			if myopt not in bad_resume_opts:
-				if myarg is True:
-					mynewargv.append(myopt)
-				elif isinstance(myarg, list):
-					# arguments like --exclude that use 'append' action
-					for x in myarg:
-						mynewargv.append("%s=%s" % (myopt, x))
-				else:
-					mynewargv.append("%s=%s" % (myopt, myarg))
-		# priority only needs to be adjusted on the first run
-		os.environ["PORTAGE_NICENESS"] = "0"
-		os.execv(mynewargv[0], mynewargv)
-
 	def _run_pkg_pretend(self):
 		"""
 		Since pkg_pretend output may be important, this method sends all
@@ -1034,6 +970,8 @@ class Scheduler(PollScheduler):
 		except self._unknown_internal_error:
 			return 1
 
+		self._handle_self_update()
+
 		for root in self.trees:
 			root_config = self.trees[root]["root_config"]
 
@@ -1379,8 +1317,6 @@ class Scheduler(PollScheduler):
 		if pkg.installed:
 			return
 
-		self._restart_if_necessary(pkg)
-
 		# Call mtimedb.commit() after each merge so that
 		# --resume still works after being interrupted
 		# by reboot, sigkill or similar.
@@ -1585,10 +1521,7 @@ class Scheduler(PollScheduler):
 
 	def _main_loop(self):
 
-		# Only allow 1 job max if a restart is scheduled
-		# due to portage update.
-		if self._is_restart_scheduled() or \
-			self._opts_no_background.intersection(self.myopts):
+		if self._opts_no_background.intersection(self.myopts):
 			self._set_max_jobs(1)
 
 		while self._schedule():

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index fda335f..65fe1fd 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -100,8 +100,6 @@ class _frozen_depgraph_config(object):
 			self.edebug = 1
 		self.spinner = spinner
 		self._running_root = trees[trees._running_eroot]["root_config"]
-		self._opts_no_restart = frozenset(["--buildpkgonly",
-			"--fetchonly", "--fetch-all-uri", "--pretend"])
 		self.pkgsettings = {}
 		self.trees = {}
 		self._trees_orig = trees

diff --git a/pym/_emerge/resolver/output.py b/pym/_emerge/resolver/output.py
index eed3019..6f1c76c 100644
--- a/pym/_emerge/resolver/output.py
+++ b/pym/_emerge/resolver/output.py
@@ -865,27 +865,6 @@ class Display(object):
 					continue
 				self.print_msg.append((myprint, self.verboseadd, self.repoadd))
 
-				if not self.conf.tree_display \
-					and not self.conf.no_restart \
-					and pkg.root == self.conf.running_root.root \
-					and match_from_list(PORTAGE_PACKAGE_ATOM, [pkg]) \
-					and not self.conf.quiet:
-
-					if not self.vardb.cpv_exists(pkg.cpv) or \
-						'9999' in pkg.cpv or \
-						'git' in pkg.inherited or \
-						'git-2' in pkg.inherited:
-						if mylist_index < len(mylist) - 1:
-							self.print_msg.append(
-								colorize(
-									"WARN", "*** Portage will stop merging "
-									"at this point and reload itself,"
-									)
-								)
-							self.print_msg.append(
-								colorize("WARN", "    then resume the merge.")
-								)
-
 		show_repos = repoadd_set and repoadd_set != set(["0"])
 
 		# now finally print out the messages

diff --git a/pym/_emerge/resolver/output_helpers.py b/pym/_emerge/resolver/output_helpers.py
index b3cdbc4..dd26534 100644
--- a/pym/_emerge/resolver/output_helpers.py
+++ b/pym/_emerge/resolver/output_helpers.py
@@ -198,7 +198,6 @@ class _DisplayConfig(object):
 		self.print_use_string = self.verbosity != 1 or "--verbose" in frozen_config.myopts
 		self.changelog = "--changelog" in frozen_config.myopts
 		self.edebug = frozen_config.edebug
-		self.no_restart = frozen_config._opts_no_restart.intersection(frozen_config.myopts)
 		self.unordered_display = "--unordered-display" in frozen_config.myopts
 
 		mywidth = 130

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 34ed031..c9b6288 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -2,20 +2,14 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import io
-import shutil
 import signal
-import tempfile
 import traceback
 
 import errno
 import fcntl
 import portage
 from portage import os, _unicode_decode
-from portage.const import PORTAGE_PACKAGE_ATOM
-from portage.dep import match_from_list
 import portage.elog.messages
-from portage.elog import _preload_elog_modules
-from portage.util import ensure_dirs
 from _emerge.PollConstants import PollConstants
 from _emerge.SpawnProcess import SpawnProcess
 
@@ -46,8 +40,6 @@ class MergeProcess(SpawnProcess):
 			settings.reset()
 			settings.setcpv(cpv, mydb=self.mydbapi)
 
-		if not self.unmerge:
-			self._handle_self_reinstall()
 		super(MergeProcess, self)._start()
 
 	def _lock_vdb(self):
@@ -69,56 +61,6 @@ class MergeProcess(SpawnProcess):
 			self.vartree.dbapi.unlock()
 			self._locked_vdb = False
 
-	def _handle_self_reinstall(self):
-		"""
-		If portage is reinstalling itself, create temporary
-		copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH in order
-		to avoid relying on the new versions which may be
-		incompatible. Register an atexit hook to clean up the
-		temporary directories. Pre-load elog modules here since
-		we won't be able to later if they get unmerged (happens
-		when namespace changes).
-		"""
-
-		settings = self.settings
-		cpv = settings.mycpv
-		reinstall_self = False
-		if self.settings["ROOT"] == "/" and \
-			match_from_list(PORTAGE_PACKAGE_ATOM, [cpv]):
-			inherited = frozenset(self.settings.get('INHERITED', '').split())
-			if not self.vartree.dbapi.cpv_exists(cpv) or \
-				'9999' in cpv or \
-				'git' in inherited or \
-				'git-2' in inherited:
-				reinstall_self = True
-
-		if reinstall_self:
-			# Load lazily referenced portage submodules into memory,
-			# so imports won't fail during portage upgrade/downgrade.
-			_preload_elog_modules(self.settings)
-			portage.proxy.lazyimport._preload_portage_submodules()
-
-			# Make the temp directory inside $PORTAGE_TMPDIR/portage, since
-			# it's common for /tmp and /var/tmp to be mounted with the
-			# "noexec" option (see bug #346899).
-			build_prefix = os.path.join(settings["PORTAGE_TMPDIR"], "portage")
-			ensure_dirs(build_prefix)
-			base_path_tmp = tempfile.mkdtemp(
-				"", "._portage_reinstall_.", build_prefix)
-			portage.process.atexit_register(shutil.rmtree, base_path_tmp)
-			dir_perms = 0o755
-			for subdir in "bin", "pym":
-				var_name = "PORTAGE_%s_PATH" % subdir.upper()
-				var_orig = settings[var_name]
-				var_new = os.path.join(base_path_tmp, subdir)
-				settings[var_name] = var_new
-				settings.backup_changes(var_name)
-				shutil.copytree(var_orig, var_new, symlinks=True)
-				os.chmod(var_new, dir_perms)
-			portage._bin_path = settings['PORTAGE_BIN_PATH']
-			portage._pym_path = settings['PORTAGE_PYM_PATH']
-			os.chmod(base_path_tmp, dir_perms)
-
 	def _elog_output_handler(self, fd, event):
 		output = None
 		if event & PollConstants.POLLIN:

diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
index 765a4f7..6d5de92 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -22,7 +22,7 @@ from portage import bsd_chflags, \
 	load_mod, os, selinux, _unicode_decode
 from portage.const import CACHE_PATH, \
 	DEPCACHE_PATH, INCREMENTALS, MAKE_CONF_FILE, \
-	MODULES_FILE_PATH, PORTAGE_BIN_PATH, PORTAGE_PYM_PATH, \
+	MODULES_FILE_PATH, \
 	PRIVATE_PATH, PROFILE_PATH, USER_CONFIG_PATH, \
 	USER_VIRTUALS_FILE
 from portage.const import _SANDBOX_COMPAT_LEVEL
@@ -722,11 +722,6 @@ class config(object):
 					self["USERLAND"] = "GNU"
 				self.backup_changes("USERLAND")
 
-			self["PORTAGE_BIN_PATH"] = PORTAGE_BIN_PATH
-			self.backup_changes("PORTAGE_BIN_PATH")
-			self["PORTAGE_PYM_PATH"] = PORTAGE_PYM_PATH
-			self.backup_changes("PORTAGE_PYM_PATH")
-
 			for var in ("PORTAGE_INST_UID", "PORTAGE_INST_GID"):
 				try:
 					self[var] = str(int(self.get(var, "0")))
@@ -2088,6 +2083,14 @@ class config(object):
 					del x[mykey]
 
 	def __getitem__(self,mykey):
+
+		# These ones point to temporary values when
+		# portage plans to update itself.
+		if mykey == "PORTAGE_BIN_PATH":
+			return portage._bin_path
+		elif mykey == "PORTAGE_PYM_PATH":
+			return portage._pym_path
+
 		for d in self.lookuplist:
 			if mykey in d:
 				return d[mykey]
@@ -2133,6 +2136,8 @@ class config(object):
 
 	def __iter__(self):
 		keys = set()
+		keys.add("PORTAGE_BIN_PATH")
+		keys.add("PORTAGE_PYM_PATH")
 		for d in self.lookuplist:
 			keys.update(d)
 		return iter(keys)

diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index bdfcbcc..49d3e89 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -44,7 +44,7 @@ from portage.dep import Atom, check_required_use, \
 from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \
 	eapi_exports_replace_vars, eapi_has_required_use, \
 	eapi_has_src_prepare_and_src_configure, eapi_has_pkg_pretend
-from portage.elog import elog_process
+from portage.elog import elog_process, _preload_elog_modules
 from portage.elog.messages import eerror, eqawarn
 from portage.exception import DigestException, FileNotFound, \
 	IncorrectParameter, InvalidDependString, PermissionDenied, \
@@ -1027,6 +1027,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0,
 			# qmerge is a special phase that implies noclean.
 			if "noclean" not in mysettings.features:
 				mysettings.features.add("noclean")
+			_handle_self_update(mysettings, vartree.dbapi)
 			#qmerge is specifically not supposed to do a runtime dep check
 			retval = merge(
 				mysettings["CATEGORY"], mysettings["PF"], mysettings["D"],
@@ -1043,6 +1044,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0,
 				# so that it's only called once.
 				elog_process(mysettings.mycpv, mysettings)
 			if retval == os.EX_OK:
+				_handle_self_update(mysettings, vartree.dbapi)
 				retval = merge(mysettings["CATEGORY"], mysettings["PF"],
 					mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"],
 					"build-info"), myroot, mysettings,
@@ -2013,3 +2015,43 @@ def _merge_unicode_error(errors):
 		lines.append("")
 
 	return lines
+
+def _prepare_self_update(settings):
+	# Load lazily referenced portage submodules into memory,
+	# so imports won't fail during portage upgrade/downgrade.
+	_preload_elog_modules(settings)
+	portage.proxy.lazyimport._preload_portage_submodules()
+
+	# Make the temp directory inside $PORTAGE_TMPDIR/portage, since
+	# it's common for /tmp and /var/tmp to be mounted with the
+	# "noexec" option (see bug #346899).
+	build_prefix = os.path.join(settings["PORTAGE_TMPDIR"], "portage")
+	portage.util.ensure_dirs(build_prefix)
+	base_path_tmp = tempfile.mkdtemp(
+		"", "._portage_reinstall_.", build_prefix)
+	portage.process.atexit_register(shutil.rmtree, base_path_tmp)
+
+	orig_bin_path = portage._bin_path
+	portage._bin_path = os.path.join(base_path_tmp, "bin")
+	shutil.copytree(orig_bin_path, portage._bin_path, symlinks=True)
+
+	orig_pym_path = portage._pym_path
+	portage._pym_path = os.path.join(base_path_tmp, "pym")
+	shutil.copytree(orig_pym_path, portage._pym_path, symlinks=True)
+
+	for dir_path in (base_path_tmp, portage._bin_path, portage._pym_path):
+		os.chmod(dir_path, 0o755)
+
+def _handle_self_update(settings, vardb):
+	cpv = settings.mycpv
+	if settings["ROOT"] == "/" and \
+		portage.dep.match_from_list(
+		portage.const.PORTAGE_PACKAGE_ATOM, [cpv]):
+		inherited = frozenset(settings.get('INHERITED', '').split())
+		if not vardb.cpv_exists(cpv) or \
+			'9999' in cpv or \
+			'git' in inherited or \
+			'git-2' in inherited:
+			_prepare_self_update(settings)
+			return True
+	return False