[gentoo-portage-dev] [PATCH] Use cached portage.getpid() function (bug 739540)

[Zac Medico <zmedico@gentoo.org>]

Use the cached portage.getpid() function to avoid unnecessary syscalls,
and update the cache after each call to os.fork() where the fork may
invoke portage APIs.

Bug: https://bugs.gentoo.org/739540
Signed-off-by: Zac Medico <zmedico@gentoo.org>
---
 bin/quickpkg                                     |  2 +-
 lib/_emerge/EbuildBinpkg.py                      |  4 +++-
 lib/_emerge/actions.py                           |  2 +-
 lib/portage/_emirrordist/FetchTask.py            |  4 ++--
 lib/portage/cache/metadata.py                    |  4 +++-
 lib/portage/elog/mod_mail_summary.py             |  5 ++---
 lib/portage/elog/mod_save_summary.py             |  2 +-
 lib/portage/locks.py                             |  2 +-
 lib/portage/package/ebuild/doebuild.py           |  2 +-
 lib/portage/process.py                           | 16 +++++++++++-----
 lib/portage/tests/locks/test_lock_nonblock.py    |  1 +
 .../futures/asyncio/test_wakeup_fd_sigchld.py    |  2 +-
 lib/portage/util/__init__.py                     |  4 ++--
 lib/portage/util/_eventloop/EventLoop.py         |  6 +++---
 .../util/_eventloop/asyncio_event_loop.py        |  4 ++--
 lib/portage/util/_eventloop/global_event_loop.py |  7 +++----
 lib/portage/util/locale.py                       |  1 +
 lib/portage/util/movefile.py                     |  2 +-
 lib/portage/util/socks5.py                       |  2 +-
 lib/portage/xpak.py                              |  2 +-
 20 files changed, 42 insertions(+), 32 deletions(-)

diff --git a/bin/quickpkg b/bin/quickpkg
index be7d1d7af..a171b3bd5 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -111,7 +111,7 @@ def quickpkg_atom(options, infos, arg, eout):
 				vardb.aux_update(cpv, update_metadata)
 			xpdata = xpak.xpak(dblnk.dbdir)
 			binpkg_tmpfile = os.path.join(bintree.pkgdir,
-				cpv + ".tbz2." + str(os.getpid()))
+				cpv + ".tbz2." + str(portage.getpid()))
 			ensure_dirs(os.path.dirname(binpkg_tmpfile))
 			binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2")
 			try:
diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
index 6e098eb8a..879b3a9aa 100644
--- a/lib/_emerge/EbuildBinpkg.py
+++ b/lib/_emerge/EbuildBinpkg.py
@@ -3,6 +3,8 @@
 
 from _emerge.CompositeTask import CompositeTask
 from _emerge.EbuildPhase import EbuildPhase
+
+import portage
 from portage import os
 
 class EbuildBinpkg(CompositeTask):
@@ -17,7 +19,7 @@ class EbuildBinpkg(CompositeTask):
 		root_config = pkg.root_config
 		bintree = root_config.trees["bintree"]
 		binpkg_tmpfile = os.path.join(bintree.pkgdir,
-			pkg.cpv + ".tbz2." + str(os.getpid()))
+			pkg.cpv + ".tbz2." + str(portage.getpid()))
 		bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
 
 		self._binpkg_tmpfile = binpkg_tmpfile
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 063f5d4a0..a4ecfe43d 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -2623,7 +2623,7 @@ def ionice(settings):
 	if not ionice_cmd:
 		return
 
-	variables = {"PID" : str(os.getpid())}
+	variables = {"PID" : str(portage.getpid())}
 	cmd = [varexpand(x, mydict=variables) for x in ionice_cmd]
 
 	try:
diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
index 457ca2ac6..41f96b962 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -415,7 +415,7 @@ class FetchTask(CompositeTask):
 			self._fetch_tmp_dir_info = 'distfiles'
 			distdir = self.config.options.distfiles
 
-		tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
+		tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % portage.getpid()
 
 		variables = {
 			"DISTDIR": distdir,
@@ -622,7 +622,7 @@ class FetchTask(CompositeTask):
 
 		head, tail = os.path.split(dest)
 		hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
-			(tail, os.getpid()))
+			(tail, portage.getpid()))
 
 		try:
 			try:
diff --git a/lib/portage/cache/metadata.py b/lib/portage/cache/metadata.py
index db81b8ba1..bd1b70fa0 100644
--- a/lib/portage/cache/metadata.py
+++ b/lib/portage/cache/metadata.py
@@ -6,6 +6,8 @@ import errno
 import re
 import stat
 from operator import attrgetter
+
+import portage
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
@@ -122,7 +124,7 @@ class database(flat_hash.database):
 
 		s = cpv.rfind("/")
 		fp = os.path.join(self.location,cpv[:s],
-			".update.%i.%s" % (os.getpid(), cpv[s+1:]))
+			".update.%i.%s" % (portage.getpid(), cpv[s+1:]))
 		try:
 			myf = open(_unicode_encode(fp,
 				encoding=_encodings['fs'], errors='strict'), 'wb')
diff --git a/lib/portage/elog/mod_mail_summary.py b/lib/portage/elog/mod_mail_summary.py
index ac260880e..789f55f4d 100644
--- a/lib/portage/elog/mod_mail_summary.py
+++ b/lib/portage/elog/mod_mail_summary.py
@@ -6,7 +6,6 @@ import portage
 from portage.exception import AlarmSignal, PortageException
 from portage.localization import _
 from portage.util import writemsg
-from portage import os
 from portage import _encodings
 from portage import _unicode_decode
 
@@ -22,7 +21,7 @@ def process(mysettings, key, logentries, fulltext):
 		time.strftime("%Y%m%d-%H%M%S %Z", time.localtime(time.time())),
 		encoding=_encodings['content'], errors='replace')
 	header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n") % \
-		{"pkg": key, "pid": os.getpid(), "time": time_str}
+		{"pkg": key, "pid": portage.getpid(), "time": time_str}
 	config_root = mysettings["PORTAGE_CONFIGROOT"]
 
 	# Copy needed variables from the config instance,
@@ -66,7 +65,7 @@ def _finalize(mysettings, items):
 	mysubject = mysubject.replace("${HOST}", socket.getfqdn())
 
 	mybody = _("elog messages for the following packages generated by "
-		"process %(pid)d on host %(host)s:\n") % {"pid": os.getpid(), "host": socket.getfqdn()}
+		"process %(pid)d on host %(host)s:\n") % {"pid": portage.getpid(), "host": socket.getfqdn()}
 	for key in items:
 		mybody += "- %s\n" % key
 
diff --git a/lib/portage/elog/mod_save_summary.py b/lib/portage/elog/mod_save_summary.py
index 946a1ad4c..6e24b608f 100644
--- a/lib/portage/elog/mod_save_summary.py
+++ b/lib/portage/elog/mod_save_summary.py
@@ -79,7 +79,7 @@ def process(mysettings, key, logentries, fulltext):
 		encoding=_encodings['content'], errors='replace')
 	elogfile.write(_(">>> Messages generated by process "
 		"%(pid)d on %(time)s for package %(pkg)s:\n\n") %
-		{"pid": os.getpid(), "time": time_str, "pkg": key})
+		{"pid": portage.getpid(), "time": time_str, "pkg": key})
 	elogfile.write(_unicode_decode(fulltext))
 	elogfile.write("\n")
 	elogfile.close()
diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 701093024..1073343be 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -501,7 +501,7 @@ def unlockfile(mytuple):
 def hardlock_name(path):
 	base, tail = os.path.split(path)
 	return os.path.join(base, ".%s.hardlock-%s-%s" %
-		(tail, portage._decode_argv([os.uname()[1]])[0], os.getpid()))
+		(tail, portage._decode_argv([os.uname()[1]])[0], portage.getpid()))
 
 def hardlink_is_mine(link, lock):
 	try:
diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
index 7bb942966..3b1991b28 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -1178,7 +1178,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0,
 					bintree = portage.db[mysettings['EROOT']]['bintree']
 					mysettings["PORTAGE_BINPKG_TMPFILE"] = \
 						bintree.getname(mysettings.mycpv) + \
-						".%s" % (os.getpid(),)
+						".%s" % (portage.getpid(),)
 					bintree._ensure_dir(os.path.dirname(
 						mysettings["PORTAGE_BINPKG_TMPFILE"]))
 				else:
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 8d4cf164e..48548bacc 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -79,11 +79,11 @@ if _fd_dir is not None:
 					raise
 				return range(max_fd_limit)
 
-elif os.path.isdir("/proc/%s/fd" % os.getpid()):
+elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
 	# In order for this function to work in forked subprocesses,
 	# os.getpid() must be called from inside the function.
 	def get_open_fds():
-		return (int(fd) for fd in os.listdir("/proc/%s/fd" % os.getpid())
+		return (int(fd) for fd in os.listdir("/proc/%s/fd" % portage.getpid())
 			if fd.isdigit())
 
 else:
@@ -363,12 +363,13 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 	# fork, so that the result is cached in the main process.
 	bool(groups)
 
-	parent_pid = os.getpid()
+	parent_pid = portage.getpid()
 	pid = None
 	try:
 		pid = os.fork()
 
 		if pid == 0:
+			portage._ForkWatcher.hook(portage._ForkWatcher)
 			try:
 				_exec(binary, mycommand, opt_name, fd_pipes,
 					env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
@@ -386,7 +387,9 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
 				sys.stderr.flush()
 
 	finally:
-		if pid == 0 or (pid is None and os.getpid() != parent_pid):
+		# Don't used portage.getpid() here, due to a race with the above
+		# portage._ForkWatcher cache update.
+		if pid == 0 or (pid is None and _os.getpid() != parent_pid):
 			# Call os._exit() from a finally block in order
 			# to suppress any finally blocks from earlier
 			# in the call stack (see bug #345289). This
@@ -603,7 +606,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 	# it is done before we start forking children
 	if cgroup:
 		with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
-			f.write('%d\n' % os.getpid())
+			f.write('%d\n' % portage.getpid())
 
 	# Unshare (while still uid==0)
 	if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
@@ -640,6 +643,9 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
 						if unshare_pid:
 							main_child_pid = os.fork()
 							if main_child_pid == 0:
+								# The portage.getpid() cache may need to be updated here,
+								# in case the pre_exec function invokes portage APIs.
+								portage._ForkWatcher.hook(portage._ForkWatcher)
 								# pid namespace requires us to become init
 								binary, myargs = portage._python_interpreter, [
 									portage._python_interpreter,
diff --git a/lib/portage/tests/locks/test_lock_nonblock.py b/lib/portage/tests/locks/test_lock_nonblock.py
index 02ba16ad9..3448b84f6 100644
--- a/lib/portage/tests/locks/test_lock_nonblock.py
+++ b/lib/portage/tests/locks/test_lock_nonblock.py
@@ -19,6 +19,7 @@ class LockNonblockTestCase(TestCase):
 			lock1 = portage.locks.lockfile(path)
 			pid = os.fork()
 			if pid == 0:
+				portage._ForkWatcher.hook(portage._ForkWatcher)
 				portage.locks._close_fds()
 				 # Disable close_fds since we don't exec
 				 # (see _setup_pipes docstring).
diff --git a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
index e5b104e0f..c37a6338b 100644
--- a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
+++ b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
@@ -39,7 +39,7 @@ proc = loop.run_until_complete(asyncio.create_subprocess_exec('sleep', '0', loop
 loop.run_until_complete(proc.wait())
 
 for i in range(8192):
-	os.kill(os.getpid(), signal.SIGCHLD)
+	os.kill(portage.getpid(), signal.SIGCHLD)
 
 # Verify that the child watcher still works correctly
 # (this will hang if it doesn't).
diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py
index c14c15fe8..0412b2b59 100644
--- a/lib/portage/util/__init__.py
+++ b/lib/portage/util/__init__.py
@@ -1266,7 +1266,7 @@ class atomic_ofstream(ObjectProxy):
 		if follow_links:
 			canonical_path = os.path.realpath(filename)
 			object.__setattr__(self, '_real_name', canonical_path)
-			tmp_name = "%s.%i" % (canonical_path, os.getpid())
+			tmp_name = "%s.%i" % (canonical_path, portage.getpid())
 			try:
 				object.__setattr__(self, '_file',
 					open_func(_unicode_encode(tmp_name,
@@ -1281,7 +1281,7 @@ class atomic_ofstream(ObjectProxy):
 				# new error if necessary.
 
 		object.__setattr__(self, '_real_name', filename)
-		tmp_name = "%s.%i" % (filename, os.getpid())
+		tmp_name = "%s.%i" % (filename, portage.getpid())
 		object.__setattr__(self, '_file',
 			open_func(_unicode_encode(tmp_name,
 				encoding=_encodings['fs'], errors='strict'),
diff --git a/lib/portage/util/_eventloop/EventLoop.py b/lib/portage/util/_eventloop/EventLoop.py
index 94e637853..ff2b73255 100644
--- a/lib/portage/util/_eventloop/EventLoop.py
+++ b/lib/portage/util/_eventloop/EventLoop.py
@@ -188,7 +188,7 @@ class EventLoop:
 		self._sigchld_read = None
 		self._sigchld_write = None
 		self._sigchld_src_id = None
-		self._pid = os.getpid()
+		self._pid = portage.getpid()
 		self._asyncio_wrapper = _PortageEventLoop(loop=self)
 		self._asyncio_child_watcher = _PortageChildWatcher(self)
 
@@ -431,7 +431,7 @@ class EventLoop:
 		# If this signal handler was not installed by the
 		# current process then the signal doesn't belong to
 		# this EventLoop instance.
-		if os.getpid() == self._pid:
+		if portage.getpid() == self._pid:
 			os.write(self._sigchld_write, b'\0')
 
 	def _sigchld_io_cb(self, fd, events):
@@ -1026,7 +1026,7 @@ class EventLoop:
 			log_lines.append('{}: {}'.format(key, value))
 
 		logging.error('\n'.join(log_lines), exc_info=exc_info)
-		os.kill(os.getpid(), signal.SIGTERM)
+		os.kill(portage.getpid(), signal.SIGTERM)
 
 	def call_exception_handler(self, context):
 		"""
diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py
index 605e7243b..836f1c30a 100644
--- a/lib/portage/util/_eventloop/asyncio_event_loop.py
+++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
@@ -69,7 +69,7 @@ class AsyncioEventLoop(_AbstractEventLoop):
 			# in order to ensure that emerge exits immediately (though
 			# uncleanly).
 			signal.signal(signal.SIGTERM, signal.SIG_DFL)
-			os.kill(os.getpid(), signal.SIGTERM)
+			os.kill(portage.getpid(), signal.SIGTERM)
 
 	def _create_future(self):
 		"""
@@ -117,7 +117,7 @@ class AsyncioEventLoop(_AbstractEventLoop):
 			self._wakeup_fd = -1
 			# Account for any signals that may have arrived between
 			# set_wakeup_fd calls.
-			os.kill(os.getpid(), signal.SIGCHLD)
+			os.kill(portage.getpid(), signal.SIGCHLD)
 		try:
 			return self._loop.run_until_complete(future)
 		finally:
diff --git a/lib/portage/util/_eventloop/global_event_loop.py b/lib/portage/util/_eventloop/global_event_loop.py
index 1db958d2e..21a1d1970 100644
--- a/lib/portage/util/_eventloop/global_event_loop.py
+++ b/lib/portage/util/_eventloop/global_event_loop.py
@@ -1,13 +1,12 @@
 # Copyright 2012-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-import os
-
+import portage
 from .EventLoop import EventLoop
 from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
 
 
-_MAIN_PID = os.getpid()
+_MAIN_PID = portage.getpid()
 _instances = {}
 
 
@@ -17,7 +16,7 @@ def global_event_loop():
 	belongs exclusively to the current process.
 	"""
 
-	pid = os.getpid()
+	pid = portage.getpid()
 	instance = _instances.get(pid)
 	if instance is not None:
 		return instance
diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
index 99c8f7ae7..58c687139 100644
--- a/lib/portage/util/locale.py
+++ b/lib/portage/util/locale.py
@@ -102,6 +102,7 @@ def check_locale(silent=False, env=None):
 
 	pid = os.fork()
 	if pid == 0:
+		portage._ForkWatcher.hook(portage._ForkWatcher)
 		try:
 			if env is not None:
 				try:
diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index 4f8054f29..a251d369d 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -209,7 +209,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None,
 	if hardlink_candidates:
 		head, tail = os.path.split(dest)
 		hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \
-			(tail, os.getpid()))
+			(tail, portage.getpid()))
 		try:
 			os.unlink(hardlink_tmp)
 		except OSError as e:
diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
index 9f22c1dbe..a76d1a741 100644
--- a/lib/portage/util/socks5.py
+++ b/lib/portage/util/socks5.py
@@ -42,7 +42,7 @@ class ProxyManager:
 		portage.util.ensure_dirs(tmpdir, **ensure_dirs_kwargs)
 
 		self.socket_path = os.path.join(tmpdir,
-				'.portage.%d.net.sock' % os.getpid())
+				'.portage.%d.net.sock' % portage.getpid())
 		server_bin = os.path.join(settings['PORTAGE_BIN_PATH'], 'socks5-server.py')
 		spawn_kwargs = {}
 		# The portage_uid check solves EPERM failures in Travis CI.
diff --git a/lib/portage/xpak.py b/lib/portage/xpak.py
index 2a4bcda21..9063c4c56 100644
--- a/lib/portage/xpak.py
+++ b/lib/portage/xpak.py
@@ -325,7 +325,7 @@ class tbz2:
 		self.scan() # Don't care about condition... We'll rewrite the data anyway.
 
 		if break_hardlinks and self.filestat and self.filestat.st_nlink > 1:
-			tmp_fname = "%s.%d" % (self.file, os.getpid())
+			tmp_fname = "%s.%d" % (self.file, portage.getpid())
 			copyfile(self.file, tmp_fname)
 			try:
 				portage.util.apply_stat_permissions(self.file, self.filestat)
-- 
2.25.3