From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 52C2B138334 for ; Mon, 11 Feb 2019 19:46:49 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 75EDEE07F2; Mon, 11 Feb 2019 19:46:48 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 42C7AE07F2 for ; Mon, 11 Feb 2019 19:46:48 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 0830A335D55 for ; Mon, 11 Feb 2019 19:46:46 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 7EACF503 for ; Mon, 11 Feb 2019 19:46:44 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <1549566008.16d3d31dbc971ce95d80d6ec0645d6ee92b6baa2.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: lib/portage/ X-VCS-Repository: proj/portage X-VCS-Files: lib/portage/locks.py X-VCS-Directories: lib/portage/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: 16d3d31dbc971ce95d80d6ec0645d6ee92b6baa2 X-VCS-Branch: master Date: Mon, 11 Feb 2019 19:46:44 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: da9a67f8-ba8a-4094-9bac-d2fe32ecba8c X-Archives-Hash: 3a15e350e5a06a0d1c737ad8aaa4118b commit: 16d3d31dbc971ce95d80d6ec0645d6ee92b6baa2 Author: Zac Medico sony com> AuthorDate: Wed Feb 6 22:21:47 2019 +0000 Commit: Zac Medico gentoo org> CommitDate: Thu Feb 7 19:00:08 2019 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=16d3d31d locks: handle lock file removal on NFS (bug 636798) Handle cases where a lock file on NFS has been removed by a concurrent process that held the lock earlier. Since stat is not reliable for removed files on NFS with the default file attribute cache behavior ('ac' mount option), create a temporary hardlink in order to prove that the file path exists on the NFS server. Bug: https://bugs.gentoo.org/636798 Copyright: Sony Interactive Entertainment Inc. Signed-off-by: Zac Medico gentoo.org> lib/portage/locks.py | 78 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 13 deletions(-) diff --git a/lib/portage/locks.py b/lib/portage/locks.py index a4e7ec53f..74c2c086a 100644 --- a/lib/portage/locks.py +++ b/lib/portage/locks.py @@ -1,5 +1,5 @@ # portage: Lock management code -# Copyright 2004-2014 Gentoo Foundation +# Copyright 2004-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \ @@ -20,6 +20,7 @@ from portage.exception import (DirectoryNotFound, FileNotFound, InvalidData, TryAgain, OperationNotPermitted, PermissionDenied, ReadOnlyFileSystem) from portage.util import writemsg +from portage.util.install_mask import _raise_exc from portage.localization import _ if sys.hexversion >= 0x3000000: @@ -148,18 +149,17 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: - try: - myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660) - except OSError as e: - func_call = "open('%s')" % lockfilename - if e.errno == OperationNotPermitted.errno: - raise OperationNotPermitted(func_call) - elif e.errno == PermissionDenied.errno: - raise PermissionDenied(func_call) - elif e.errno == ReadOnlyFileSystem.errno: - raise ReadOnlyFileSystem(func_call) + while True: + try: + myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660) + except OSError as e: + if e.errno in (errno.ENOENT, errno.ESTALE) and os.path.isdir(os.path.dirname(lockfilename)): + # Retry required for NFS (see bug 636798). + continue + else: + _raise_exc(e) else: - raise + break if not preexisting: try: @@ -273,7 +273,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, if isinstance(lockfilename, basestring) and \ - myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0: + myfd != HARDLINK_FD and _lockfile_was_removed(myfd, lockfilename): # The file was deleted on us... Keep trying to make one... os.close(myfd) writemsg(_("lockfile recurse\n"), 1) @@ -298,6 +298,58 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method) + +def _lockfile_was_removed(lock_fd, lock_path): + """ + Check if lock_fd still refers to a file located at lock_path, since + the file may have been removed by a concurrent process that held the + lock earlier. This implementation includes support for NFS, where + stat is not reliable for removed files due to the default file + attribute cache behavior ('ac' mount option). + + @param lock_fd: an open file descriptor for a lock file + @type lock_fd: int + @param lock_path: path of lock file + @type lock_path: str + @rtype: bool + @return: True if lock_path exists and corresponds to lock_fd, False otherwise + """ + try: + fstat_st = os.fstat(lock_fd) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + _raise_exc(e) + return True + + # Since stat is not reliable for removed files on NFS with the default + # file attribute cache behavior ('ac' mount option), create a temporary + # hardlink in order to prove that the file path exists on the NFS server. + hardlink_path = hardlock_name(lock_path) + try: + os.unlink(hardlink_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + _raise_exc(e) + try: + try: + os.link(lock_path, hardlink_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + _raise_exc(e) + return True + + hardlink_stat = os.stat(hardlink_path) + if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev: + return True + finally: + try: + os.unlink(hardlink_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + _raise_exc(e) + return False + + def _fstat_nlink(fd): """ @param fd: an open file descriptor