From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id D824013888F for ; Sun, 11 Oct 2015 01:54:45 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id B36BEE07F0; Sun, 11 Oct 2015 01:54:42 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 42E07E07F0 for ; Sun, 11 Oct 2015 01:54:42 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by smtp.gentoo.org (Postfix) with ESMTP id 38FA2340680 for ; Sun, 11 Oct 2015 01:54:41 +0000 (UTC) From: Mike Frysinger To: gentoo-catalyst@lists.gentoo.org Subject: [gentoo-catalyst] [PATCH 2/2] lock: gut & replace with snakeoil Date: Sat, 10 Oct 2015 21:54:39 -0400 Message-Id: <1444528479-3858-2-git-send-email-vapier@gentoo.org> X-Mailer: git-send-email 2.5.2 In-Reply-To: <1444528479-3858-1-git-send-email-vapier@gentoo.org> References: <1444528479-3858-1-git-send-email-vapier@gentoo.org> Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-catalyst@lists.gentoo.org Reply-to: gentoo-catalyst@lists.gentoo.org X-Archives-Salt: 9ca686df-29df-40da-bce5-99870e81f7fb X-Archives-Hash: 874bf54fe586fe0ffa17728258360830 The hardlink logic is unused, so start by deleting all of that. If someone wants that (multiple builds on NFS?), we can look at restoring it. --- catalyst/lock.py | 467 ++----------------------------------------------------- 1 file changed, 15 insertions(+), 452 deletions(-) diff --git a/catalyst/lock.py b/catalyst/lock.py index 8095a82..39926dd 100644 --- a/catalyst/lock.py +++ b/catalyst/lock.py @@ -1,467 +1,30 @@ - import os -import fcntl -import errno -import sys -import time -from catalyst.support import CatalystError, normpath -def writemsg(mystr): - sys.stderr.write(mystr) - sys.stderr.flush() +from snakeoil import fileutils +from snakeoil import osutils -class LockInUse(Exception): - def __init__(self, message): - if message: - #(type,value)=sys.exc_info()[:2] - #if value!=None: - #print - #kprint traceback.print_exc(file=sys.stdout) - print - print "!!! catalyst lock file in use: "+message - print +LockInUse = osutils.LockException class LockDir(object): - locking_method=fcntl.flock - lock_dirs_in_use=[] - die_on_failed_lock=True - - def __del__(self): - #print "Lock.__del__() 1" - self.clean_my_hardlocks() - #print "Lock.__del__() 2" - self.delete_lock_from_path_list() - #print "Lock.__del__() 3" - if self.islocked(): - #print "Lock.__del__() 4" - self.fcntl_unlock() - #print "Lock.__del__() finnished" - - def __init__(self,lockdir): - self.locked=False - self.myfd=None - self.set_gid(250) - self.locking_method=LockDir.locking_method - self.set_lockdir(lockdir) - self.set_lockfilename(".catalyst_lock") - self.set_lockfile() - - if LockDir.lock_dirs_in_use.count(lockdir)>0: - raise "This directory already associated with a lock object" - else: - LockDir.lock_dirs_in_use.append(lockdir) - - self.myhardlock = None - self.hardlock_paths={} - - def delete_lock_from_path_list(self): - try: - LockDir.lock_dirs_in_use.remove(self.lockdir) - except ValueError: - pass - - def islocked(self): - if self.locked: - return True - else: - return False + """An object that creates locks inside dirs""" - def set_gid(self,gid): - if not self.islocked(): -# if self.settings["DEBUG"]: -# print "setting gid to", gid - self.gid=gid - - def set_lockdir(self,lockdir): - if not os.path.exists(lockdir): - os.makedirs(lockdir) - if os.path.isdir(lockdir): - if not self.islocked(): - if lockdir[-1] == "/": - lockdir=lockdir[:-1] - self.lockdir=normpath(lockdir) -# if self.settings["DEBUG"]: -# print "setting lockdir to", self.lockdir - else: - raise "the lock object needs a path to a dir" - - def set_lockfilename(self,lockfilename): - if not self.islocked(): - self.lockfilename=lockfilename -# if self.settings["DEBUG"]: -# print "setting lockfilename to", self.lockfilename - - def set_lockfile(self): - if not self.islocked(): - self.lockfile=normpath(self.lockdir+'/'+self.lockfilename) -# if self.settings["DEBUG"]: -# print "setting lockfile to", self.lockfile + def __init__(self, lockdir): + self.gid = 250 + self.lockfile = os.path.join(lockdir, '.catalyst_lock') + osutils.ensure_dirs(lockdir) + fileutils.touch(self.lockfile, mode=0o664) + os.chown(self.lockfile, -1, self.gid) + self.lock = osutils.FsLock(self.lockfile) def read_lock(self): - if not self.locking_method == "HARDLOCK": - self.fcntl_lock("read") - else: - print "HARDLOCKING doesnt support shared-read locks" - print "using exclusive write locks" - self.hard_lock() + self.lock.acquire_read_lock() def write_lock(self): - if not self.locking_method == "HARDLOCK": - self.fcntl_lock("write") - else: - self.hard_lock() + self.lock.acquire_write_lock() def unlock(self): - if not self.locking_method == "HARDLOCK": - self.fcntl_unlock() - else: - self.hard_unlock() - - def fcntl_lock(self,locktype): - if self.myfd==None: - if not os.path.exists(os.path.dirname(self.lockdir)): - raise CatalystError("DirectoryNotFound: %s" - % os.path.dirname(self.lockdir), print_traceback=True) - if not os.path.exists(self.lockfile): - old_mask=os.umask(000) - self.myfd = os.open(self.lockfile, os.O_CREAT|os.O_RDWR,0660) - try: - if os.stat(self.lockfile).st_gid != self.gid: - os.chown(self.lockfile,os.getuid(),self.gid) - except OSError, e: - if e[0] == 2: #XXX: No such file or directory - return self.fcntl_locking(locktype) - else: - writemsg("Cannot chown a lockfile. This could cause inconvenience later.\n") - - os.umask(old_mask) - else: - self.myfd = os.open(self.lockfile, os.O_CREAT|os.O_RDWR,0660) - - try: - if locktype == "read": - self.locking_method(self.myfd,fcntl.LOCK_SH|fcntl.LOCK_NB) - else: - self.locking_method(self.myfd,fcntl.LOCK_EX|fcntl.LOCK_NB) - except IOError, e: - if "errno" not in dir(e): - raise - if e.errno == errno.EAGAIN: - if not LockDir.die_on_failed_lock: - # Resource temp unavailable; eg, someone beat us to the lock. - writemsg("waiting for lock on %s\n" % self.lockfile) - - # Try for the exclusive or shared lock again. - if locktype == "read": - self.locking_method(self.myfd,fcntl.LOCK_SH) - else: - self.locking_method(self.myfd,fcntl.LOCK_EX) - else: - raise LockInUse,self.lockfile - elif e.errno == errno.ENOLCK: - pass - else: - raise - if not os.path.exists(self.lockfile): - os.close(self.myfd) - self.myfd=None - #writemsg("lockfile recurse\n") - self.fcntl_lock(locktype) - else: - self.locked=True - #writemsg("Lockfile obtained\n") - - def fcntl_unlock(self): - unlinkfile = 1 - if not os.path.exists(self.lockfile): - print "lockfile does not exist '%s'" % self.lockfile - #print "fcntl_unlock() , self.myfd:", self.myfd, type(self.myfd) - if self.myfd != None: - #print "fcntl_unlock() trying to close it " - try: - os.close(self.myfd) - self.myfd=None - except Exception: - pass - return False - - try: - if self.myfd == None: - self.myfd = os.open(self.lockfile, os.O_WRONLY,0660) - unlinkfile = 1 - self.locking_method(self.myfd,fcntl.LOCK_UN) - except Exception, e: - #if self.myfd is not None: - #print "fcntl_unlock() trying to close", self.myfd - #os.close(self.myfd) - #self.myfd=None - #raise IOError, "Failed to unlock file '%s'\n%s" % (self.lockfile, str(e)) - try: - # This sleep call was added to allow other processes that are - # waiting for a lock to be able to grab it before it is deleted. - # lockfile() already accounts for this situation, however, and - # the sleep here adds more time than is saved overall, so am - # commenting until it is proved necessary. - #time.sleep(0.0001) - if unlinkfile: - InUse=False - try: - self.locking_method(self.myfd,fcntl.LOCK_EX|fcntl.LOCK_NB) - except Exception: - print "Read lock may be in effect. skipping lockfile delete..." - InUse=True - # We won the lock, so there isn't competition for it. - # We can safely delete the file. - #writemsg("Got the lockfile...\n") - #writemsg("Unlinking...\n") - self.locking_method(self.myfd,fcntl.LOCK_UN) - if not InUse: - os.unlink(self.lockfile) - os.close(self.myfd) - self.myfd=None -# if self.settings["DEBUG"]: -# print "Unlinked lockfile..." - except Exception, e: - # We really don't care... Someone else has the lock. - # So it is their problem now. - print "Failed to get lock... someone took it." - print str(e) - - # Why test lockfilename? Because we may have been handed an - # fd originally, and the caller might not like having their - # open fd closed automatically on them. - #if type(lockfilename) == types.StringType: - # os.close(myfd) - #print "fcntl_unlock() trying a last ditch close", self.myfd - if self.myfd != None: - os.close(self.myfd) - self.myfd=None - self.locked=False - time.sleep(.0001) - - def hard_lock(self,max_wait=14400): - """Does the NFS, hardlink shuffle to ensure locking on the disk. - We create a PRIVATE lockfile, that is just a placeholder on the disk. - Then we HARDLINK the real lockfile to that private file. - If our file can 2 references, then we have the lock. :) - Otherwise we lather, rise, and repeat. - We default to a 4 hour timeout. - """ - - self.myhardlock = self.hardlock_name(self.lockdir) - - start_time = time.time() - reported_waiting = False - - while time.time() < (start_time + max_wait): - # We only need it to exist. - self.myfd = os.open(self.myhardlock, os.O_CREAT|os.O_RDWR,0660) - os.close(self.myfd) - - self.add_hardlock_file_to_cleanup() - if not os.path.exists(self.myhardlock): - raise CatalystError("FileNotFound: Created lockfile is missing: " - "%(filename)s" % {"filename":self.myhardlock}, - print_traceback=True) - try: - os.link(self.myhardlock, self.lockfile) - except Exception: -# if self.settings["DEBUG"]: -# print "lockfile(): Hardlink: Link failed." -# print "Exception: ",e - pass - - if self.hardlink_is_mine(self.myhardlock, self.lockfile): - # We have the lock. - if reported_waiting: - print - return True - - if reported_waiting: - writemsg(".") - else: - reported_waiting = True - print - print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)" - print "Lockfile: " + self.lockfile - time.sleep(3) - - os.unlink(self.myhardlock) - return False - - def hard_unlock(self): - try: - if os.path.exists(self.myhardlock): - os.unlink(self.myhardlock) - if os.path.exists(self.lockfile): - os.unlink(self.lockfile) - except Exception: - writemsg("Something strange happened to our hardlink locks.\n") - - def add_hardlock_file_to_cleanup(self): - #mypath = self.normpath(path) - if os.path.isdir(self.lockdir) and os.path.isfile(self.myhardlock): - self.hardlock_paths[self.lockdir]=self.myhardlock - - def remove_hardlock_file_from_cleanup(self): - if self.lockdir in self.hardlock_paths: - del self.hardlock_paths[self.lockdir] - print self.hardlock_paths - - @staticmethod - def hardlock_name(path): - mypath=path+"/.hardlock-"+os.uname()[1]+"-"+str(os.getpid()) - newpath = os.path.normpath(mypath) - if len(newpath) > 1: - if newpath[1] == "/": - newpath = "/"+newpath.lstrip("/") - return newpath - - @staticmethod - def hardlink_is_mine(link, lock): - import stat - try: - myhls = os.stat(link) - mylfs = os.stat(lock) - except Exception: - myhls = None - mylfs = None - - if myhls: - if myhls[stat.ST_NLINK] == 2: - return True - if mylfs: - if mylfs[stat.ST_INO] == myhls[stat.ST_INO]: - return True - return False - - @staticmethod - def hardlink_active(lock): - if not os.path.exists(lock): - return False - - def clean_my_hardlocks(self): - try: - for x in self.hardlock_paths.keys(): - self.hardlock_cleanup(x) - except AttributeError: - pass - - def hardlock_cleanup(self,path): - #mypid = str(os.getpid()) - myhost = os.uname()[1] - mydl = os.listdir(path) - results = [] - mycount = 0 - - mylist = {} - for x in mydl: - filepath=path+"/"+x - if os.path.isfile(filepath): - parts = filepath.split(".hardlock-") - if len(parts) == 2: - filename = parts[0] - hostpid = parts[1].split("-") - host = "-".join(hostpid[:-1]) - pid = hostpid[-1] - if filename not in mylist: - mylist[filename] = {} - - if host not in mylist[filename]: - mylist[filename][host] = [] - mylist[filename][host].append(pid) - mycount += 1 - else: - mylist[filename][host].append(pid) - mycount += 1 - - - results.append("Found %(count)s locks" % {"count":mycount}) - for x in mylist.keys(): - if myhost in mylist[x]: - mylockname = self.hardlock_name(x) - if self.hardlink_is_mine(mylockname, self.lockfile) or \ - not os.path.exists(self.lockfile): - for y in mylist[x].keys(): - for z in mylist[x][y]: - filename = x+".hardlock-"+y+"-"+z - if filename == mylockname: - self.hard_unlock() - continue - try: - # We're sweeping through, unlinking everyone's locks. - os.unlink(filename) - results.append("Unlinked: " + filename) - except Exception: - pass - try: - os.unlink(x) - results.append("Unlinked: " + x) - os.unlink(mylockname) - results.append("Unlinked: " + mylockname) - except Exception: - pass - else: - try: - os.unlink(mylockname) - results.append("Unlinked: " + mylockname) - except Exception: - pass - return results - - -if __name__ == "__main__": - - def lock_work(): - print - for i in range(1,6): - print i,time.time() - time.sleep(1) - print - - print "Lock 5 starting" - Lock1=LockDir("/tmp/lock_path") - Lock1.write_lock() - print "Lock1 write lock" - - lock_work() - - Lock1.unlock() - print "Lock1 unlock" - - Lock1.read_lock() - print "Lock1 read lock" - - lock_work() - - Lock1.unlock() - print "Lock1 unlock" - - Lock1.read_lock() - print "Lock1 read lock" - - Lock1.write_lock() - print "Lock1 write lock" - - lock_work() - - Lock1.unlock() - print "Lock1 unlock" - - Lock1.read_lock() - print "Lock1 read lock" - - lock_work() - - Lock1.unlock() - print "Lock1 unlock" - -#Lock1.write_lock() -#time.sleep(2) -#Lock1.unlock() - ##Lock1.write_lock() - #time.sleep(2) - #Lock1.unlock() + # Releasing a write lock is the same as a read lock. + self.lock.release_write_lock() -- 2.5.2