From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id F1CD81387D3 for ; Sun, 26 Oct 2014 11:12:34 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id C7D87E0898; Sun, 26 Oct 2014 11:12:28 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 3A8FBE083A for ; Sun, 26 Oct 2014 11:12:28 +0000 (UTC) Received: from localhost.localdomain (ip70-181-96-121.oc.oc.cox.net [70.181.96.121]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) (Authenticated sender: zmedico) by smtp.gentoo.org (Postfix) with ESMTPSA id 43F7934016B; Sun, 26 Oct 2014 11:12:27 +0000 (UTC) From: zmedico@gentoo.org To: gentoo-portage-dev@lists.gentoo.org Cc: Zac Medico Subject: [gentoo-portage-dev] [PATCH 2/3] dispatch-conf: symlink support for bug #485598 Date: Sun, 26 Oct 2014 04:12:15 -0700 Message-Id: <1414321936-22851-2-git-send-email-zmedico@gentoo.org> X-Mailer: git-send-email 2.0.4 In-Reply-To: <1414321936-22851-1-git-send-email-zmedico@gentoo.org> References: <1414321936-22851-1-git-send-email-zmedico@gentoo.org> Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org X-Archives-Salt: 71c0e164-6eea-439e-aa9e-dfa793263023 X-Archives-Hash: eed84b5a9af93f675ba7ae75281ceb96 From: Zac Medico This includes numerous logic adjustments that are needed to support protected symlinks. The new diff_mixed function is used for diffs between arbitrary file types. For example, a diff between two symlinks looks like this: -SYM: /foo/bar -> baz +SYM: /foo/bar -> blah X-Gentoo-Bug: 485598 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=485598 --- bin/dispatch-conf | 40 +++++++------ pym/portage/dispatch_conf.py | 137 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 139 insertions(+), 38 deletions(-) diff --git a/bin/dispatch-conf b/bin/dispatch-conf index 6d2ae94..80dafd6 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -15,15 +15,15 @@ from __future__ import print_function from stat import ST_GID, ST_MODE, ST_UID from random import random -import atexit, re, shutil, stat, sys +import atexit, io, re, functools, shutil, sys from os import path as osp if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage import os -from portage import _unicode_decode -from portage.dispatch_conf import diffstatusoutput +from portage import _encodings, _unicode_decode +from portage.dispatch_conf import diffstatusoutput, diff_mixed from portage.process import find_binary, spawn FIND_EXTANT_CONFIGS = "find '%s' %s -name '._cfg????_%s' ! -name '.*~' ! -iname '.*.bak' -print" @@ -72,6 +72,11 @@ def cmd_var_is_valid(cmd): return find_binary(cmd[0]) is not None +def diff(file1, file2): + return diff_mixed( + functools.partial(diffstatusoutput, DIFF_CONTENTS), + file1, file2) + class dispatch: options = {} @@ -89,8 +94,6 @@ class dispatch: or not os.path.exists(self.options["log-file"]): open(self.options["log-file"], 'w').close() # Truncate it os.chmod(self.options["log-file"], 0o600) - else: - self.options["log-file"] = "/dev/null" pager = self.options.get("pager") if pager is None or not cmd_var_is_valid(pager): @@ -148,9 +151,6 @@ class dispatch: portage.util.shlex_split( portage.settings.get('CONFIG_PROTECT_MASK', ''))) - def diff(file1, file2): - return diffstatusoutput(DIFF_CONTENTS, file1, file2) - # # Remove new configs identical to current # and @@ -166,7 +166,7 @@ class dispatch: mrgfail = portage.dispatch_conf.rcs_archive(archive, conf['current'], conf['new'], mrgconf) else: mrgfail = portage.dispatch_conf.file_archive(archive, conf['current'], conf['new'], mrgconf) - if os.path.exists(archive + '.dist'): + if os.path.lexists(archive + '.dist'): unmodified = len(diff(conf['current'], archive + '.dist')[1]) == 0 else: unmodified = 0 @@ -181,7 +181,7 @@ class dispatch: if newconf == mrgconf and \ self.options.get('ignore-previously-merged') != 'yes' and \ - os.path.exists(archive+'.dist') and \ + os.path.lexists(archive+'.dist') and \ len(diff(archive+'.dist', conf['new'])[1]) == 0: # The current update is identical to the archived .dist # version that has previously been merged. @@ -254,6 +254,11 @@ class dispatch: valid_input = "qhtnmlezu" + def diff_pager(file1, file2): + cmd = self.options['diff'] % (file1, file2) + cmd += pager + spawn_shell(cmd) + for conf in confs: count = count + 1 @@ -266,14 +271,10 @@ class dispatch: while 1: clear_screen() if show_new_diff: - cmd = self.options['diff'] % (conf['new'], mrgconf) - cmd += pager - spawn_shell(cmd) + diff_mixed(diff_pager, conf['new'], mrgconf) show_new_diff = 0 else: - cmd = self.options['diff'] % (conf['current'], newconf) - cmd += pager - spawn_shell(cmd) + diff_mixed(diff_pager, conf['current'], newconf) print() print('>> (%i of %i) -- %s' % (count, len(confs), conf ['current'])) @@ -357,7 +358,12 @@ class dispatch: def replace (self, newconf, curconf): """Replace current config with the new/merged version. Also logs the diff of what changed into the configured log file.""" - os.system((DIFF_CONTENTS % (curconf, newconf)) + '>>' + self.options["log-file"]) + if "log-file" in self.options: + status, output = diff(curconf, newconf) + with io.open(self.options["log-file"], mode="a", + encoding=_encodings["stdio"]) as f: + f.write(output + "\n") + try: os.rename(newconf, curconf) except (IOError, os.error) as why: diff --git a/pym/portage/dispatch_conf.py b/pym/portage/dispatch_conf.py index 113d965..7fb99d0 100644 --- a/pym/portage/dispatch_conf.py +++ b/pym/portage/dispatch_conf.py @@ -6,11 +6,12 @@ # Library by Wayne Davison , derived from code # written by Jeremy Wohl (http://igmus.org) -from __future__ import print_function +from __future__ import print_function, unicode_literals -import os, shutil, subprocess, sys +import functools, io, os, shutil, stat, subprocess, sys, tempfile import portage +from portage import _encodings from portage.env.loaders import KeyValuePairFileLoader from portage.localization import _ from portage.util import shlex_split, varexpand @@ -50,6 +51,53 @@ def diffstatusoutput(cmd, file1, file2): output = output[:-1] return (proc.wait(), output) +def diff_mixed(func, file1, file2): + tempdir = None + try: + if os.path.islink(file1) and \ + not os.path.islink(file2) and \ + os.path.isfile(file1) and \ + os.path.isfile(file2): + # If a regular file replaces a symlink to a regular + # file, then show the diff between the regular files + # (bug #330221). + diff_files = (file2, file2) + else: + files = [file1, file2] + diff_files = [file1, file2] + for i in range(len(diff_files)): + st = os.lstat(diff_files[i]) + if stat.S_ISREG(st.st_mode): + continue + + if tempdir is None: + tempdir = tempfile.mkdtemp() + diff_files[i] = os.path.join(tempdir, "%d" % i) + if stat.S_ISLNK(st.st_mode): + link_dest = os.readlink(files[i]) + content = "SYM: %s -> %s\n" % \ + (file1, link_dest) + elif stat.S_ISDIR(st.st_mode): + content = "DIR: %s\n" % (file1,) + elif stat.S_ISFIFO(st.st_mode): + content = "FIF: %s\n" % (file1,) + else: + content = "DEV: %s\n" % (file1,) + with io.open(diff_files[i], mode='w', + encoding=_encodings['stdio']) as f: + f.write(content) + + return func(diff_files[0], diff_files[1]) + + finally: + if tempdir is not None: + shutil.rmtree(tempdir) + +def diffstatusoutput_symlink(cmd, file1, file2): + return diff_mixed( + functools.partial(diffstatusoutput, cmd), + file1, file2) + def read_config(mandatory_opts): eprefix = portage.settings["EPREFIX"] if portage._not_installed: @@ -103,35 +151,57 @@ def rcs_archive(archive, curconf, newconf, mrgconf): except OSError: pass - if os.path.isfile(curconf): + try: + curconf_st = os.lstat(curconf) + except OSError: + curconf_st = None + + if curconf_st is not None and \ + (stat.S_ISREG(curconf_st.st_mode) or + stat.S_ISLNK(curconf_st.st_mode)): try: - shutil.copy2(curconf, archive) + if stat.S_ISLNK(curconf_st.st_mode): + os.symlink(os.readlink(curconf), archive) + else: + shutil.copy2(curconf, archive) except(IOError, os.error) as why: print(_('dispatch-conf: Error copying %(curconf)s to %(archive)s: %(reason)s; fatal') % \ {"curconf": curconf, "archive": archive, "reason": str(why)}, file=sys.stderr) - if os.path.exists(archive + ',v'): + if os.path.lexists(archive + ',v'): os.system(RCS_LOCK + ' ' + archive) os.system(RCS_PUT + ' ' + archive) ret = 0 - if newconf != '': + mystat = None + if newconf: + try: + mystat = os.lstat(newconf) + except OSError: + pass + + if mystat is not None and \ + (stat.S_ISREG(mystat.st_mode) or + stat.S_ISLNK(mystat.st_mode)): os.system(RCS_GET + ' -r' + RCS_BRANCH + ' ' + archive) - has_branch = os.path.exists(archive) + has_branch = os.path.lexists(archive) if has_branch: os.rename(archive, archive + '.dist') try: - shutil.copy2(newconf, archive) + if stat.S_ISLNK(mystat.st_mode): + os.symlink(os.readlink(newconf), archive) + else: + shutil.copy2(newconf, archive) except(IOError, os.error) as why: print(_('dispatch-conf: Error copying %(newconf)s to %(archive)s: %(reason)s; fatal') % \ {"newconf": newconf, "archive": archive, "reason": str(why)}, file=sys.stderr) if has_branch: - if mrgconf != '': + if mrgconf and os.path.isfile(archive) and \ + os.path.isfile(mrgconf): # This puts the results of the merge into mrgconf. ret = os.system(RCS_MERGE % (archive, mrgconf)) - mystat = os.lstat(newconf) os.chmod(mrgconf, mystat.st_mode) os.chown(mrgconf, mystat.st_uid, mystat.st_gid) os.rename(archive, archive + '.dist.new') @@ -153,10 +223,11 @@ def file_archive(archive, curconf, newconf, mrgconf): pass # Archive the current config file if it isn't already saved - if (os.path.exists(archive) and - len(diffstatusoutput("diff -aq '%s' '%s'", curconf, archive)[1]) != 0): + if (os.path.lexists(archive) and + len(diffstatusoutput_symlink( + "diff -aq '%s' '%s'", curconf, archive)[1]) != 0): suf = 1 - while suf < 9 and os.path.exists(archive + '.' + str(suf)): + while suf < 9 and os.path.lexists(archive + '.' + str(suf)): suf += 1 while suf > 1: @@ -165,26 +236,50 @@ def file_archive(archive, curconf, newconf, mrgconf): os.rename(archive, archive + '.1') - if os.path.isfile(curconf): + try: + curconf_st = os.lstat(curconf) + except OSError: + curconf_st = None + + if curconf_st is not None and \ + (stat.S_ISREG(curconf_st.st_mode) or + stat.S_ISLNK(curconf_st.st_mode)): try: - shutil.copy2(curconf, archive) + if stat.S_ISLNK(curconf_st.st_mode): + os.symlink(os.readlink(curconf), archive) + else: + shutil.copy2(curconf, archive) except(IOError, os.error) as why: print(_('dispatch-conf: Error copying %(curconf)s to %(archive)s: %(reason)s; fatal') % \ {"curconf": curconf, "archive": archive, "reason": str(why)}, file=sys.stderr) - if newconf != '': + mystat = None + if newconf: + try: + mystat = os.lstat(newconf) + except OSError: + pass + + if mystat is not None and \ + (stat.S_ISREG(mystat.st_mode) or + stat.S_ISLNK(mystat.st_mode)): # Save off new config file in the archive dir with .dist.new suffix + newconf_archive = archive + '.dist.new' try: - shutil.copy2(newconf, archive + '.dist.new') + if stat.S_ISLNK(mystat.st_mode): + os.symlink(os.readlink(newconf), newconf_archive) + else: + shutil.copy2(newconf, newconf_archive) except(IOError, os.error) as why: print(_('dispatch-conf: Error copying %(newconf)s to %(archive)s: %(reason)s; fatal') % \ {"newconf": newconf, "archive": archive + '.dist.new', "reason": str(why)}, file=sys.stderr) ret = 0 - if mrgconf != '' and os.path.exists(archive + '.dist'): + if mrgconf and os.path.isfile(curconf) and \ + os.path.isfile(newconf) and \ + os.path.isfile(archive + '.dist'): # This puts the results of the merge into mrgconf. ret = os.system(DIFF3_MERGE % (curconf, archive + '.dist', newconf, mrgconf)) - mystat = os.lstat(newconf) os.chmod(mrgconf, mystat.st_mode) os.chown(mrgconf, mystat.st_uid, mystat.st_gid) @@ -195,7 +290,7 @@ def rcs_archive_post_process(archive): """Check in the archive file with the .dist.new suffix on the branch and remove the one with the .dist suffix.""" os.rename(archive + '.dist.new', archive) - if os.path.exists(archive + '.dist'): + if os.path.lexists(archive + '.dist'): # Commit the last-distributed version onto the branch. os.system(RCS_LOCK + RCS_BRANCH + ' ' + archive) os.system(RCS_PUT + ' -r' + RCS_BRANCH + ' ' + archive) @@ -207,5 +302,5 @@ def rcs_archive_post_process(archive): def file_archive_post_process(archive): """Rename the archive file with the .dist.new suffix to a .dist suffix""" - if os.path.exists(archive + '.dist.new'): + if os.path.lexists(archive + '.dist.new'): os.rename(archive + '.dist.new', archive + '.dist') -- 2.0.4