* [gentoo-portage-dev] [PATCH] unpack: use chmod-lite helper for bug 554084 @ 2015-10-02 7:22 Zac Medico 2015-10-02 7:32 ` [gentoo-portage-dev] " Zac Medico 2015-10-02 18:57 ` [gentoo-portage-dev] [PATCH v2] " Zac Medico 0 siblings, 2 replies; 6+ messages in thread From: Zac Medico @ 2015-10-02 7:22 UTC (permalink / raw To: gentoo-portage-dev; +Cc: Zac Medico Use the apply_recursive_permissions function to minimize the number of chmod calls. Also, fix an UnboundLocalError triggered in portage.data._get_global by chmod-lite. X-Gentoo-Bug: 554084 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554084 --- bin/chmod-lite | 10 ++++++++++ bin/chmod-lite.py | 30 ++++++++++++++++++++++++++++++ bin/phase-helpers.sh | 2 +- pym/portage/data.py | 2 +- 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100755 bin/chmod-lite create mode 100755 bin/chmod-lite.py diff --git a/bin/chmod-lite b/bin/chmod-lite new file mode 100755 index 0000000..ffa8d4d --- /dev/null +++ b/bin/chmod-lite @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +export __PORTAGE_HELPER_CWD=${PWD} + +# Use safe cwd, avoiding unsafe import for bug #469338. +cd "${PORTAGE_PYM_PATH}" || exit 1 +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + exec "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH/chmod-lite.py" "$@" diff --git a/bin/chmod-lite.py b/bin/chmod-lite.py new file mode 100755 index 0000000..697cf77 --- /dev/null +++ b/bin/chmod-lite.py @@ -0,0 +1,30 @@ +#!/usr/bin/python -b +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import sys + +from portage.util import apply_recursive_permissions + +# Change back to original cwd _after_ all imports (bug #469338). +os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) + +def main(files): + + if sys.hexversion >= 0x3000000: + # We can't trust that the filesystem encoding (locale dependent) + # correctly matches the arguments, so use surrogateescape to + # pass through the original argv bytes for Python 3. + fs_encoding = sys.getfilesystemencoding() + files = [x.encode(fs_encoding, 'surrogateescape') for x in files] + + for filename in files: + # Emulate 'chmod -fR a+rX,u+w,g-w,o-w' with mininal chmod calls. + apply_recursive_permissions(filename, filemode=0o644, + filemask=0o002, dirmode=0o755, dirmask=0o002) + + return os.EX_OK + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh index efd2cfa..0c25ffe 100644 --- a/bin/phase-helpers.sh +++ b/bin/phase-helpers.sh @@ -532,7 +532,7 @@ unpack() { # Do not chmod '.' since it's probably ${WORKDIR} and PORTAGE_WORKDIR_MODE # should be preserved. find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ - ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w + ${XARGS} -0 "${PORTAGE_BIN_PATH}/chmod-lite" } econf() { diff --git a/pym/portage/data.py b/pym/portage/data.py index 2fd287d..2c99548 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -139,7 +139,7 @@ def _get_global(k): v = 2 elif unprivileged: v = 2 - elif portage_gid in os.getgroups(): + elif _get_global('portage_gid') in os.getgroups(): v = 1 elif k in ('portage_gid', 'portage_uid'): -- 2.4.6 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* [gentoo-portage-dev] Re: [PATCH] unpack: use chmod-lite helper for bug 554084 2015-10-02 7:22 [gentoo-portage-dev] [PATCH] unpack: use chmod-lite helper for bug 554084 Zac Medico @ 2015-10-02 7:32 ` Zac Medico 2015-10-02 18:57 ` [gentoo-portage-dev] [PATCH v2] " Zac Medico 1 sibling, 0 replies; 6+ messages in thread From: Zac Medico @ 2015-10-02 7:32 UTC (permalink / raw To: gentoo-portage-dev; +Cc: Zac Medico On Fri, Oct 2, 2015 at 12:22 AM, Zac Medico <zmedico@gentoo.org> wrote: > Use the apply_recursive_permissions function to minimize the number of > chmod calls. Actually, apply_recursive_permissions fails with www-apache/mod_auth_token-1.0.6_beta, so it needs some work. ^ permalink raw reply [flat|nested] 6+ messages in thread
* [gentoo-portage-dev] [PATCH v2] unpack: use chmod-lite helper for bug 554084 2015-10-02 7:22 [gentoo-portage-dev] [PATCH] unpack: use chmod-lite helper for bug 554084 Zac Medico 2015-10-02 7:32 ` [gentoo-portage-dev] " Zac Medico @ 2015-10-02 18:57 ` Zac Medico 2015-10-02 19:11 ` [gentoo-portage-dev] [PATCH v3] " Zac Medico 2015-10-03 16:15 ` [gentoo-portage-dev] [PATCH v4] " Zac Medico 1 sibling, 2 replies; 6+ messages in thread From: Zac Medico @ 2015-10-02 18:57 UTC (permalink / raw To: gentoo-portage-dev; +Cc: Zac Medico Use the apply_recursive_permissions function to minimize the number of chmod calls. Also, fix an UnboundLocalError triggered in portage.data._get_global by chmod-lite. X-Gentoo-Bug: 554084 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554084 --- [PATCH v2] fixes apply_recursive_permissions to apply permissions to a directory before that directory is traversed. bin/chmod-lite | 10 ++++++++++ bin/chmod-lite.py | 30 ++++++++++++++++++++++++++++++ bin/phase-helpers.sh | 2 +- pym/portage/data.py | 2 +- pym/portage/util/__init__.py | 30 +++++++++++++++++------------- 5 files changed, 59 insertions(+), 15 deletions(-) create mode 100755 bin/chmod-lite create mode 100755 bin/chmod-lite.py diff --git a/bin/chmod-lite b/bin/chmod-lite new file mode 100755 index 0000000..ffa8d4d --- /dev/null +++ b/bin/chmod-lite @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +export __PORTAGE_HELPER_CWD=${PWD} + +# Use safe cwd, avoiding unsafe import for bug #469338. +cd "${PORTAGE_PYM_PATH}" || exit 1 +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + exec "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH/chmod-lite.py" "$@" diff --git a/bin/chmod-lite.py b/bin/chmod-lite.py new file mode 100755 index 0000000..a552b25 --- /dev/null +++ b/bin/chmod-lite.py @@ -0,0 +1,30 @@ +#!/usr/bin/python -b +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import sys + +from portage.util import apply_recursive_permissions + +# Change back to original cwd _after_ all imports (bug #469338). +os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) + +def main(files): + + if sys.hexversion >= 0x3000000: + # We can't trust that the filesystem encoding (locale dependent) + # correctly matches the arguments, so use surrogateescape to + # pass through the original argv bytes for Python 3. + fs_encoding = sys.getfilesystemencoding() + files = [x.encode(fs_encoding, 'surrogateescape') for x in files] + + for filename in files: + # Emulate 'chmod -fR a+rX,u+w,g-w,o-w' with mininal chmod calls. + apply_recursive_permissions(filename, filemode=0o644, + filemask=0o022, dirmode=0o755, dirmask=0o022) + + return os.EX_OK + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh index efd2cfa..0c25ffe 100644 --- a/bin/phase-helpers.sh +++ b/bin/phase-helpers.sh @@ -532,7 +532,7 @@ unpack() { # Do not chmod '.' since it's probably ${WORKDIR} and PORTAGE_WORKDIR_MODE # should be preserved. find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ - ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w + ${XARGS} -0 "${PORTAGE_BIN_PATH}/chmod-lite" } econf() { diff --git a/pym/portage/data.py b/pym/portage/data.py index 2fd287d..2c99548 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -139,7 +139,7 @@ def _get_global(k): v = 2 elif unprivileged: v = 2 - elif portage_gid in os.getgroups(): + elif _get_global('portage_gid') in os.getgroups(): v = 1 elif k in ('portage_gid', 'portage_uid'): diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index c0b509b..2dcd5b6 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -17,9 +17,9 @@ from copy import deepcopy import errno import io try: - from itertools import filterfalse + from itertools import chain, filterfalse except ImportError: - from itertools import ifilterfalse as filterfalse + from itertools import chain, ifilterfalse as filterfalse import logging import re import shlex @@ -1177,22 +1177,26 @@ def apply_recursive_permissions(top, uid=-1, gid=-1, else: raise + # For bug 554084, always apply permissions to a directory before + # that directory is traversed. all_applied = True - for dirpath, dirnames, filenames in os.walk(top): - try: - applied = apply_secpass_permissions(dirpath, - uid=uid, gid=gid, mode=dirmode, mask=dirmask, - follow_links=follow_links) - if not applied: - all_applied = False - except PortageException as e: + + try: + applied = apply_secpass_permissions(top, + uid=uid, gid=gid, mode=dirmode, mask=dirmask, + follow_links=follow_links) + if not applied: all_applied = False - onerror(e) + except PortageException as e: + all_applied = False + onerror(e) - for name in filenames: + for dirpath, dirnames, filenames in os.walk(top): + for name, mode in chain(((x, filemode) for x in filenames), + ((x, dirmode) for x in dirnames)): try: applied = apply_secpass_permissions(os.path.join(dirpath, name), - uid=uid, gid=gid, mode=filemode, mask=filemask, + uid=uid, gid=gid, mode=mode, mask=filemask, follow_links=follow_links) if not applied: all_applied = False -- 2.4.6 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* [gentoo-portage-dev] [PATCH v3] unpack: use chmod-lite helper for bug 554084 2015-10-02 18:57 ` [gentoo-portage-dev] [PATCH v2] " Zac Medico @ 2015-10-02 19:11 ` Zac Medico 2015-10-03 16:15 ` [gentoo-portage-dev] [PATCH v4] " Zac Medico 1 sibling, 0 replies; 6+ messages in thread From: Zac Medico @ 2015-10-02 19:11 UTC (permalink / raw To: gentoo-portage-dev; +Cc: Zac Medico Use the apply_recursive_permissions function to minimize the number of chmod calls. Also, fix an UnboundLocalError triggered in portage.data._get_global by chmod-lite. X-Gentoo-Bug: 554084 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554084 --- [PATCH v3] fixes apply_recursive_permissions to properly separate modes and masks for files and directories bin/chmod-lite | 10 +++++ bin/chmod-lite.py | 30 +++++++++++++++ bin/phase-helpers.sh | 2 +- pym/portage/data.py | 2 +- pym/portage/util/__init__.py | 88 ++++++++++++++++++++++---------------------- 5 files changed, 87 insertions(+), 45 deletions(-) create mode 100755 bin/chmod-lite create mode 100755 bin/chmod-lite.py diff --git a/bin/chmod-lite b/bin/chmod-lite new file mode 100755 index 0000000..ffa8d4d --- /dev/null +++ b/bin/chmod-lite @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +export __PORTAGE_HELPER_CWD=${PWD} + +# Use safe cwd, avoiding unsafe import for bug #469338. +cd "${PORTAGE_PYM_PATH}" || exit 1 +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + exec "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH/chmod-lite.py" "$@" diff --git a/bin/chmod-lite.py b/bin/chmod-lite.py new file mode 100755 index 0000000..a552b25 --- /dev/null +++ b/bin/chmod-lite.py @@ -0,0 +1,30 @@ +#!/usr/bin/python -b +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import sys + +from portage.util import apply_recursive_permissions + +# Change back to original cwd _after_ all imports (bug #469338). +os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) + +def main(files): + + if sys.hexversion >= 0x3000000: + # We can't trust that the filesystem encoding (locale dependent) + # correctly matches the arguments, so use surrogateescape to + # pass through the original argv bytes for Python 3. + fs_encoding = sys.getfilesystemencoding() + files = [x.encode(fs_encoding, 'surrogateescape') for x in files] + + for filename in files: + # Emulate 'chmod -fR a+rX,u+w,g-w,o-w' with mininal chmod calls. + apply_recursive_permissions(filename, filemode=0o644, + filemask=0o022, dirmode=0o755, dirmask=0o022) + + return os.EX_OK + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh index efd2cfa..0c25ffe 100644 --- a/bin/phase-helpers.sh +++ b/bin/phase-helpers.sh @@ -532,7 +532,7 @@ unpack() { # Do not chmod '.' since it's probably ${WORKDIR} and PORTAGE_WORKDIR_MODE # should be preserved. find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ - ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w + ${XARGS} -0 "${PORTAGE_BIN_PATH}/chmod-lite" } econf() { diff --git a/pym/portage/data.py b/pym/portage/data.py index 2fd287d..2c99548 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -139,7 +139,7 @@ def _get_global(k): v = 2 elif unprivileged: v = 2 - elif portage_gid in os.getgroups(): + elif _get_global('portage_gid') in os.getgroups(): v = 1 elif k in ('portage_gid', 'portage_uid'): diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index c0b509b..38780a7 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -17,9 +17,9 @@ from copy import deepcopy import errno import io try: - from itertools import filterfalse + from itertools import chain, filterfalse except ImportError: - from itertools import ifilterfalse as filterfalse + from itertools import chain, ifilterfalse as filterfalse import logging import re import shlex @@ -1041,6 +1041,23 @@ def unique_everseen(iterable, key=None): seen_add(k) yield element +def _do_stat(filename, follow_links=True): + try: + if follow_links: + return os.stat(filename) + else: + return os.lstat(filename) + except OSError as oe: + func_call = "stat('%s')" % filename + if oe.errno == errno.EPERM: + raise OperationNotPermitted(func_call) + elif oe.errno == errno.EACCES: + raise PermissionDenied(func_call) + elif oe.errno == errno.ENOENT: + raise FileNotFound(filename) + else: + raise + def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, stat_cached=None, follow_links=True): """Apply user, group, and mode bits to a file if the existing bits do not @@ -1058,21 +1075,7 @@ def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, gid = int(gid) if stat_cached is None: - try: - if follow_links: - stat_cached = os.stat(filename) - else: - stat_cached = os.lstat(filename) - except OSError as oe: - func_call = "stat('%s')" % filename - if oe.errno == errno.EPERM: - raise OperationNotPermitted(func_call) - elif oe.errno == errno.EACCES: - raise PermissionDenied(func_call) - elif oe.errno == errno.ENOENT: - raise FileNotFound(filename) - else: - raise + stat_cached = _do_stat(filename, follow_links=follow_links) if (uid != -1 and uid != stat_cached.st_uid) or \ (gid != -1 and gid != stat_cached.st_gid): @@ -1177,22 +1180,35 @@ def apply_recursive_permissions(top, uid=-1, gid=-1, else: raise + # For bug 554084, always apply permissions to a directory before + # that directory is traversed. all_applied = True - for dirpath, dirnames, filenames in os.walk(top): - try: - applied = apply_secpass_permissions(dirpath, - uid=uid, gid=gid, mode=dirmode, mask=dirmask, - follow_links=follow_links) - if not applied: - all_applied = False - except PortageException as e: + + stat_cached = _do_stat(top, follow_links=follow_links) + if stat.S_ISDIR(stat_cached.st_mode): + mode = dirmode + mask = dirmask + else: + mode = filemode + mask = filemask + + try: + applied = apply_secpass_permissions(top, + uid=uid, gid=gid, mode=mode, mask=mask, + stat_cached=stat_cached, follow_links=follow_links) + if not applied: all_applied = False - onerror(e) + except PortageException as e: + all_applied = False + onerror(e) - for name in filenames: + for dirpath, dirnames, filenames in os.walk(top): + for name, mode, mask in chain( + ((x, filemode, filemask) for x in filenames), + ((x, dirmode, dirmask) for x in dirnames)): try: applied = apply_secpass_permissions(os.path.join(dirpath, name), - uid=uid, gid=gid, mode=filemode, mask=filemask, + uid=uid, gid=gid, mode=mode, mask=mask, follow_links=follow_links) if not applied: all_applied = False @@ -1216,21 +1232,7 @@ def apply_secpass_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, unapplied.""" if stat_cached is None: - try: - if follow_links: - stat_cached = os.stat(filename) - else: - stat_cached = os.lstat(filename) - except OSError as oe: - func_call = "stat('%s')" % filename - if oe.errno == errno.EPERM: - raise OperationNotPermitted(func_call) - elif oe.errno == errno.EACCES: - raise PermissionDenied(func_call) - elif oe.errno == errno.ENOENT: - raise FileNotFound(filename) - else: - raise + stat_cached = _do_stat(filename, follow_links=follow_links) all_applied = True -- 2.4.6 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* [gentoo-portage-dev] [PATCH v4] unpack: use chmod-lite helper for bug 554084 2015-10-02 18:57 ` [gentoo-portage-dev] [PATCH v2] " Zac Medico 2015-10-02 19:11 ` [gentoo-portage-dev] [PATCH v3] " Zac Medico @ 2015-10-03 16:15 ` Zac Medico 2015-10-03 16:24 ` Brian Dolbec 1 sibling, 1 reply; 6+ messages in thread From: Zac Medico @ 2015-10-03 16:15 UTC (permalink / raw To: gentoo-portage-dev; +Cc: Zac Medico Use the apply_recursive_permissions function to minimize the number of chmod calls. Also, fix an UnboundLocalError triggered in portage.data._get_global by chmod-lite. X-Gentoo-Bug: 554084 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554084 --- [PATCH v4] adds backward compatibility to apply_recursive_permissions for the case where the given path does not exist bin/chmod-lite | 10 +++++ bin/chmod-lite.py | 30 ++++++++++++++ bin/phase-helpers.sh | 2 +- pym/portage/data.py | 2 +- pym/portage/util/__init__.py | 93 ++++++++++++++++++++++++-------------------- 5 files changed, 92 insertions(+), 45 deletions(-) create mode 100755 bin/chmod-lite create mode 100755 bin/chmod-lite.py diff --git a/bin/chmod-lite b/bin/chmod-lite new file mode 100755 index 0000000..ffa8d4d --- /dev/null +++ b/bin/chmod-lite @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +export __PORTAGE_HELPER_CWD=${PWD} + +# Use safe cwd, avoiding unsafe import for bug #469338. +cd "${PORTAGE_PYM_PATH}" || exit 1 +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + exec "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH/chmod-lite.py" "$@" diff --git a/bin/chmod-lite.py b/bin/chmod-lite.py new file mode 100755 index 0000000..177be7e --- /dev/null +++ b/bin/chmod-lite.py @@ -0,0 +1,30 @@ +#!/usr/bin/python -b +# Copyright 2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import sys + +from portage.util import apply_recursive_permissions + +# Change back to original cwd _after_ all imports (bug #469338). +os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) + +def main(files): + + if sys.hexversion >= 0x3000000: + # We can't trust that the filesystem encoding (locale dependent) + # correctly matches the arguments, so use surrogateescape to + # pass through the original argv bytes for Python 3. + fs_encoding = sys.getfilesystemencoding() + files = [x.encode(fs_encoding, 'surrogateescape') for x in files] + + for filename in files: + # Emulate 'chmod -fR a+rX,u+w,g-w,o-w' with minimal chmod calls. + apply_recursive_permissions(filename, filemode=0o644, + filemask=0o022, dirmode=0o755, dirmask=0o022) + + return os.EX_OK + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh index efd2cfa..0c25ffe 100644 --- a/bin/phase-helpers.sh +++ b/bin/phase-helpers.sh @@ -532,7 +532,7 @@ unpack() { # Do not chmod '.' since it's probably ${WORKDIR} and PORTAGE_WORKDIR_MODE # should be preserved. find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ - ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w + ${XARGS} -0 "${PORTAGE_BIN_PATH}/chmod-lite" } econf() { diff --git a/pym/portage/data.py b/pym/portage/data.py index 2fd287d..2c99548 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -139,7 +139,7 @@ def _get_global(k): v = 2 elif unprivileged: v = 2 - elif portage_gid in os.getgroups(): + elif _get_global('portage_gid') in os.getgroups(): v = 1 elif k in ('portage_gid', 'portage_uid'): diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index c0b509b..2b7ff8d 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -17,9 +17,9 @@ from copy import deepcopy import errno import io try: - from itertools import filterfalse + from itertools import chain, filterfalse except ImportError: - from itertools import ifilterfalse as filterfalse + from itertools import chain, ifilterfalse as filterfalse import logging import re import shlex @@ -1041,6 +1041,23 @@ def unique_everseen(iterable, key=None): seen_add(k) yield element +def _do_stat(filename, follow_links=True): + try: + if follow_links: + return os.stat(filename) + else: + return os.lstat(filename) + except OSError as oe: + func_call = "stat('%s')" % filename + if oe.errno == errno.EPERM: + raise OperationNotPermitted(func_call) + elif oe.errno == errno.EACCES: + raise PermissionDenied(func_call) + elif oe.errno == errno.ENOENT: + raise FileNotFound(filename) + else: + raise + def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, stat_cached=None, follow_links=True): """Apply user, group, and mode bits to a file if the existing bits do not @@ -1058,21 +1075,7 @@ def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, gid = int(gid) if stat_cached is None: - try: - if follow_links: - stat_cached = os.stat(filename) - else: - stat_cached = os.lstat(filename) - except OSError as oe: - func_call = "stat('%s')" % filename - if oe.errno == errno.EPERM: - raise OperationNotPermitted(func_call) - elif oe.errno == errno.EACCES: - raise PermissionDenied(func_call) - elif oe.errno == errno.ENOENT: - raise FileNotFound(filename) - else: - raise + stat_cached = _do_stat(filename, follow_links=follow_links) if (uid != -1 and uid != stat_cached.st_uid) or \ (gid != -1 and gid != stat_cached.st_gid): @@ -1177,22 +1180,40 @@ def apply_recursive_permissions(top, uid=-1, gid=-1, else: raise + # For bug 554084, always apply permissions to a directory before + # that directory is traversed. all_applied = True - for dirpath, dirnames, filenames in os.walk(top): - try: - applied = apply_secpass_permissions(dirpath, - uid=uid, gid=gid, mode=dirmode, mask=dirmask, - follow_links=follow_links) - if not applied: - all_applied = False - except PortageException as e: + + try: + stat_cached = _do_stat(top, follow_links=follow_links) + except FileNotFound: + # backward compatibility + return True + + if stat.S_ISDIR(stat_cached.st_mode): + mode = dirmode + mask = dirmask + else: + mode = filemode + mask = filemask + + try: + applied = apply_secpass_permissions(top, + uid=uid, gid=gid, mode=mode, mask=mask, + stat_cached=stat_cached, follow_links=follow_links) + if not applied: all_applied = False - onerror(e) + except PortageException as e: + all_applied = False + onerror(e) - for name in filenames: + for dirpath, dirnames, filenames in os.walk(top): + for name, mode, mask in chain( + ((x, filemode, filemask) for x in filenames), + ((x, dirmode, dirmask) for x in dirnames)): try: applied = apply_secpass_permissions(os.path.join(dirpath, name), - uid=uid, gid=gid, mode=filemode, mask=filemask, + uid=uid, gid=gid, mode=mode, mask=mask, follow_links=follow_links) if not applied: all_applied = False @@ -1216,21 +1237,7 @@ def apply_secpass_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, unapplied.""" if stat_cached is None: - try: - if follow_links: - stat_cached = os.stat(filename) - else: - stat_cached = os.lstat(filename) - except OSError as oe: - func_call = "stat('%s')" % filename - if oe.errno == errno.EPERM: - raise OperationNotPermitted(func_call) - elif oe.errno == errno.EACCES: - raise PermissionDenied(func_call) - elif oe.errno == errno.ENOENT: - raise FileNotFound(filename) - else: - raise + stat_cached = _do_stat(filename, follow_links=follow_links) all_applied = True -- 2.4.6 ^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [gentoo-portage-dev] [PATCH v4] unpack: use chmod-lite helper for bug 554084 2015-10-03 16:15 ` [gentoo-portage-dev] [PATCH v4] " Zac Medico @ 2015-10-03 16:24 ` Brian Dolbec 0 siblings, 0 replies; 6+ messages in thread From: Brian Dolbec @ 2015-10-03 16:24 UTC (permalink / raw To: gentoo-portage-dev On Sat, 3 Oct 2015 09:15:56 -0700 Zac Medico <zmedico@gentoo.org> wrote: > Use the apply_recursive_permissions function to minimize the number of > chmod calls. > > Also, fix an UnboundLocalError triggered in portage.data._get_global > by chmod-lite. > > X-Gentoo-Bug: 554084 > X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554084 > --- > [PATCH v4] adds backward compatibility to apply_recursive_permissions > for the case where the given path does not exist > > bin/chmod-lite | 10 +++++ > bin/chmod-lite.py | 30 ++++++++++++++ > bin/phase-helpers.sh | 2 +- > pym/portage/data.py | 2 +- > pym/portage/util/__init__.py | 93 > ++++++++++++++++++++++++-------------------- 5 files changed, 92 > insertions(+), 45 deletions(-) create mode 100755 bin/chmod-lite > create mode 100755 bin/chmod-lite.py > > diff --git a/bin/chmod-lite b/bin/chmod-lite > new file mode 100755 > index 0000000..ffa8d4d > --- /dev/null > +++ b/bin/chmod-lite > @@ -0,0 +1,10 @@ > +#!/bin/bash > +# Copyright 2015 Gentoo Foundation > +# Distributed under the terms of the GNU General Public License v2 > + > +export __PORTAGE_HELPER_CWD=${PWD} > + > +# Use safe cwd, avoiding unsafe import for bug #469338. > +cd "${PORTAGE_PYM_PATH}" || exit 1 > +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ > + exec "${PORTAGE_PYTHON:-/usr/bin/python}" > "$PORTAGE_BIN_PATH/chmod-lite.py" "$@" diff --git a/bin/chmod-lite.py > b/bin/chmod-lite.py new file mode 100755 > index 0000000..177be7e > --- /dev/null > +++ b/bin/chmod-lite.py > @@ -0,0 +1,30 @@ > +#!/usr/bin/python -b > +# Copyright 2015 Gentoo Foundation > +# Distributed under the terms of the GNU General Public License v2 > + > +import os > +import sys > + > +from portage.util import apply_recursive_permissions > + > +# Change back to original cwd _after_ all imports (bug #469338). > +os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) > + > +def main(files): > + > + if sys.hexversion >= 0x3000000: > + # We can't trust that the filesystem encoding > (locale dependent) > + # correctly matches the arguments, so use > surrogateescape to > + # pass through the original argv bytes for Python 3. > + fs_encoding = sys.getfilesystemencoding() > + files = [x.encode(fs_encoding, 'surrogateescape') > for x in files] + > + for filename in files: > + # Emulate 'chmod -fR a+rX,u+w,g-w,o-w' with minimal > chmod calls. > + apply_recursive_permissions(filename, filemode=0o644, > + filemask=0o022, dirmode=0o755, dirmask=0o022) > + > + return os.EX_OK > + > +if __name__ == "__main__": > + sys.exit(main(sys.argv[1:])) > diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh > index efd2cfa..0c25ffe 100644 > --- a/bin/phase-helpers.sh > +++ b/bin/phase-helpers.sh > @@ -532,7 +532,7 @@ unpack() { > # Do not chmod '.' since it's probably ${WORKDIR} and > PORTAGE_WORKDIR_MODE # should be preserved. > find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ > - ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w > + ${XARGS} -0 "${PORTAGE_BIN_PATH}/chmod-lite" > } > > econf() { > diff --git a/pym/portage/data.py b/pym/portage/data.py > index 2fd287d..2c99548 100644 > --- a/pym/portage/data.py > +++ b/pym/portage/data.py > @@ -139,7 +139,7 @@ def _get_global(k): > v = 2 > elif unprivileged: > v = 2 > - elif portage_gid in os.getgroups(): > + elif _get_global('portage_gid') in os.getgroups(): > v = 1 > > elif k in ('portage_gid', 'portage_uid'): > diff --git a/pym/portage/util/__init__.py > b/pym/portage/util/__init__.py index c0b509b..2b7ff8d 100644 > --- a/pym/portage/util/__init__.py > +++ b/pym/portage/util/__init__.py > @@ -17,9 +17,9 @@ from copy import deepcopy > import errno > import io > try: > - from itertools import filterfalse > + from itertools import chain, filterfalse > except ImportError: > - from itertools import ifilterfalse as filterfalse > + from itertools import chain, ifilterfalse as filterfalse > import logging > import re > import shlex > @@ -1041,6 +1041,23 @@ def unique_everseen(iterable, key=None): > seen_add(k) > yield element > > +def _do_stat(filename, follow_links=True): > + try: > + if follow_links: > + return os.stat(filename) > + else: > + return os.lstat(filename) > + except OSError as oe: > + func_call = "stat('%s')" % filename > + if oe.errno == errno.EPERM: > + raise OperationNotPermitted(func_call) > + elif oe.errno == errno.EACCES: > + raise PermissionDenied(func_call) > + elif oe.errno == errno.ENOENT: > + raise FileNotFound(filename) > + else: > + raise > + > def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, > stat_cached=None, follow_links=True): > """Apply user, group, and mode bits to a file if the > existing bits do not @@ -1058,21 +1075,7 @@ def > apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, gid = > int(gid) > if stat_cached is None: > - try: > - if follow_links: > - stat_cached = os.stat(filename) > - else: > - stat_cached = os.lstat(filename) > - except OSError as oe: > - func_call = "stat('%s')" % filename > - if oe.errno == errno.EPERM: > - raise > OperationNotPermitted(func_call) > - elif oe.errno == errno.EACCES: > - raise PermissionDenied(func_call) > - elif oe.errno == errno.ENOENT: > - raise FileNotFound(filename) > - else: > - raise > + stat_cached = _do_stat(filename, > follow_links=follow_links) > if (uid != -1 and uid != stat_cached.st_uid) or \ > (gid != -1 and gid != stat_cached.st_gid): > @@ -1177,22 +1180,40 @@ def apply_recursive_permissions(top, uid=-1, > gid=-1, else: > raise > > + # For bug 554084, always apply permissions to a directory > before > + # that directory is traversed. > all_applied = True > - for dirpath, dirnames, filenames in os.walk(top): > - try: > - applied = apply_secpass_permissions(dirpath, > - uid=uid, gid=gid, mode=dirmode, > mask=dirmask, > - follow_links=follow_links) > - if not applied: > - all_applied = False > - except PortageException as e: > + > + try: > + stat_cached = _do_stat(top, > follow_links=follow_links) > + except FileNotFound: > + # backward compatibility > + return True > + > + if stat.S_ISDIR(stat_cached.st_mode): > + mode = dirmode > + mask = dirmask > + else: > + mode = filemode > + mask = filemask > + > + try: > + applied = apply_secpass_permissions(top, > + uid=uid, gid=gid, mode=mode, mask=mask, > + stat_cached=stat_cached, > follow_links=follow_links) > + if not applied: > all_applied = False > - onerror(e) > + except PortageException as e: > + all_applied = False > + onerror(e) > > - for name in filenames: > + for dirpath, dirnames, filenames in os.walk(top): > + for name, mode, mask in chain( > + ((x, filemode, filemask) for x in filenames), > + ((x, dirmode, dirmask) for x in dirnames)): > try: > applied = > apply_secpass_permissions(os.path.join(dirpath, name), > - uid=uid, gid=gid, > mode=filemode, mask=filemask, > + uid=uid, gid=gid, mode=mode, > mask=mask, follow_links=follow_links) > if not applied: > all_applied = False > @@ -1216,21 +1237,7 @@ def apply_secpass_permissions(filename, > uid=-1, gid=-1, mode=-1, mask=-1, unapplied.""" > > if stat_cached is None: > - try: > - if follow_links: > - stat_cached = os.stat(filename) > - else: > - stat_cached = os.lstat(filename) > - except OSError as oe: > - func_call = "stat('%s')" % filename > - if oe.errno == errno.EPERM: > - raise > OperationNotPermitted(func_call) > - elif oe.errno == errno.EACCES: > - raise PermissionDenied(func_call) > - elif oe.errno == errno.ENOENT: > - raise FileNotFound(filename) > - else: > - raise > + stat_cached = _do_stat(filename, > follow_links=follow_links) > all_applied = True > LGTM -- Brian Dolbec <dolsen> ^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2015-10-03 16:25 UTC | newest] Thread overview: 6+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2015-10-02 7:22 [gentoo-portage-dev] [PATCH] unpack: use chmod-lite helper for bug 554084 Zac Medico 2015-10-02 7:32 ` [gentoo-portage-dev] " Zac Medico 2015-10-02 18:57 ` [gentoo-portage-dev] [PATCH v2] " Zac Medico 2015-10-02 19:11 ` [gentoo-portage-dev] [PATCH v3] " Zac Medico 2015-10-03 16:15 ` [gentoo-portage-dev] [PATCH v4] " Zac Medico 2015-10-03 16:24 ` Brian Dolbec
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox