From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 39FE41389F5 for ; Sun, 16 Nov 2014 10:42:28 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id ED8A8E09E7; Sun, 16 Nov 2014 10:42:22 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id BD527E09E5 for ; Sun, 16 Nov 2014 10:42:21 +0000 (UTC) Received: from localhost.localdomain (ip70-181-96-121.oc.oc.cox.net [70.181.96.121]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) (Authenticated sender: zmedico) by smtp.gentoo.org (Postfix) with ESMTPSA id F10BC3404DE; Sun, 16 Nov 2014 10:42:20 +0000 (UTC) From: Zac Medico To: gentoo-portage-dev@lists.gentoo.org Cc: Zac Medico Subject: [gentoo-portage-dev] [PATCH 2/2] FEATURES=case-insensitive-fs for bug #524236 Date: Sun, 16 Nov 2014 02:41:55 -0800 Message-Id: <1416134515-31943-2-git-send-email-zmedico@gentoo.org> X-Mailer: git-send-email 2.0.4 In-Reply-To: <1416134515-31943-1-git-send-email-zmedico@gentoo.org> References: <5464F129.6080404@gentoo.org> <1416134515-31943-1-git-send-email-zmedico@gentoo.org> Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org X-Archives-Salt: 9d201a20-2ef4-48bf-b0bd-fd0f88a9ede5 X-Archives-Hash: c07e64bb76093ecec4cdc72fbe0d36e2 When case-insensitive-fs is enabled in FEATURES, the dblink.isowner method, _owners_db class, and ConfigProtect class will be case-insensitive. This causes the collision-protect and unmerge code to behave correctly for a case-insensitive file system. If the file system is case-insensitive but case-preserving, then case is preserved in the CONTENTS data of installed packages. X-Gentoo-Bug: 524236 X-Gentoo-Url: https://bugs.gentoo.org/show_bug.cgi?id=524236 --- bin/dispatch-conf | 8 ++++++- bin/etc-update | 10 ++++++-- bin/portageq | 7 +++--- bin/quickpkg | 4 +++- man/make.conf.5 | 6 +++++ pym/_emerge/depgraph.py | 4 +++- pym/portage/_global_updates.py | 4 +++- pym/portage/const.py | 1 + pym/portage/dbapi/vartree.py | 53 +++++++++++++++++++++++++++--------------- pym/portage/update.py | 6 +++-- pym/portage/util/__init__.py | 8 ++++++- 11 files changed, 80 insertions(+), 31 deletions(-) diff --git a/bin/dispatch-conf b/bin/dispatch-conf index 8058d6f..b679910 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -35,6 +35,10 @@ from portage.process import find_binary, spawn FIND_EXTANT_CONFIGS = "find '%s' %s -name '._cfg????_%s' ! -name '.*~' ! -iname '.*.bak' -print" DIFF_CONTENTS = "diff -Nu '%s' '%s'" +if "case-insensitive-fs" in portage.settings.features: + FIND_EXTANT_CONFIGS = FIND_EXTANT_CONFIGS.replace( + "-name '._cfg", "-iname '._cfg") + # We need a secure scratch dir and python does silly verbose errors on the use of tempnam oldmask = os.umask(0o077) SCRATCH_DIR = None @@ -152,7 +156,9 @@ class dispatch: protect_obj = portage.util.ConfigProtect( config_root, config_paths, portage.util.shlex_split( - portage.settings.get('CONFIG_PROTECT_MASK', ''))) + portage.settings.get('CONFIG_PROTECT_MASK', '')), + case_insensitive=("case-insensitive-fs" + in portage.settings.features)) # # Remove new configs identical to current diff --git a/bin/etc-update b/bin/etc-update index 0307688..e0f7224 100755 --- a/bin/etc-update +++ b/bin/etc-update @@ -113,12 +113,15 @@ scan() { [[ -d ${path%/*} ]] || continue local name_opt=$(get_basename_find_opt "${path##*/}") path="${path%/*}" - find_opts=( -maxdepth 1 -name "$name_opt" ) + find_opts=( -maxdepth 1 ) else # Do not traverse hidden directories such as .svn or .git. local name_opt=$(get_basename_find_opt '*') - find_opts=( -name '.*' -type d -prune -o -name "$name_opt" ) + find_opts=( -name '.*' -type d -prune -o ) fi + ${case_insensitive} && \ + find_opts+=( -iname ) || find_opts+=( -name ) + find_opts+=( "$name_opt" ) find_opts+=( ! -name '.*~' ! -iname '.*.bak' -print ) if [ ! -w "${path}" ] ; then @@ -743,6 +746,7 @@ fi portage_vars=( CONFIG_PROTECT{,_MASK} + FEATURES PORTAGE_CONFIGROOT PORTAGE_INST_{G,U}ID PORTAGE_TMPDIR @@ -759,6 +763,8 @@ fi export PORTAGE_TMPDIR SCAN_PATHS=${*:-${CONFIG_PROTECT}} +[[ " ${FEATURES} " == *" case-insensitive-fs "* ]] && \ + case_insensitive=true || case_insensitive=false TMP="${PORTAGE_TMPDIR}/etc-update-$$" trap "die terminated" SIGTERM diff --git a/bin/portageq b/bin/portageq index ef565d1..1618b44 100755 --- a/bin/portageq +++ b/bin/portageq @@ -379,8 +379,8 @@ def is_protected(argv): protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) protect_mask = portage.util.shlex_split( settings.get("CONFIG_PROTECT_MASK", "")) - protect_obj = ConfigProtect(root, protect, protect_mask) - + protect_obj = ConfigProtect(root, protect, protect_mask, + case_insensitive=("case-insensitive-fs" in settings.features)) if protect_obj.isprotected(f): return 0 return 1 @@ -414,7 +414,8 @@ def filter_protected(argv): protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) protect_mask = portage.util.shlex_split( settings.get("CONFIG_PROTECT_MASK", "")) - protect_obj = ConfigProtect(root, protect, protect_mask) + protect_obj = ConfigProtect(root, protect, protect_mask, + case_insensitive=("case-insensitive-fs" in settings.features)) errors = 0 diff --git a/bin/quickpkg b/bin/quickpkg index cf75791..2c69a69 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -102,7 +102,9 @@ def quickpkg_atom(options, infos, arg, eout): if not include_config: confprot = ConfigProtect(eroot, shlex_split(settings.get("CONFIG_PROTECT", "")), - shlex_split(settings.get("CONFIG_PROTECT_MASK", ""))) + shlex_split(settings.get("CONFIG_PROTECT_MASK", "")), + case_insensitive=("case-insensitive-fs" + in settings.features)) def protect(filename): if not confprot.isprotected(filename): return False diff --git a/man/make.conf.5 b/man/make.conf.5 index 84e894b..69d95fc 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -265,6 +265,12 @@ Build binary packages for just packages in the system set. Enable a special progress indicator when \fBemerge\fR(1) is calculating dependencies. .TP +.B case\-insensitive\-fs +Use case\-insensitive file name comparisions when merging and unmerging +files. Most users should not enable this feature, since most filesystems +are case\-sensitive. You should only enable this feature if you are +using portage to install files to a case\-insensitive filesystem. +.TP .B ccache Enable portage support for the ccache package. If the ccache dir is not present in the user's environment, then portage will default to diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 94eaed8..6bf2be5 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -7805,7 +7805,9 @@ class depgraph(object): settings = self._frozen_config.roots[root].settings protect_obj[root] = ConfigProtect(settings["EROOT"], \ shlex_split(settings.get("CONFIG_PROTECT", "")), - shlex_split(settings.get("CONFIG_PROTECT_MASK", ""))) + shlex_split(settings.get("CONFIG_PROTECT_MASK", "")), + case_insensitive=("case-insensitive-fs" + in settings.features)) def write_changes(root, changes, file_to_write_to): file_contents = None diff --git a/pym/portage/_global_updates.py b/pym/portage/_global_updates.py index 17dc080..87085de 100644 --- a/pym/portage/_global_updates.py +++ b/pym/portage/_global_updates.py @@ -208,7 +208,9 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): update_config_files(root, shlex_split(mysettings.get("CONFIG_PROTECT", "")), shlex_split(mysettings.get("CONFIG_PROTECT_MASK", "")), - repo_map, match_callback=_config_repo_match) + repo_map, match_callback = _config_repo_match, + case_insensitive="case-insensitive-fs" + in mysettings.features) # The above global updates proceed quickly, so they # are considered a single mtimedb transaction. diff --git a/pym/portage/const.py b/pym/portage/const.py index d472075..febdb4a 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -125,6 +125,7 @@ SUPPORTED_FEATURES = frozenset([ "buildpkg", "buildsyspkg", "candy", + "case-insensitive-fs", "ccache", "cgroup", "chflags", diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 2d1003f..bf7985a 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1051,13 +1051,13 @@ class vardbapi(dbapi): def add(self, cpv): eroot_len = len(self._vardb._eroot) - contents = self._vardb._dblink(cpv).getcontents() pkg_hash = self._hash_pkg(cpv) - if not contents: + db = self._vardb._dblink(cpv) + if not db.getcontents(): # Empty path is a code used to represent empty contents. self._add_path("", pkg_hash) - for x in contents: + for x in db._contents_iter(): self._add_path(x[eroot_len:], pkg_hash) self._vardb._aux_cache["modified"].add(cpv) @@ -1189,6 +1189,8 @@ class vardbapi(dbapi): hash_pkg = owners_cache._hash_pkg hash_str = owners_cache._hash_str base_names = self._vardb._aux_cache["owners"]["base_names"] + case_insensitive = "case-insensitive-fs" \ + in vardb.settings.features dblink_cache = {} @@ -1205,6 +1207,8 @@ class vardbapi(dbapi): while path_iter: path = path_iter.pop() + if case_insensitive: + path = path.lower() is_basename = os.sep != path[:1] if is_basename: name = path @@ -1235,7 +1239,7 @@ class vardbapi(dbapi): continue if is_basename: - for p in dblink(cpv).getcontents(): + for p in dblink(cpv)._contents_iter(): if os.path.basename(p) == name: owners.append((cpv, p[len(root):])) else: @@ -1265,8 +1269,12 @@ class vardbapi(dbapi): if not path_list: return + case_insensitive = "case-insensitive-fs" \ + in self._vardb.settings.features path_info_list = [] for path in path_list: + if case_insensitive: + path = path.lower() is_basename = os.sep != path[:1] if is_basename: name = path @@ -1284,9 +1292,11 @@ class vardbapi(dbapi): dblnk = self._vardb._dblink(cpv) for path, name, is_basename in path_info_list: if is_basename: - for p in dblnk.getcontents(): + for p in dblnk._contents_iter(): if os.path.basename(p) == name: - search_pkg.results.append((dblnk, p[len(root):])) + search_pkg.results.append((dblnk, + dblnk._contents_case_reverse_map( + p)[len(root):])) else: if dblnk.isowner(path): search_pkg.results.append((dblnk, path)) @@ -1551,7 +1561,9 @@ class dblink(object): portage.util.shlex_split( self.settings.get("CONFIG_PROTECT", "")), portage.util.shlex_split( - self.settings.get("CONFIG_PROTECT_MASK", ""))) + self.settings.get("CONFIG_PROTECT_MASK", "")), + case_insensitive=("case-insensitive-fs" + in self.settings.features)) return self._protect_obj @@ -2804,15 +2816,18 @@ class dblink(object): os_filename_arg.path.join(destroot, filename.lstrip(os_filename_arg.path.sep))) - pkgfiles = self.getcontents() - if pkgfiles and destfile in pkgfiles: - return destfile - if pkgfiles: + if "case-insensitive-fs" in self.settings.features: + destfile = destfile.lower() + + if self._contents_contains(destfile): + return self._contents_key(destfile) + + if self.getcontents(): basename = os_filename_arg.path.basename(destfile) if self._contents_basenames is None: try: - for x in pkgfiles: + for x in self._contents_iter(): _unicode_encode(x, encoding=_encodings['merge'], errors='strict') @@ -2821,7 +2836,7 @@ class dblink(object): # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: - for x in pkgfiles: + for x in self._contents_iter(): _unicode_encode(x, encoding=_encodings['fs'], errors='strict') @@ -2831,7 +2846,7 @@ class dblink(object): os = portage.os self._contents_basenames = set( - os.path.basename(x) for x in pkgfiles) + os.path.basename(x) for x in self._contents_iter()) if basename not in self._contents_basenames: # This is a shortcut that, in most cases, allows us to # eliminate this package as an owner without the need @@ -2852,7 +2867,7 @@ class dblink(object): if os is _os_merge: try: - for x in pkgfiles: + for x in self._contents_iter(): _unicode_encode(x, encoding=_encodings['merge'], errors='strict') @@ -2861,7 +2876,7 @@ class dblink(object): # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: - for x in pkgfiles: + for x in self._contents_iter(): _unicode_encode(x, encoding=_encodings['fs'], errors='strict') @@ -2872,7 +2887,7 @@ class dblink(object): self._contents_inodes = {} parent_paths = set() - for x in pkgfiles: + for x in self._contents_iter(): p_path = os.path.dirname(x) if p_path in parent_paths: continue @@ -2897,8 +2912,8 @@ class dblink(object): if p_path_list: for p_path in p_path_list: x = os_filename_arg.path.join(p_path, basename) - if x in pkgfiles: - return x + if self._contents_contains(x): + return self._contents_key(x) return False diff --git a/pym/portage/update.py b/pym/portage/update.py index df4e11b..83fc3d2 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -282,7 +282,8 @@ def parse_updates(mycontent): myupd.append(mysplit) return myupd, errors -def update_config_files(config_root, protect, protect_mask, update_iter, match_callback = None): +def update_config_files(config_root, protect, protect_mask, update_iter, + match_callback=None, case_insensitive=False): """Perform global updates on /etc/portage/package.*, /etc/portage/profile/package.*, /etc/portage/profile/packages and /etc/portage/sets. config_root - location of files to update @@ -406,7 +407,8 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c sys.stdout.flush() protect_obj = ConfigProtect( - config_root, protect, protect_mask) + config_root, protect, protect_mask, + case_insensitive=case_insensitive) for x in update_files: updating_file = os.path.join(abs_user_config, x) if protect_obj.isprotected(updating_file): diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index ad3a351..d0cca5b 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -1555,10 +1555,12 @@ class LazyItemsDict(UserDict): return result class ConfigProtect(object): - def __init__(self, myroot, protect_list, mask_list): + def __init__(self, myroot, protect_list, mask_list, + case_insensitive=False): self.myroot = myroot self.protect_list = protect_list self.mask_list = mask_list + self.case_insensitive = case_insensitive self.updateprotect() def updateprotect(self): @@ -1586,6 +1588,8 @@ class ConfigProtect(object): for x in self.mask_list: ppath = normalize_path( os.path.join(self.myroot, x.lstrip(os.path.sep))) + if self.case_insensitive: + ppath = ppath.lower() try: """Use lstat so that anything, even a broken symlink can be protected.""" @@ -1606,6 +1610,8 @@ class ConfigProtect(object): masked = 0 protected = 0 sep = os.path.sep + if self.case_insensitive: + obj = obj.lower() for ppath in self.protect: if len(ppath) > masked and obj.startswith(ppath): if ppath in self._dirs: -- 2.0.4