From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 25F5615808B for ; Mon, 4 Apr 2022 19:05:04 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 4F9D1E0896; Mon, 4 Apr 2022 19:05:01 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [140.211.166.183]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id D2C8BE0882 for ; Mon, 4 Apr 2022 19:05:00 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 95EBC341295 for ; Mon, 4 Apr 2022 19:04:59 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id E1224388 for ; Mon, 4 Apr 2022 19:04:57 +0000 (UTC) From: "Sam James" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Sam James" Message-ID: <1649099074.9e24d0143450628f334cdb62e579efafd1bfd2ba.sam@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: lib/portage/ X-VCS-Repository: proj/portage X-VCS-Files: lib/portage/manifest.py lib/portage/metadata.py lib/portage/module.py lib/portage/news.py lib/portage/output.py X-VCS-Directories: lib/portage/ X-VCS-Committer: sam X-VCS-Committer-Name: Sam James X-VCS-Revision: 9e24d0143450628f334cdb62e579efafd1bfd2ba X-VCS-Branch: master Date: Mon, 4 Apr 2022 19:04:57 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 060ce4ee-12a5-4268-be24-6e541db3ae3e X-Archives-Hash: ee709862a864f817aaa01d2b59bc9460 commit: 9e24d0143450628f334cdb62e579efafd1bfd2ba Author: Kenneth Raplee kennethraplee com> AuthorDate: Sat Apr 2 01:13:57 2022 +0000 Commit: Sam James gentoo org> CommitDate: Mon Apr 4 19:04:34 2022 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e24d014 Simplify with declarative programming Signed-off-by: Kenneth Raplee kennethraplee.com> Signed-off-by: Sam James gentoo.org> lib/portage/manifest.py | 144 ++++++++++++++++++++++++------------------------ lib/portage/metadata.py | 15 ++--- lib/portage/module.py | 23 +++++--- lib/portage/news.py | 14 ++--- lib/portage/output.py | 13 ++--- 5 files changed, 105 insertions(+), 104 deletions(-) diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py index 5472e8fb1..ff166faa8 100644 --- a/lib/portage/manifest.py +++ b/lib/portage/manifest.py @@ -3,6 +3,7 @@ import errno import io +import itertools import logging import re import stat @@ -107,9 +108,8 @@ class Manifest2Entry(ManifestEntry): myhashkeys = list(self.hashes) myhashkeys.remove("size") myhashkeys.sort() - for h in myhashkeys: - myline += " " + h + " " + str(self.hashes[h]) - return myline + with_hashes = " ".join(f"{h} {self.hashes[h]}" for h in myhashkeys) + return f"{myline} {with_hashes}" def __eq__(self, other): if ( @@ -162,7 +162,6 @@ class Manifest: find_invalid_path_char = _find_invalid_path_char self._find_invalid_path_char = find_invalid_path_char self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep - self.fhashdict = {} self.hashes = set() self.required_hashes = set() @@ -182,8 +181,8 @@ class Manifest: self.required_hashes.update(required_hashes) self.required_hashes.intersection_update(self.hashes) - for t in MANIFEST2_IDENTIFIERS: - self.fhashdict[t] = {} + self.fhashdict = {t: {} for t in MANIFEST2_IDENTIFIERS} + if not from_scratch: self._read() if fetchlist_dict != None: @@ -206,9 +205,9 @@ class Manifest: def getDigests(self): """Compability function for old digest/manifest code, returns dict of filename:{hashfunction:hashvalue}""" - rval = {} - for t in MANIFEST2_IDENTIFIERS: - rval.update(self.fhashdict[t]) + rval = { + k: v for t in MANIFEST2_IDENTIFIERS for k, v in self.fhashdict[t].items() + } return rval def getTypeDigests(self, ftype): @@ -269,18 +268,16 @@ class Manifest: def _getDigestData(self, distlist): """create a hash dict for a specific list of files""" - myhashdict = {} - for myname in distlist: - for mytype in self.fhashdict: - if myname in self.fhashdict[mytype]: - myhashdict.setdefault(mytype, {}) - myhashdict[mytype].setdefault(myname, {}) - myhashdict[mytype][myname].update(self.fhashdict[mytype][myname]) + myhashdict = { + mytype: {myname: self.fhashdict[mytype][myname]} + for myname in distlist + for mytype in self.fhashdict + if myname in self.fhashdict[mytype] + } return myhashdict def _createManifestEntries(self): - valid_hashes = set(get_valid_checksum_keys()) - valid_hashes.add("size") + valid_hashes = set(itertools.chain(get_valid_checksum_keys(), ("size"))) mytypes = list(self.fhashdict) mytypes.sort() for t in mytypes: @@ -296,13 +293,22 @@ class Manifest: yield myentry def checkIntegrity(self): - for t in self.fhashdict: - for f in self.fhashdict[t]: - diff = self.required_hashes.difference(set(self.fhashdict[t][f])) - if diff: - raise MissingParameter( - _("Missing %s checksum(s): %s %s") % (" ".join(diff), t, f) + manifest_data = ( + ( + self.required_hashes.difference(set(self.fhashdict[mytype][myfile])), + mytype, + myfile, + ) + for mytype in self.fhashdict + for myfile in self.fhashdict[mytype] + ) + for needed_hashes, its_type, its_file in manifest_data: + if needed_hashes: + raise MissingParameter( + _( + f"Missing {' '.join(needed_hashes)} checksum(s): {its_type} {its_file}" ) + ) def write(self, sign=False, force=False): """Write Manifest instance to disk, optionally signing it. Returns @@ -488,10 +494,8 @@ class Manifest: def findFile(self, fname): """Return entrytype of the given file if present in Manifest or None if not present""" - for t in MANIFEST2_IDENTIFIERS: - if fname in self.fhashdict[t]: - return t - return None + found_entries = (t for t in MANIFEST2_IDENTIFIERS if fname in self.fhashdict[t]) + return next(found_entries, None) def create( self, @@ -529,18 +533,19 @@ class Manifest: find_invalid_path_char=self._find_invalid_path_char, strict_misc_digests=self.strict_misc_digests, ) - pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) - cat = self._pkgdir_category() - pkgdir = self.pkgdir + update_pkgdir = self._update_thick_pkgdir if self.thin: - cpvlist = self._update_thin_pkgdir(cat, pn, pkgdir) - else: - cpvlist = self._update_thick_pkgdir(cat, pn, pkgdir) + update_pkgdir = self._update_thin_pkgdir - distlist = set() - for cpv in cpvlist: - distlist.update(self._getCpvDistfiles(cpv)) + cpvlist = update_pkgdir( + self._pkgdir_category(), + os.path.basename(self.pkgdir.rstrip(os.path.sep)), + self.pkgdir, + ) + distlist = set( + distfile for cpv in cpvlist for distfile in self._getCpvDistfiles(cpv) + ) if requiredDistfiles is None: # This allows us to force removal of stale digests for the @@ -550,9 +555,7 @@ class Manifest: # repoman passes in an empty list, which implies that all distfiles # are required. requiredDistfiles = distlist.copy() - required_hash_types = set() - required_hash_types.add("size") - required_hash_types.update(self.required_hashes) + required_hash_types = set(itertools.chain(self.required_hashes, ("size"))) for f in distlist: fname = os.path.join(self.distdir, f) mystat = None @@ -599,25 +602,28 @@ class Manifest: return cpv def _update_thin_pkgdir(self, cat, pn, pkgdir): - for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir): - break - cpvlist = [] - for f in pkgdir_files: + _, _, pkgdir_files = next(os.walk(pkgdir), (None, None, None)) + + def _process_for_cpv(filename): try: - f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict") + filename = _unicode_decode( + filename, encoding=_encodings["fs"], errors="strict" + ) except UnicodeDecodeError: - continue - if f[:1] == ".": - continue - pf = self._is_cpv(cat, pn, f) + return None + if filename.startswith("."): + return None + pf = self._is_cpv(cat, pn, filename) if pf is not None: - cpvlist.append(pf) + return pf + + processed = (_process_for_cpv(filename) for filename in pkgdir_files) + cpvlist = [pf for pf in processed if pf] return cpvlist def _update_thick_pkgdir(self, cat, pn, pkgdir): + _, _, pkgdir_files = next(os.walk(pkgdir), (None, None, None)) cpvlist = [] - for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir): - break for f in pkgdir_files: try: f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict") @@ -714,9 +720,7 @@ class Manifest: return self.fetchlist_dict[cpv] def getDistfilesSize(self, fetchlist): - total_bytes = 0 - for f in fetchlist: - total_bytes += int(self.fhashdict["DIST"][f]["size"]) + total_bytes = sum(int(self.fhashdict["DIST"][f]["size"]) for f in fetchlist) return total_bytes def updateFileHashes( @@ -784,28 +788,26 @@ class Manifest: def getVersions(self): """Returns a list of manifest versions present in the manifest file.""" - rVal = [] mfname = self.getFullname() if not os.path.exists(mfname): - return rVal - myfile = io.open( + return [] + with io.open( _unicode_encode(mfname, encoding=_encodings["fs"], errors="strict"), mode="r", encoding=_encodings["repo.content"], errors="replace", - ) - lines = myfile.readlines() - myfile.close() - for l in lines: - mysplit = l.split() - if ( - len(mysplit) > 4 - and mysplit[0] in MANIFEST2_IDENTIFIERS - and ((len(mysplit) - 3) % 2) == 0 - and not 2 in rVal - ): - rVal.append(2) - return rVal + ) as myfile: + line_splits = (line.split() for line in myfile.readlines()) + validation = ( + True + for line_split in line_splits + if len(line_split) > 4 + and line_split[0] in MANIFEST2_IDENTIFIERS + and (len(line_split) - 3) % 2 == 0 + ) + if any(validation): + return [2] + return [] def _catsplit(self, pkg_key): """Split a category and package, returning a list of [cat, pkg]. diff --git a/lib/portage/metadata.py b/lib/portage/metadata.py index 357917051..869c10bb3 100644 --- a/lib/portage/metadata.py +++ b/lib/portage/metadata.py @@ -75,8 +75,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None): eclass_db.update_eclasses() porttrees_data.append(TreeData(portdb.auxdb[path], eclass_db, path, src_db)) - porttrees = [tree_data.path for tree_data in porttrees_data] - quiet = ( settings.get("TERM") == "dumb" or "--quiet" in myopts or not sys.stdout.isatty() ) @@ -100,7 +98,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None): # Temporarily override portdb.porttrees so portdb.cp_all() # will only return the relevant subset. portdb_porttrees = portdb.porttrees - portdb.porttrees = porttrees + portdb.porttrees = (tree_data.path for tree_data in porttrees_data) try: cp_all = portdb.cp_all() finally: @@ -119,7 +117,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None): for cp in cp_all: for tree_data in porttrees_data: - src_chf = tree_data.src_db.validation_chf dest_chf = tree_data.dest_db.validation_chf dest_chf_key = f"_{dest_chf}_" @@ -190,11 +187,11 @@ def action_metadata(settings, portdb, myopts, porttrees=None): # We don't want to skip the write unless we're really # sure that the existing cache is identical, so don't # trust _mtime_ and _eclasses_ alone. - for k in auxdbkeys: - if dest.get(k, "") != src.get(k, ""): - dest = None - break - + cache_is_identical = ( + True for k in auxdbkeys if dest.get(k, "") != src.get(k, "") + ) + if any(cache_is_identical): + dest = None if dest is not None: # The existing data is valid and identical, # so there's no need to overwrite it. diff --git a/lib/portage/module.py b/lib/portage/module.py index 61c85aa47..8e63cd545 100644 --- a/lib/portage/module.py +++ b/lib/portage/module.py @@ -110,19 +110,27 @@ class Modules: @rtype: dictionary of module_plugins """ module_dir = self._module_path - importables = [] names = os.listdir(module_dir) - for entry in names: - # skip any __init__ or __pycache__ files or directories - if entry.startswith("__"): - continue + + def _a_real_module(entry): try: # test for statinfo to ensure it should a real module # it will bail if it errors os.lstat(os.path.join(module_dir, entry, "__init__.py")) - importables.append(entry) except EnvironmentError: - pass + return False + return True + + # The importables list cannot be a generator. + # If it was a generator, it would be consumed by self.parents.extend() + # and the following for loop wouldn't have anything to iterate with. + importables = [ + entry + for entry in names + if not entry.startswith("__") and _a_real_module(entry) + ] + self.parents.extend(importables) + kids = {} for entry in importables: new_module = Module(entry, self._namepath) @@ -131,7 +139,6 @@ class Modules: kid = new_module.kids[module_name] kid["parent"] = new_module kids[kid["name"]] = kid - self.parents.append(entry) return kids def get_module_names(self): diff --git a/lib/portage/news.py b/lib/portage/news.py index 9ef6efde0..9f373d3d7 100644 --- a/lib/portage/news.py +++ b/lib/portage/news.py @@ -280,14 +280,12 @@ class NewsItem: kwargs = {"vardb": vardb, "config": config, "profile": profile} - all_match = True - for values in self.restrictions.values(): - any_match = False - for restriction in values: - if restriction.checkRestriction(**kwargs): - any_match = True - if not any_match: - all_match = False + all_match = all( + True + for values in self.restrictions.values() + for restriction in values + if restriction.checkRestriction(**kwargs) + ) return all_match diff --git a/lib/portage/output.py b/lib/portage/output.py index 33c477012..e20046fc5 100644 --- a/lib/portage/output.py +++ b/lib/portage/output.py @@ -5,6 +5,7 @@ __docformat__ = "epytext" import errno import io +import itertools import re import subprocess import sys @@ -74,16 +75,12 @@ codes["bg_darkyellow"] = codes["bg_brown"] def color(fg, bg="default", attr=["normal"]): - mystr = codes[fg] - for x in [bg] + attr: - mystr += codes[x] - return mystr + myansicodechain = itertools.chain((codes[fg]), (codes[x] for x in [bg, *attr])) + return "".join(myansicodechain) -ansi_codes = [] -for x in range(30, 38): - ansi_codes.append("%im" % x) - ansi_codes.append("%i;01m" % x) +ansi_codes = [y for x in range(30, 38) for y in (f"{x}m", f"{x};01m")] + rgb_ansi_colors = [ "0x000000",