From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id EC69D15800F for ; Fri, 20 Jan 2023 20:47:23 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id C83DBE0867; Fri, 20 Jan 2023 20:47:22 +0000 (UTC) Received: from smtp.gentoo.org (dev.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id AC712E0867 for ; Fri, 20 Jan 2023 20:47:22 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id A081F335D8D for ; Fri, 20 Jan 2023 20:47:21 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id D7D5F62F for ; Fri, 20 Jan 2023 20:47:19 +0000 (UTC) From: "Arthur Zamarin" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Arthur Zamarin" Message-ID: <1674247002.09559c09f2389246ea98261832e281a9baaedbdf.arthurzam@gentoo> Subject: [gentoo-commits] proj/pkgcore/pkgcheck:master commit in: src/pkgcheck/checks/, tests/checks/ X-VCS-Repository: proj/pkgcore/pkgcheck X-VCS-Files: src/pkgcheck/checks/git.py tests/checks/test_git.py X-VCS-Directories: src/pkgcheck/checks/ tests/checks/ X-VCS-Committer: arthurzam X-VCS-Committer-Name: Arthur Zamarin X-VCS-Revision: 09559c09f2389246ea98261832e281a9baaedbdf X-VCS-Branch: master Date: Fri, 20 Jan 2023 20:47:19 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 128e754b-7e2a-494a-ab86-a308fc0b8192 X-Archives-Hash: 4bbc1b763a8b489c111294ed350929b9 commit: 09559c09f2389246ea98261832e281a9baaedbdf Author: Arthur Zamarin gentoo org> AuthorDate: Sat Nov 26 17:06:06 2022 +0000 Commit: Arthur Zamarin gentoo org> CommitDate: Fri Jan 20 20:36:42 2023 +0000 URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcheck.git/commit/?id=09559c09 GitPkgCommitsCheck: catch SRC_URI mistakes Resolves: https://github.com/pkgcore/pkgcheck/issues/493 Signed-off-by: Arthur Zamarin gentoo.org> src/pkgcheck/checks/git.py | 67 +++++++++++++++++++++++++++++++++++++++++++++- tests/checks/test_git.py | 55 ++++++++++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/src/pkgcheck/checks/git.py b/src/pkgcheck/checks/git.py index 6e48d47f..764dfc5d 100644 --- a/src/pkgcheck/checks/git.py +++ b/src/pkgcheck/checks/git.py @@ -13,9 +13,11 @@ from urllib.parse import urlparse from pkgcore.ebuild.misc import sort_keywords from pkgcore.ebuild.repository import UnconfiguredTree +from pkgcore.fetch import fetchable from snakeoil import klass from snakeoil.mappings import ImmutableDict from snakeoil.osutils import pjoin +from snakeoil.sequences import iflatten_instance from snakeoil.strings import pluralism from .. import base, results, sources @@ -169,6 +171,38 @@ class MissingMove(results.PackageResult, results.Error): return f"renamed package: {self.old} -> {self.new}" +class SrcUriChecksumChange(results.PackageResult, results.Error): + """SRC_URI changing checksum without distfile rename.""" + + def __init__(self, filename, **kwargs): + super().__init__(**kwargs) + self.filename = filename + + @property + def desc(self): + return f"{self.filename!r} has different checksums across commits" + + +class SuspiciousSrcUriChange(results.PackageResult, results.Warning): + """Suspicious SRC_URI changing URI without distfile rename.""" + + def __init__(self, old_uri, new_uri, filename, **kwargs): + super().__init__(**kwargs) + if isinstance(old_uri, tuple): + self.old_uri = f"mirror://{old_uri[0].mirror_name}/{old_uri[1]}" + else: + self.old_uri = str(old_uri) + if isinstance(new_uri, tuple): + self.new_uri = f"mirror://{new_uri[0].mirror_name}/{new_uri[1]}" + else: + self.new_uri = str(new_uri) + self.filename = filename + + @property + def desc(self): + return f"{self.filename!r} has changed SRC_URI from {self.old_uri!r} to {self.new_uri!r}" + + class _RemovalRepo(UnconfiguredTree): """Repository of removed packages stored in a temporary directory.""" @@ -235,6 +269,8 @@ class GitPkgCommitsCheck(GentooRepoCheck, GitCommitsCheck): DroppedUnstableKeywords, MissingSlotmove, MissingMove, + SrcUriChecksumChange, + SuspiciousSrcUriChange, ] ) @@ -345,7 +381,34 @@ class GitPkgCommitsCheck(GentooRepoCheck, GitCommitsCheck): else: yield MissingSlotmove(old_slot, new_slot, pkg=new_pkg) - def feed(self, pkgset): + def src_uri_changes(self, pkgset): + pkg = pkgset[0].unversioned_atom + + try: + new_checksums = { + fetch.filename: (fetch.chksums, tuple(fetch.uri._uri_source)) + for pkg in self.repo.match(pkg) + for fetch in iflatten_instance(pkg.fetchables, fetchable) + } + + old_checksums = { + fetch.filename: (fetch.chksums, tuple(fetch.uri._uri_source)) + for pkg in self.modified_repo(pkgset).match(pkg) + for fetch in iflatten_instance(pkg.fetchables, fetchable) + } + except (IndexError, FileNotFoundError, tarfile.ReadError): + # ignore broken ebuild + return + + for filename in old_checksums.keys() & new_checksums.keys(): + old_checksum, old_uri = old_checksums[filename] + new_checksum, new_uri = new_checksums[filename] + if old_checksum != new_checksum: + yield SrcUriChecksumChange(filename, pkg=pkg) + elif old_uri != new_uri: + yield SuspiciousSrcUriChange(old_uri[0], new_uri[0], filename, pkg=pkg) + + def feed(self, pkgset: list[git.GitPkgChange]): # Mapping of commit types to pkgs, available commit types can be seen # under the --diff-filter option in git log parsing support and are # disambiguated as follows: @@ -407,6 +470,8 @@ class GitPkgCommitsCheck(GentooRepoCheck, GitCommitsCheck): if not pkg.maintainers and newly_added: yield DirectNoMaintainer(pkg=pkg) + yield from self.src_uri_changes(pkgset) + class MissingSignOff(results.CommitResult, results.Error): """Local commit with missing sign offs. diff --git a/tests/checks/test_git.py b/tests/checks/test_git.py index 0294f0b3..b69893d8 100644 --- a/tests/checks/test_git.py +++ b/tests/checks/test_git.py @@ -7,7 +7,7 @@ import pytest from pkgcheck.base import PkgcheckUserException from pkgcheck.checks import git as git_mod from pkgcheck.addons.git import GitCommit -from pkgcore.ebuild.cpv import VersionedCPV as CPV +from pkgcore.ebuild.cpv import VersionedCPV as CPV, UnversionedCPV as CP from pkgcore.test.misc import FakeRepo from snakeoil.cli import arghparse from snakeoil.fileutils import touch @@ -650,6 +650,59 @@ class TestGitPkgCommitsCheck(ReportTestCase): self.init_check() self.assertNoReport(self.check, self.source) + def test_checksum_change(self): + distfile = [ + "DIST", + "pkgcheck-1.tar.gz", + "549746", + "BLAKE2B", + "72ed97d93674ffd311978d03ad3738494a752bf1b02bea5eaaaf1b066c48e8c9ec5f82b79baeeabf3e56e618c76614ee6179b7115d1d875364ac6e3fbc3c6028", + "SHA512", + "6a8c135ca44ccbfe15548bd396aba9448c29f60147920b18b8be5aa5fcd1200e0b75bc5de50fc7892ad5460ddad1e7d28a7e44025bdc581a518d136eda8b0df2", + ] + with open(pjoin(self.parent_repo.path, "profiles/thirdpartymirrors"), "a") as f: + f.write("gentoo https://gentoo.org/distfiles\n") + self.parent_repo.create_ebuild("cat/pkg-1", src_uri=f"mirror://gentoo/{distfile[1]}") + with open(pjoin(self.parent_repo.path, "cat/pkg/Manifest"), "w") as f: + f.write(" ".join(distfile) + "\n") + self.parent_git_repo.add_all("cat/pkg: add 1", signoff=True) + # pull changes and change checksum in child repo + self.child_git_repo.run(["git", "pull", "origin", "main"]) + self.child_repo.create_ebuild("cat/pkg-1-r1", src_uri=f"mirror://gentoo/{distfile[1]}") + distfile[-1] = distfile[-1][:-1] + "0" + with open(pjoin(self.child_repo.path, "cat/pkg/Manifest"), "w") as f: + f.write(" ".join(distfile) + "\n") + self.child_git_repo.add_all("cat/pkg: revbump", signoff=True) + self.init_check() + r = self.assertReport(self.check, self.source) + assert r == git_mod.SrcUriChecksumChange(distfile[1], pkg=CP("cat/pkg")) + + def test_src_uri_change(self): + distfile = [ + "DIST", + "pkgcheck-1.tar.gz", + "549746", + "BLAKE2B", + "72ed97d93674ffd311978d03ad3738494a752bf1b02bea5eaaaf1b066c48e8c9ec5f82b79baeeabf3e56e618c76614ee6179b7115d1d875364ac6e3fbc3c6028", + "SHA512", + "6a8c135ca44ccbfe15548bd396aba9448c29f60147920b18b8be5aa5fcd1200e0b75bc5de50fc7892ad5460ddad1e7d28a7e44025bdc581a518d136eda8b0df2", + ] + old_url = f"mirror://gentoo/{distfile[1]}" + new_url = f"https://pkgcore.github.io/pkgcheck/{distfile[1]}" + with open(pjoin(self.parent_repo.path, "profiles/thirdpartymirrors"), "a") as f: + f.write("gentoo https://gentoo.org/distfiles\n") + self.parent_repo.create_ebuild("cat/pkg-1", src_uri=old_url) + with open(pjoin(self.parent_repo.path, "cat/pkg/Manifest"), "w") as f: + f.write(" ".join(distfile) + "\n") + self.parent_git_repo.add_all("cat/pkg: add 1", signoff=True) + # pull changes and change checksum in child repo + self.child_git_repo.run(["git", "pull", "origin", "main"]) + self.child_repo.create_ebuild("cat/pkg-1", src_uri=new_url) + self.child_git_repo.add_all("cat/pkg: change SRC_URI", signoff=True) + self.init_check() + r = self.assertReport(self.check, self.source) + assert r == git_mod.SuspiciousSrcUriChange(old_url, new_url, distfile[1], pkg=CP("cat/pkg")) + class TestGitEclassCommitsCheck(ReportTestCase):