From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 2A707158020 for ; Mon, 17 Oct 2022 07:48:34 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id D8B1DE0CE2; Mon, 17 Oct 2022 07:48:32 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [140.211.166.183]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id BA5DFE0CE2 for ; Mon, 17 Oct 2022 07:48:32 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 79EF234105A for ; Mon, 17 Oct 2022 07:48:31 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id D32D35A0 for ; Mon, 17 Oct 2022 07:48:29 +0000 (UTC) From: "Arthur Zamarin" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Arthur Zamarin" Message-ID: <1665992733.8ef7ab63c0fbd294b37e851bce709336a1bd6e5a.arthurzam@gentoo> Subject: [gentoo-commits] proj/pkgcore/pkgcheck:master commit in: testdata/data/repos/network/MetadataUrlCheck/RedirectedUrl/, ... X-VCS-Repository: proj/pkgcore/pkgcheck X-VCS-Files: src/pkgcheck/checks/network.py testdata/data/repos/network/MetadataUrlCheck/RedirectedUrl/expected.json testdata/repos/network/MetadataUrlCheck/RedirectedUrl/metadata.xml testdata/repos/network/MetadataUrlCheck/RedirectedUrl/responses.py X-VCS-Directories: testdata/data/repos/network/MetadataUrlCheck/RedirectedUrl/ testdata/repos/network/MetadataUrlCheck/RedirectedUrl/ src/pkgcheck/checks/ X-VCS-Committer: arthurzam X-VCS-Committer-Name: Arthur Zamarin X-VCS-Revision: 8ef7ab63c0fbd294b37e851bce709336a1bd6e5a X-VCS-Branch: master Date: Mon, 17 Oct 2022 07:48:29 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 6e51b5a0-f595-4737-8f7e-3fcd5917d299 X-Archives-Hash: 42a5e7fa8a2012a3e1f67707de64a2e7 commit: 8ef7ab63c0fbd294b37e851bce709336a1bd6e5a Author: Arthur Zamarin gentoo org> AuthorDate: Mon Oct 17 07:23:21 2022 +0000 Commit: Arthur Zamarin gentoo org> CommitDate: Mon Oct 17 07:45:33 2022 +0000 URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcheck.git/commit/?id=8ef7ab63 network: fix results misattributed with same URL When a URL is collected from multiple different attributes, the results weren't updating the selected attribute, which resulted in wrong report. Resolves: https://github.com/pkgcore/pkgcheck/issues/403 Signed-off-by: Arthur Zamarin gentoo.org> src/pkgcheck/checks/network.py | 12 ++++++---- .../MetadataUrlCheck/RedirectedUrl/expected.json | 2 ++ .../MetadataUrlCheck/RedirectedUrl/metadata.xml | 2 ++ .../MetadataUrlCheck/RedirectedUrl/responses.py | 28 ++++++++++++++++------ 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/pkgcheck/checks/network.py b/src/pkgcheck/checks/network.py index cfe76687..cad8e536 100644 --- a/src/pkgcheck/checks/network.py +++ b/src/pkgcheck/checks/network.py @@ -167,7 +167,7 @@ class _UrlCheck(NetworkCheck): result = DeadUrl(attr, url, str(e), pkg=pkg) return result - def task_done(self, pkg, future): + def task_done(self, pkg, attr, future): """Determine the result of a given URL verification task.""" exc = future.exception() if exc is not None: @@ -180,8 +180,10 @@ class _UrlCheck(NetworkCheck): result = future.result() if result is not None: if pkg is not None: - # recreate result object with different pkg target - result = result._create(**result._attrs, pkg=pkg) + # recreate result object with different pkg target and attr + attrs = result._attrs.copy() + attrs['attr'] = attr + result = result._create(**attrs, pkg=pkg) self.results_q.put([result]) def _get_urls(self, pkg): @@ -198,10 +200,10 @@ class _UrlCheck(NetworkCheck): future = futures.get(url) if future is None: future = executor.submit(func, attr, url, **kwargs) - future.add_done_callback(partial(self.task_done, None)) + future.add_done_callback(partial(self.task_done, None, None)) futures[url] = future else: - future.add_done_callback(partial(self.task_done, kwargs['pkg'])) + future.add_done_callback(partial(self.task_done, kwargs['pkg'], attr)) def schedule(self, pkg, executor, futures): """Schedule verification methods to run in separate threads for all flagged URLs.""" diff --git a/testdata/data/repos/network/MetadataUrlCheck/RedirectedUrl/expected.json b/testdata/data/repos/network/MetadataUrlCheck/RedirectedUrl/expected.json index 789b87d2..ef65eafc 100644 --- a/testdata/data/repos/network/MetadataUrlCheck/RedirectedUrl/expected.json +++ b/testdata/data/repos/network/MetadataUrlCheck/RedirectedUrl/expected.json @@ -1 +1,3 @@ {"__class__": "RedirectedUrl", "category": "MetadataUrlCheck", "package": "RedirectedUrl", "version": "0", "attr": "metadata.xml: changelog", "url": "https://github.com/pkgcore/pkgcheck/changelog", "new_url": "https://github.com/pkgcore/pkgcheck/news"} +{"__class__": "RedirectedUrl", "category": "MetadataUrlCheck", "package": "RedirectedUrl", "version": "0", "attr": "metadata.xml: remote-id", "url": "https://github.com/pkgcore/pkgcheck", "new_url": "https://github.com/pkgcore/pkgcheck/"} +{"__class__": "RedirectedUrl", "category": "MetadataUrlCheck", "package": "RedirectedUrl", "version": "0", "attr": "metadata.xml: doc", "url": "https://github.com/pkgcore/pkgcheck", "new_url": "https://github.com/pkgcore/pkgcheck/"} diff --git a/testdata/repos/network/MetadataUrlCheck/RedirectedUrl/metadata.xml b/testdata/repos/network/MetadataUrlCheck/RedirectedUrl/metadata.xml index 429b3b6b..97b42470 100644 --- a/testdata/repos/network/MetadataUrlCheck/RedirectedUrl/metadata.xml +++ b/testdata/repos/network/MetadataUrlCheck/RedirectedUrl/metadata.xml @@ -2,6 +2,8 @@ + pkgcore/pkgcheck + https://github.com/pkgcore/pkgcheck https://github.com/pkgcore/pkgcheck/changelog diff --git a/testdata/repos/network/MetadataUrlCheck/RedirectedUrl/responses.py b/testdata/repos/network/MetadataUrlCheck/RedirectedUrl/responses.py index 560443ee..0c4e3565 100644 --- a/testdata/repos/network/MetadataUrlCheck/RedirectedUrl/responses.py +++ b/testdata/repos/network/MetadataUrlCheck/RedirectedUrl/responses.py @@ -9,11 +9,25 @@ r_hist.url = 'https://github.com/pkgcore/pkgcheck/changelog' r_hist.headers = {'location': 'https://github.com/pkgcore/pkgcheck/news'} r_hist.raw = io.StringIO() -r = Response() -r.status_code = 200 -r.reason = 'OK' -r.url = 'https://github.com/pkgcore/pkgcheck/changelog' -r.raw = io.StringIO() -r.history = [r_hist] +r1 = Response() +r1.status_code = 200 +r1.reason = 'OK' +r1.url = 'https://github.com/pkgcore/pkgcheck/changelog' +r1.raw = io.StringIO() +r1.history = [r_hist] -responses = [r] +r_hist = Response() +r_hist.status_code = 301 +r_hist.reason = 'Moved Permanently' +r_hist.url = 'https://github.com/pkgcore/pkgcheck' +r_hist.headers = {'location': 'https://github.com/pkgcore/pkgcheck/'} +r_hist.raw = io.StringIO() + +r2 = Response() +r2.status_code = 301 +r2.reason = 'OK' +r2.url = 'https://github.com/pkgcore/pkgcheck' +r2.raw = io.StringIO() +r2.history = [r_hist] + +responses = [r1, r2]