From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id A213B158015 for ; Tue, 26 Dec 2023 21:01:40 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id E71CD2BC014; Tue, 26 Dec 2023 21:01:39 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [140.211.166.183]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id C04CA2BC014 for ; Tue, 26 Dec 2023 21:01:39 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id A5F54335DCC for ; Tue, 26 Dec 2023 21:01:38 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 1895FED2 for ; Tue, 26 Dec 2023 21:01:37 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <1703572948.1db44d18578a7aee58449cb97e1991cb06c915c3.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/_async/, lib/portage/tests/emerge/, ... X-VCS-Repository: proj/portage X-VCS-Files: lib/_emerge/BinpkgFetcher.py lib/portage/dbapi/bintree.py lib/portage/tests/emerge/conftest.py lib/portage/util/_async/AsyncTaskFuture.py X-VCS-Directories: lib/portage/dbapi/ lib/portage/tests/emerge/ lib/portage/util/_async/ lib/_emerge/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: 1db44d18578a7aee58449cb97e1991cb06c915c3 X-VCS-Branch: master Date: Tue, 26 Dec 2023 21:01:37 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 6bd41540-6c08-4126-a3ca-7536bd778b46 X-Archives-Hash: fa18aea0cd81f25e3c127a405bb570cc commit: 1db44d18578a7aee58449cb97e1991cb06c915c3 Author: Zac Medico gentoo org> AuthorDate: Tue Dec 26 06:42:28 2023 +0000 Commit: Zac Medico gentoo org> CommitDate: Tue Dec 26 06:42:28 2023 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=1db44d18 bintree: support file scheme for binhost src-uri Add local file scheme support for binhost src-uri in bintree and BinpkgFetcher. Make BinpkgFetcher use a coroutine to avoid using callbacks. Add AsyncTaskFuture isAlive method that BinpkgFetcher can use to check FileCopier state. Add test command to test emerge -f --getbinpkgonly with local file scheme in binrepos.conf. Also, fix the getbinpkgonly_fetchonly test command so that it does not rename binhost_dir to pkgdir. It seems like it should not do this, and it caused the getbinpkgonly_file_uri command to fail. Bug: https://bugs.gentoo.org/920537 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/BinpkgFetcher.py | 117 +++++++++++++++++++---------- lib/portage/dbapi/bintree.py | 19 +++-- lib/portage/tests/emerge/conftest.py | 21 +++++- lib/portage/util/_async/AsyncTaskFuture.py | 8 +- 4 files changed, 111 insertions(+), 54 deletions(-) diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index 10f9b6e427..01b2bae637 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -1,8 +1,6 @@ -# Copyright 1999-2020 Gentoo Authors +# Copyright 1999-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import functools - from _emerge.AsynchronousLock import AsynchronousLock from _emerge.CompositeTask import CompositeTask from _emerge.SpawnProcess import SpawnProcess @@ -14,6 +12,7 @@ from portage import os from portage.binpkg import get_binpkg_format from portage.exception import FileNotFound from portage.util._async.AsyncTaskFuture import AsyncTaskFuture +from portage.util._async.FileCopier import FileCopier from portage.util._pty import _create_pty_or_pipe @@ -40,6 +39,22 @@ class BinpkgFetcher(CompositeTask): self.pkg_path = self.pkg_allocated_path + ".partial" def _start(self): + self._start_task( + AsyncTaskFuture(future=self._main(), scheduler=self.scheduler), + self._main_exit, + ) + + async def _main(self) -> int: + """ + Main coroutine which saves the binary package to self.pkg_path + and returns the exit status of the fetcher or copier. + + @rtype: int + @return: Exit status of fetcher or copier. + """ + pkg = self.pkg + bintree = pkg.root_config.trees["bintree"] + fetcher = _BinpkgFetcherProcess( background=self.background, logfile=self.logfile, @@ -52,47 +67,67 @@ class BinpkgFetcher(CompositeTask): if not self.pretend: portage.util.ensure_dirs(os.path.dirname(self.pkg_path)) if "distlocks" in self.pkg.root_config.settings.features: - self._start_task( - AsyncTaskFuture(future=fetcher.async_lock()), - functools.partial(self._start_locked, fetcher), - ) - return - - self._start_task(fetcher, self._fetcher_exit) - - def _start_locked(self, fetcher, lock_task): - self._assert_current(lock_task) - if lock_task.cancelled: - self._default_final_exit(lock_task) - return - - lock_task.future.result() - self._start_task(fetcher, self._fetcher_exit) - - def _fetcher_exit(self, fetcher): - self._assert_current(fetcher) - if not self.pretend and fetcher.returncode == os.EX_OK: - fetcher.sync_timestamp() - if fetcher.locked: - self._start_task( - AsyncTaskFuture(future=fetcher.async_unlock()), - functools.partial(self._fetcher_exit_unlocked, fetcher), - ) - else: - self._fetcher_exit_unlocked(fetcher) + await fetcher.async_lock() + + try: + if bintree._remote_has_index: + remote_metadata = bintree._remotepkgs[ + bintree.dbapi._instance_key(pkg.cpv) + ] + rel_uri = remote_metadata.get("PATH") + if not rel_uri: + # Assume that the remote index is out of date. No path should + # never happen in new portage versions. + rel_uri = pkg.cpv + ".tbz2" + remote_base_uri = remote_metadata["BASE_URI"] + uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") + else: + raise FileNotFound("Binary packages index not found") - def _fetcher_exit_unlocked(self, fetcher, unlock_task=None): - if unlock_task is not None: - self._assert_current(unlock_task) - if unlock_task.cancelled: - self._default_final_exit(unlock_task) - return + uri_parsed = urllib_parse_urlparse(uri) - unlock_task.future.result() + copier = None + if not self.pretend and uri_parsed.scheme in ("", "file"): + copier = FileCopier( + src_path=uri_parsed.path, + dest_path=self.pkg_path, + scheduler=self.scheduler, + ) + copier.start() + try: + await copier.async_wait() + copier.future.result() + except FileNotFoundError: + await self.scheduler.async_output( + f"!!! File not found: {uri_parsed.path}\n", + log_file=self.logfile, + background=self.background, + ) + finally: + if copier.isAlive(): + copier.cancel() - self._current_task = None - self.returncode = fetcher.returncode - self._async_wait() + else: + fetcher.start() + try: + await fetcher.async_wait() + finally: + if fetcher.isAlive(): + fetcher.cancel() + + if not self.pretend and fetcher.returncode == os.EX_OK: + fetcher.sync_timestamp() + finally: + if fetcher.locked: + await fetcher.async_unlock() + + return fetcher.returncode if copier is None else copier.returncode + + def _main_exit(self, main_task): + if not main_task.cancelled: + # Use the fetcher or copier returncode. + main_task.returncode = main_task.future.result() + self._default_final_exit(main_task) class _BinpkgFetcherProcess(SpawnProcess): diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index b9f8d6795e..a20c8dfe26 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1411,15 +1411,18 @@ class binarytree: # Don't use urlopen for https, unless # PEP 476 is supported (bug #469888). - if repo.fetchcommand is None and ( - parsed_url.scheme not in ("https",) or _have_pep_476() - ): + if ( + repo.fetchcommand is None or parsed_url.scheme in ("", "file") + ) and (parsed_url.scheme not in ("https",) or _have_pep_476()): try: - f = _urlopen( - url, if_modified_since=local_timestamp, proxies=proxies - ) - if hasattr(f, "headers") and f.headers.get("timestamp", ""): - remote_timestamp = f.headers.get("timestamp") + if parsed_url.scheme in ("", "file"): + f = open(f"{parsed_url.path.rstrip('/')}/Packages", "rb") + else: + f = _urlopen( + url, if_modified_since=local_timestamp, proxies=proxies + ) + if hasattr(f, "headers") and f.headers.get("timestamp", ""): + remote_timestamp = f.headers.get("timestamp") except OSError as err: if ( hasattr(err, "code") and err.code == 304 diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py index c534f5e9d3..d9aec7041e 100644 --- a/lib/portage/tests/emerge/conftest.py +++ b/lib/portage/tests/emerge/conftest.py @@ -814,9 +814,8 @@ def _generate_all_baseline_commands(playground, binhost): ) # Remove binrepos.conf and test PORTAGE_BINHOST. - def _replace_pkgdir_and_rm_binrepos_conf_file(): + def _rm_pkgdir_and_rm_binrepos_conf_file(): shutil.rmtree(pkgdir) - os.rename(binhost_dir, pkgdir) os.unlink(binrepos_conf_file) getbinpkgonly_fetchonly = Emerge( @@ -824,11 +823,25 @@ def _generate_all_baseline_commands(playground, binhost): "--getbinpkgonly", "dev-libs/A", env_mod={"PORTAGE_BINHOST": binhost_uri}, - preparation=_replace_pkgdir_and_rm_binrepos_conf_file, + preparation=_rm_pkgdir_and_rm_binrepos_conf_file, + ) + + # Test bug 920537 binrepos.conf with local file src-uri. + def _rm_pkgdir_and_create_binrepos_conf_with_file_uri(): + shutil.rmtree(pkgdir) + with open(binrepos_conf_file, "w") as f: + f.write("[test-binhost]\n") + f.write(f"sync-uri = file://{binhost_dir}\n") + + getbinpkgonly_file_uri = Emerge( + "-fe", + "--getbinpkgonly", + "dev-libs/A", + preparation=_rm_pkgdir_and_create_binrepos_conf_with_file_uri, ) fetch_sequence = PortageCommandSequence( - make_package, getbinpkgonly, getbinpkgonly_fetchonly + make_package, getbinpkgonly, getbinpkgonly_fetchonly, getbinpkgonly_file_uri ) test_commands["binhost emerge"] = fetch_sequence yield test_commands diff --git a/lib/portage/util/_async/AsyncTaskFuture.py b/lib/portage/util/_async/AsyncTaskFuture.py index 0cd034c971..4c2f7a5712 100644 --- a/lib/portage/util/_async/AsyncTaskFuture.py +++ b/lib/portage/util/_async/AsyncTaskFuture.py @@ -1,4 +1,4 @@ -# Copyright 2018-2021 Gentoo Foundation +# Copyright 2018-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os @@ -20,6 +20,12 @@ class AsyncTaskFuture(AsynchronousTask): self.future = asyncio.ensure_future(self.future, self.scheduler) self.future.add_done_callback(self._done_callback) + def isAlive(self): + """ + Returns True if self.future is an asyncio.Future that is not done. + """ + return isinstance(self.future, asyncio.Future) and not self.future.done() + def _cancel(self): if not self.future.done(): self.future.cancel()