* [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/_async/, lib/portage/tests/emerge/, ...
@ 2023-12-26 21:01 Zac Medico
0 siblings, 0 replies; only message in thread
From: Zac Medico @ 2023-12-26 21:01 UTC (permalink / raw
To: gentoo-commits
commit: 1db44d18578a7aee58449cb97e1991cb06c915c3
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 26 06:42:28 2023 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Dec 26 06:42:28 2023 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=1db44d18
bintree: support file scheme for binhost src-uri
Add local file scheme support for binhost src-uri in bintree
and BinpkgFetcher. Make BinpkgFetcher use a coroutine to avoid
using callbacks. Add AsyncTaskFuture isAlive method that
BinpkgFetcher can use to check FileCopier state. Add test
command to test emerge -f --getbinpkgonly with local file
scheme in binrepos.conf.
Also, fix the getbinpkgonly_fetchonly test command so that
it does not rename binhost_dir to pkgdir. It seems like it
should not do this, and it caused the getbinpkgonly_file_uri
command to fail.
Bug: https://bugs.gentoo.org/920537
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/_emerge/BinpkgFetcher.py | 117 +++++++++++++++++++----------
lib/portage/dbapi/bintree.py | 19 +++--
lib/portage/tests/emerge/conftest.py | 21 +++++-
lib/portage/util/_async/AsyncTaskFuture.py | 8 +-
4 files changed, 111 insertions(+), 54 deletions(-)
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 10f9b6e427..01b2bae637 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,8 +1,6 @@
-# Copyright 1999-2020 Gentoo Authors
+# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import functools
-
from _emerge.AsynchronousLock import AsynchronousLock
from _emerge.CompositeTask import CompositeTask
from _emerge.SpawnProcess import SpawnProcess
@@ -14,6 +12,7 @@ from portage import os
from portage.binpkg import get_binpkg_format
from portage.exception import FileNotFound
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.FileCopier import FileCopier
from portage.util._pty import _create_pty_or_pipe
@@ -40,6 +39,22 @@ class BinpkgFetcher(CompositeTask):
self.pkg_path = self.pkg_allocated_path + ".partial"
def _start(self):
+ self._start_task(
+ AsyncTaskFuture(future=self._main(), scheduler=self.scheduler),
+ self._main_exit,
+ )
+
+ async def _main(self) -> int:
+ """
+ Main coroutine which saves the binary package to self.pkg_path
+ and returns the exit status of the fetcher or copier.
+
+ @rtype: int
+ @return: Exit status of fetcher or copier.
+ """
+ pkg = self.pkg
+ bintree = pkg.root_config.trees["bintree"]
+
fetcher = _BinpkgFetcherProcess(
background=self.background,
logfile=self.logfile,
@@ -52,47 +67,67 @@ class BinpkgFetcher(CompositeTask):
if not self.pretend:
portage.util.ensure_dirs(os.path.dirname(self.pkg_path))
if "distlocks" in self.pkg.root_config.settings.features:
- self._start_task(
- AsyncTaskFuture(future=fetcher.async_lock()),
- functools.partial(self._start_locked, fetcher),
- )
- return
-
- self._start_task(fetcher, self._fetcher_exit)
-
- def _start_locked(self, fetcher, lock_task):
- self._assert_current(lock_task)
- if lock_task.cancelled:
- self._default_final_exit(lock_task)
- return
-
- lock_task.future.result()
- self._start_task(fetcher, self._fetcher_exit)
-
- def _fetcher_exit(self, fetcher):
- self._assert_current(fetcher)
- if not self.pretend and fetcher.returncode == os.EX_OK:
- fetcher.sync_timestamp()
- if fetcher.locked:
- self._start_task(
- AsyncTaskFuture(future=fetcher.async_unlock()),
- functools.partial(self._fetcher_exit_unlocked, fetcher),
- )
- else:
- self._fetcher_exit_unlocked(fetcher)
+ await fetcher.async_lock()
+
+ try:
+ if bintree._remote_has_index:
+ remote_metadata = bintree._remotepkgs[
+ bintree.dbapi._instance_key(pkg.cpv)
+ ]
+ rel_uri = remote_metadata.get("PATH")
+ if not rel_uri:
+ # Assume that the remote index is out of date. No path should
+ # never happen in new portage versions.
+ rel_uri = pkg.cpv + ".tbz2"
+ remote_base_uri = remote_metadata["BASE_URI"]
+ uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
+ else:
+ raise FileNotFound("Binary packages index not found")
- def _fetcher_exit_unlocked(self, fetcher, unlock_task=None):
- if unlock_task is not None:
- self._assert_current(unlock_task)
- if unlock_task.cancelled:
- self._default_final_exit(unlock_task)
- return
+ uri_parsed = urllib_parse_urlparse(uri)
- unlock_task.future.result()
+ copier = None
+ if not self.pretend and uri_parsed.scheme in ("", "file"):
+ copier = FileCopier(
+ src_path=uri_parsed.path,
+ dest_path=self.pkg_path,
+ scheduler=self.scheduler,
+ )
+ copier.start()
+ try:
+ await copier.async_wait()
+ copier.future.result()
+ except FileNotFoundError:
+ await self.scheduler.async_output(
+ f"!!! File not found: {uri_parsed.path}\n",
+ log_file=self.logfile,
+ background=self.background,
+ )
+ finally:
+ if copier.isAlive():
+ copier.cancel()
- self._current_task = None
- self.returncode = fetcher.returncode
- self._async_wait()
+ else:
+ fetcher.start()
+ try:
+ await fetcher.async_wait()
+ finally:
+ if fetcher.isAlive():
+ fetcher.cancel()
+
+ if not self.pretend and fetcher.returncode == os.EX_OK:
+ fetcher.sync_timestamp()
+ finally:
+ if fetcher.locked:
+ await fetcher.async_unlock()
+
+ return fetcher.returncode if copier is None else copier.returncode
+
+ def _main_exit(self, main_task):
+ if not main_task.cancelled:
+ # Use the fetcher or copier returncode.
+ main_task.returncode = main_task.future.result()
+ self._default_final_exit(main_task)
class _BinpkgFetcherProcess(SpawnProcess):
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index b9f8d6795e..a20c8dfe26 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1411,15 +1411,18 @@ class binarytree:
# Don't use urlopen for https, unless
# PEP 476 is supported (bug #469888).
- if repo.fetchcommand is None and (
- parsed_url.scheme not in ("https",) or _have_pep_476()
- ):
+ if (
+ repo.fetchcommand is None or parsed_url.scheme in ("", "file")
+ ) and (parsed_url.scheme not in ("https",) or _have_pep_476()):
try:
- f = _urlopen(
- url, if_modified_since=local_timestamp, proxies=proxies
- )
- if hasattr(f, "headers") and f.headers.get("timestamp", ""):
- remote_timestamp = f.headers.get("timestamp")
+ if parsed_url.scheme in ("", "file"):
+ f = open(f"{parsed_url.path.rstrip('/')}/Packages", "rb")
+ else:
+ f = _urlopen(
+ url, if_modified_since=local_timestamp, proxies=proxies
+ )
+ if hasattr(f, "headers") and f.headers.get("timestamp", ""):
+ remote_timestamp = f.headers.get("timestamp")
except OSError as err:
if (
hasattr(err, "code") and err.code == 304
diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py
index c534f5e9d3..d9aec7041e 100644
--- a/lib/portage/tests/emerge/conftest.py
+++ b/lib/portage/tests/emerge/conftest.py
@@ -814,9 +814,8 @@ def _generate_all_baseline_commands(playground, binhost):
)
# Remove binrepos.conf and test PORTAGE_BINHOST.
- def _replace_pkgdir_and_rm_binrepos_conf_file():
+ def _rm_pkgdir_and_rm_binrepos_conf_file():
shutil.rmtree(pkgdir)
- os.rename(binhost_dir, pkgdir)
os.unlink(binrepos_conf_file)
getbinpkgonly_fetchonly = Emerge(
@@ -824,11 +823,25 @@ def _generate_all_baseline_commands(playground, binhost):
"--getbinpkgonly",
"dev-libs/A",
env_mod={"PORTAGE_BINHOST": binhost_uri},
- preparation=_replace_pkgdir_and_rm_binrepos_conf_file,
+ preparation=_rm_pkgdir_and_rm_binrepos_conf_file,
+ )
+
+ # Test bug 920537 binrepos.conf with local file src-uri.
+ def _rm_pkgdir_and_create_binrepos_conf_with_file_uri():
+ shutil.rmtree(pkgdir)
+ with open(binrepos_conf_file, "w") as f:
+ f.write("[test-binhost]\n")
+ f.write(f"sync-uri = file://{binhost_dir}\n")
+
+ getbinpkgonly_file_uri = Emerge(
+ "-fe",
+ "--getbinpkgonly",
+ "dev-libs/A",
+ preparation=_rm_pkgdir_and_create_binrepos_conf_with_file_uri,
)
fetch_sequence = PortageCommandSequence(
- make_package, getbinpkgonly, getbinpkgonly_fetchonly
+ make_package, getbinpkgonly, getbinpkgonly_fetchonly, getbinpkgonly_file_uri
)
test_commands["binhost emerge"] = fetch_sequence
yield test_commands
diff --git a/lib/portage/util/_async/AsyncTaskFuture.py b/lib/portage/util/_async/AsyncTaskFuture.py
index 0cd034c971..4c2f7a5712 100644
--- a/lib/portage/util/_async/AsyncTaskFuture.py
+++ b/lib/portage/util/_async/AsyncTaskFuture.py
@@ -1,4 +1,4 @@
-# Copyright 2018-2021 Gentoo Foundation
+# Copyright 2018-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -20,6 +20,12 @@ class AsyncTaskFuture(AsynchronousTask):
self.future = asyncio.ensure_future(self.future, self.scheduler)
self.future.add_done_callback(self._done_callback)
+ def isAlive(self):
+ """
+ Returns True if self.future is an asyncio.Future that is not done.
+ """
+ return isinstance(self.future, asyncio.Future) and not self.future.done()
+
def _cancel(self):
if not self.future.done():
self.future.cancel()
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2023-12-26 21:01 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2023-12-26 21:01 [gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/_async/, lib/portage/tests/emerge/, Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox