From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 8F9311382C5 for ; Thu, 18 Jun 2020 18:06:42 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 7AAF7E089C; Thu, 18 Jun 2020 18:06:40 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 54FCBE089C for ; Thu, 18 Jun 2020 18:06:40 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 5435834F160 for ; Thu, 18 Jun 2020 18:06:36 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 03E6E28F for ; Thu, 18 Jun 2020 18:06:35 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <1592502877.92be5a02e452eb0810d2974bc7fa5ee2056ef8e7.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/ X-VCS-Repository: proj/portage X-VCS-Files: lib/portage/util/futures/_asyncio/process.py lib/portage/util/futures/_asyncio/streams.py X-VCS-Directories: lib/portage/util/futures/_asyncio/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: 92be5a02e452eb0810d2974bc7fa5ee2056ef8e7 X-VCS-Branch: master Date: Thu, 18 Jun 2020 18:06:35 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: b0e22f8e-bca0-470c-8937-feff2d686529 X-Archives-Hash: f68d97a3bce977573ff726fe496d3593 commit: 92be5a02e452eb0810d2974bc7fa5ee2056ef8e7 Author: Zac Medico gentoo org> AuthorDate: Thu Jun 18 05:33:26 2020 +0000 Commit: Zac Medico gentoo org> CommitDate: Thu Jun 18 17:54:37 2020 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=92be5a02 _writer: fix unsafe finally clause (bug 728580) In the coroutine finally clause, do not call remove_writer in cases where fd has been closed and then re-allocated to a concurrent coroutine as in bug 716636. Also, assume that the caller will put the file in non-blocking mode and close the file when done, so that this function is suitable for use within a loop. Bug: https://bugs.gentoo.org/728580 Reviewed-by: Brian Dolbec gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/util/futures/_asyncio/process.py | 11 ++++-- lib/portage/util/futures/_asyncio/streams.py | 50 +++++++++++++--------------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/process.py b/lib/portage/util/futures/_asyncio/process.py index 020164c9b..2d3e9b0fd 100644 --- a/lib/portage/util/futures/_asyncio/process.py +++ b/lib/portage/util/futures/_asyncio/process.py @@ -1,9 +1,12 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import os + import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures:asyncio', + 'portage.util.futures.unix_events:_set_nonblocking', ) from portage.util.futures._asyncio.streams import _reader, _writer from portage.util.futures.compat_coroutine import coroutine, coroutine_return @@ -59,7 +62,11 @@ class _Process(object): if input is not None: if self._proc.stdin is None: raise TypeError('communicate: expected file or int, got {}'.format(type(self._proc.stdin))) - writer = asyncio.ensure_future(_writer(self._proc.stdin, input), loop=self._loop) + stdin = self._proc.stdin + stdin = os.fdopen(stdin, 'wb', 0) if isinstance(stdin, int) else stdin + _set_nonblocking(stdin.fileno()) + writer = asyncio.ensure_future(_writer(stdin, input, loop=self._loop), loop=self._loop) + writer.add_done_callback(lambda writer: stdin.close()) try: yield asyncio.wait(futures + [self.wait()], loop=self._loop) diff --git a/lib/portage/util/futures/_asyncio/streams.py b/lib/portage/util/futures/_asyncio/streams.py index 650a16491..870307e1e 100644 --- a/lib/portage/util/futures/_asyncio/streams.py +++ b/lib/portage/util/futures/_asyncio/streams.py @@ -1,4 +1,4 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -8,7 +8,6 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), '_emerge.PipeReader:PipeReader', 'portage.util.futures:asyncio', - 'portage.util.futures.unix_events:_set_nonblocking', ) from portage.util.futures.compat_coroutine import coroutine @@ -59,38 +58,37 @@ class _Reader(object): @coroutine def _writer(output_file, content, loop=None): """ - Asynchronously write bytes to output file, and close it when - done. If an EnvironmentError other than EAGAIN is encountered, - which typically indicates that the other end of the pipe has - close, the error is raised. This function is a coroutine. + Asynchronously write bytes to output file. The output file is + assumed to be in non-blocking mode. If an EnvironmentError + other than EAGAIN is encountered, which typically indicates that + the other end of the pipe has closed, the error is raised. + This function is a coroutine. - @param output_file: output file descriptor - @type output_file: file or int + @param output_file: output file + @type output_file: file object @param content: content to write @type content: bytes @param loop: asyncio.AbstractEventLoop (or compatible) @type loop: event loop """ - fd = output_file if isinstance(output_file, int) else output_file.fileno() - _set_nonblocking(fd) loop = asyncio._wrap_loop(loop) - try: - while content: + fd = output_file.fileno() + while content: + try: + content = content[os.write(fd, content):] + except EnvironmentError as e: + if e.errno != errno.EAGAIN: + raise waiter = loop.create_future() - loop.add_writer(fd, lambda: waiter.set_result(None)) + loop.add_writer(fd, lambda: waiter.done() or waiter.set_result(None)) try: yield waiter - while content: - try: - content = content[os.write(fd, content):] - except EnvironmentError as e: - if e.errno == errno.EAGAIN: - break - else: - raise finally: - loop.remove_writer(fd) - except GeneratorExit: - raise - finally: - os.close(output_file) if isinstance(output_file, int) else output_file.close() + # The loop and output file may have been closed. + if not loop.is_closed(): + waiter.done() or waiter.cancel() + # Do not call remove_writer in cases where fd has + # been closed and then re-allocated to a concurrent + # coroutine as in bug 716636. + if not output_file.closed: + loop.remove_writer(fd)