From 9d9c2f85f6a762c719b1b52438b1baa185421a82 Mon Sep 17 00:00:00 2001 From: "Laura F. D" Date: Wed, 22 Aug 2018 22:00:30 +0100 Subject: [PATCH 1/4] Add support for asynchronous waitpid on Linux systems. --- newsfragments/622.feature.rst | 1 + trio/_waitpid_linux.py | 85 ++++++++++++++++++++++++++++++++ trio/tests/test_waitpid_linux.py | 29 +++++++++++ 3 files changed, 115 insertions(+) create mode 100644 newsfragments/622.feature.rst create mode 100644 trio/_waitpid_linux.py create mode 100644 trio/tests/test_waitpid_linux.py diff --git a/newsfragments/622.feature.rst b/newsfragments/622.feature.rst new file mode 100644 index 0000000000..d287877083 --- /dev/null +++ b/newsfragments/622.feature.rst @@ -0,0 +1 @@ +Add support for asynchronous ``waitpid`` on Linux systems. \ No newline at end of file diff --git a/trio/_waitpid_linux.py b/trio/_waitpid_linux.py new file mode 100644 index 0000000000..c54423592d --- /dev/null +++ b/trio/_waitpid_linux.py @@ -0,0 +1,85 @@ +import attr +import functools +import os +import outcome +from typing import Dict, Any + +from . import _core +from ._sync import Event +from ._threads import run_sync_in_worker_thread + +# type: Dict[int, WaitpidResult] +_pending_waitpids = {} + + +@attr.s +class WaitpidResult: + event = attr.ib(default=attr.Factory(Event)) + outcome = attr.ib(default=None) + _cached_result = attr.ib(default=None) + + def unwrap(self): + if self._cached_result is not None: + if isinstance(self._cached_result, BaseException): + raise self._cached_result + return self._cached_result + + try: + self._cached_result = self.outcome.unwrap() + return self._cached_result + except BaseException as e: + self._cached_result = e + raise e from None + + +# https://github.com/python-trio/trio/issues/618 +class StubLimiter: + def release_on_behalf_of(self, x): + pass + + async def acquire_on_behalf_of(self, x): + pass + + +waitpid_limiter = StubLimiter() + + +# adapted from +# https://github.com/python-trio/trio/issues/4#issuecomment-398967572 +async def _task(pid: int) -> None: + """The waitpid thread runner task. This must be spawned as a system + task.""" + partial = functools.partial( + os.waitpid, # function + pid, # pid + 0 # no options + ) + try: + tresult = await run_sync_in_worker_thread( + outcome.capture, + partial, + cancellable=True, + limiter=waitpid_limiter + ) + except Exception as e: + result = _pending_waitpids.pop(pid) + result.outcome = outcome.Error(e) + result.event.set() + raise + else: + result = _pending_waitpids.pop(pid) + result.outcome = tresult + result.event.set() + + +async def waitpid(pid: int) -> Any: + """Waits for a child process with the specified PID to finish running.""" + try: + waiter = _pending_waitpids[pid] + except KeyError: + waiter = WaitpidResult() + _pending_waitpids[pid] = waiter + _core.spawn_system_task(_task, pid) + + await waiter.event.wait() + return waiter.unwrap() diff --git a/trio/tests/test_waitpid_linux.py b/trio/tests/test_waitpid_linux.py new file mode 100644 index 0000000000..91341a94f4 --- /dev/null +++ b/trio/tests/test_waitpid_linux.py @@ -0,0 +1,29 @@ +import sys + +import os +import pytest +import signal + +from .._core.tests.tutil import slow +from .._waitpid_linux import waitpid + +pytestmark = pytest.mark.skipif( + sys.platform != "linux", reason="linux waitpid only works on linux" +) + + +@slow +async def test_waitpid(): + pid = os.spawnvp(os.P_NOWAIT, "/bin/false", ("false",)) + result = await waitpid(pid) + # exit code is a 16-bit int: (code, signal) + assert result == (pid, 256) + + pid2 = os.spawnvp(os.P_NOWAIT, "/bin/sleep", ("/bin/sleep", "1")) + result = await waitpid(pid2) + assert result == (pid2, 0) + + pid3 = os.spawnvp(os.P_NOWAIT, "/bin/sleep", ("/bin/sleep", "5")) + os.kill(pid3, signal.SIGKILL) + result = await waitpid(pid3) + assert result == (pid3, 9) From 210c8c959cc8a60b3b44ee0129e1fc00eb086f44 Mon Sep 17 00:00:00 2001 From: "Laura F. D" Date: Thu, 23 Aug 2018 02:29:05 +0100 Subject: [PATCH 2/4] Add more waitpid tests. --- trio/tests/test_waitpid_linux.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/trio/tests/test_waitpid_linux.py b/trio/tests/test_waitpid_linux.py index 91341a94f4..74de00ba88 100644 --- a/trio/tests/test_waitpid_linux.py +++ b/trio/tests/test_waitpid_linux.py @@ -4,7 +4,7 @@ import pytest import signal -from .._core.tests.tutil import slow +from .. import _core from .._waitpid_linux import waitpid pytestmark = pytest.mark.skipif( @@ -12,14 +12,13 @@ ) -@slow async def test_waitpid(): pid = os.spawnvp(os.P_NOWAIT, "/bin/false", ("false",)) result = await waitpid(pid) # exit code is a 16-bit int: (code, signal) assert result == (pid, 256) - pid2 = os.spawnvp(os.P_NOWAIT, "/bin/sleep", ("/bin/sleep", "1")) + pid2 = os.spawnvp(os.P_NOWAIT, "/bin/true", ("true",)) result = await waitpid(pid2) assert result == (pid2, 0) @@ -27,3 +26,23 @@ async def test_waitpid(): os.kill(pid3, signal.SIGKILL) result = await waitpid(pid3) assert result == (pid3, 9) + + +async def test_waitpid_multiple_accesses(): + pid = os.spawnvp(os.P_NOWAIT, "/bin/sleep", ("/bin/sleep", "5")) + + async def waiter(): + result = await waitpid(pid) + assert result == (pid, 9) + + async with _core.open_nursery() as n: + n.start_soon(waiter) + n.start_soon(waiter) + + os.kill(pid, signal.SIGKILL) + + +async def test_waitpid_no_process(): + with pytest.raises(ChildProcessError): + # this PID probably doesn't exist + await waitpid(100000) From 28f7d525ad9797dd8fe2b1f0788166c3e58f6871 Mon Sep 17 00:00:00 2001 From: "Laura F. D" Date: Wed, 29 Aug 2018 17:24:32 +0100 Subject: [PATCH 3/4] Rearrange files. --- newsfragments/622.feature.rst | 1 - trio/{_waitpid_linux.py => _subprocess/linux_waitpid.py} | 6 +++--- trio/tests/{ => subprocess}/test_waitpid_linux.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 newsfragments/622.feature.rst rename trio/{_waitpid_linux.py => _subprocess/linux_waitpid.py} (95%) rename trio/tests/{ => subprocess}/test_waitpid_linux.py (94%) diff --git a/newsfragments/622.feature.rst b/newsfragments/622.feature.rst deleted file mode 100644 index d287877083..0000000000 --- a/newsfragments/622.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for asynchronous ``waitpid`` on Linux systems. \ No newline at end of file diff --git a/trio/_waitpid_linux.py b/trio/_subprocess/linux_waitpid.py similarity index 95% rename from trio/_waitpid_linux.py rename to trio/_subprocess/linux_waitpid.py index c54423592d..819a7d0c93 100644 --- a/trio/_waitpid_linux.py +++ b/trio/_subprocess/linux_waitpid.py @@ -4,9 +4,9 @@ import outcome from typing import Dict, Any -from . import _core -from ._sync import Event -from ._threads import run_sync_in_worker_thread +from .. import _core +from .._sync import Event +from .._threads import run_sync_in_worker_thread # type: Dict[int, WaitpidResult] _pending_waitpids = {} diff --git a/trio/tests/test_waitpid_linux.py b/trio/tests/subprocess/test_waitpid_linux.py similarity index 94% rename from trio/tests/test_waitpid_linux.py rename to trio/tests/subprocess/test_waitpid_linux.py index 74de00ba88..c37423f73e 100644 --- a/trio/tests/test_waitpid_linux.py +++ b/trio/tests/subprocess/test_waitpid_linux.py @@ -4,8 +4,8 @@ import pytest import signal -from .. import _core -from .._waitpid_linux import waitpid +from ... import _core +from ..._subprocess.linux_waitpid import waitpid pytestmark = pytest.mark.skipif( sys.platform != "linux", reason="linux waitpid only works on linux" From e8b88612b9e4a6eb6178489a6f52d7d0385d04c3 Mon Sep 17 00:00:00 2001 From: "Laura F. D" Date: Tue, 4 Sep 2018 04:10:40 +0100 Subject: [PATCH 4/4] Make requested changes. --- trio/_subprocess/linux_waitpid.py | 58 +++++---------------- trio/tests/subprocess/test_waitpid_linux.py | 29 ++++------- 2 files changed, 24 insertions(+), 63 deletions(-) diff --git a/trio/_subprocess/linux_waitpid.py b/trio/_subprocess/linux_waitpid.py index 819a7d0c93..33cd1c01f1 100644 --- a/trio/_subprocess/linux_waitpid.py +++ b/trio/_subprocess/linux_waitpid.py @@ -2,34 +2,18 @@ import functools import os import outcome -from typing import Dict, Any +from typing import Any from .. import _core from .._sync import Event from .._threads import run_sync_in_worker_thread -# type: Dict[int, WaitpidResult] -_pending_waitpids = {} - @attr.s -class WaitpidResult: +class WaitpidState: + pid = attr.ib() event = attr.ib(default=attr.Factory(Event)) outcome = attr.ib(default=None) - _cached_result = attr.ib(default=None) - - def unwrap(self): - if self._cached_result is not None: - if isinstance(self._cached_result, BaseException): - raise self._cached_result - return self._cached_result - - try: - self._cached_result = self.outcome.unwrap() - return self._cached_result - except BaseException as e: - self._cached_result = e - raise e from None # https://github.com/python-trio/trio/issues/618 @@ -46,40 +30,26 @@ async def acquire_on_behalf_of(self, x): # adapted from # https://github.com/python-trio/trio/issues/4#issuecomment-398967572 -async def _task(pid: int) -> None: +async def _task(state: WaitpidState) -> None: """The waitpid thread runner task. This must be spawned as a system task.""" partial = functools.partial( os.waitpid, # function - pid, # pid + state.pid, # pid 0 # no options ) - try: - tresult = await run_sync_in_worker_thread( - outcome.capture, - partial, - cancellable=True, - limiter=waitpid_limiter - ) - except Exception as e: - result = _pending_waitpids.pop(pid) - result.outcome = outcome.Error(e) - result.event.set() - raise - else: - result = _pending_waitpids.pop(pid) - result.outcome = tresult - result.event.set() + + tresult = await run_sync_in_worker_thread( + outcome.capture, partial, cancellable=True, limiter=waitpid_limiter + ) + state.outcome = tresult + state.event.set() async def waitpid(pid: int) -> Any: """Waits for a child process with the specified PID to finish running.""" - try: - waiter = _pending_waitpids[pid] - except KeyError: - waiter = WaitpidResult() - _pending_waitpids[pid] = waiter - _core.spawn_system_task(_task, pid) + waiter = WaitpidState(pid=pid) + _core.spawn_system_task(_task, waiter) await waiter.event.wait() - return waiter.unwrap() + return waiter.outcome.unwrap() diff --git a/trio/tests/subprocess/test_waitpid_linux.py b/trio/tests/subprocess/test_waitpid_linux.py index c37423f73e..53a37c61a1 100644 --- a/trio/tests/subprocess/test_waitpid_linux.py +++ b/trio/tests/subprocess/test_waitpid_linux.py @@ -16,33 +16,24 @@ async def test_waitpid(): pid = os.spawnvp(os.P_NOWAIT, "/bin/false", ("false",)) result = await waitpid(pid) # exit code is a 16-bit int: (code, signal) - assert result == (pid, 256) + assert result[0] == pid + assert os.WIFEXITED(result[1]) and os.WEXITSTATUS(result[1]) == 1 pid2 = os.spawnvp(os.P_NOWAIT, "/bin/true", ("true",)) result = await waitpid(pid2) - assert result == (pid2, 0) + assert result[0] == pid2 + assert os.WIFEXITED(result[1]) and os.WEXITSTATUS(result[1]) == 0 pid3 = os.spawnvp(os.P_NOWAIT, "/bin/sleep", ("/bin/sleep", "5")) os.kill(pid3, signal.SIGKILL) result = await waitpid(pid3) - assert result == (pid3, 9) - - -async def test_waitpid_multiple_accesses(): - pid = os.spawnvp(os.P_NOWAIT, "/bin/sleep", ("/bin/sleep", "5")) - - async def waiter(): - result = await waitpid(pid) - assert result == (pid, 9) - - async with _core.open_nursery() as n: - n.start_soon(waiter) - n.start_soon(waiter) - - os.kill(pid, signal.SIGKILL) + assert result[0] == pid3 + status = os.WTERMSIG(result[1]) + assert os.WIFSIGNALED(result[1]) and status == 9 async def test_waitpid_no_process(): with pytest.raises(ChildProcessError): - # this PID probably doesn't exist - await waitpid(100000) + # this PID does exist, but it's ourselves + # which doesn't work + await waitpid(os.getpid())